diff --git a/native/Cargo.lock b/native/Cargo.lock index 05b673346e..230fc2a535 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -1826,7 +1826,7 @@ dependencies = [ [[package]] name = "datafusion-comet" -version = "0.14.0" +version = "0.15.0" dependencies = [ "arrow", "assertables", @@ -1901,7 +1901,7 @@ dependencies = [ [[package]] name = "datafusion-comet-objectstore-hdfs" -version = "0.14.0" +version = "0.15.0" dependencies = [ "async-trait", "bytes", @@ -1915,7 +1915,7 @@ dependencies = [ [[package]] name = "datafusion-comet-proto" -version = "0.14.0" +version = "0.15.0" dependencies = [ "prost", "prost-build", @@ -1923,7 +1923,7 @@ dependencies = [ [[package]] name = "datafusion-comet-spark-expr" -version = "0.14.0" +version = "0.15.0" dependencies = [ "arrow", "base64", diff --git a/spark/src/main/scala/org/apache/comet/serde/arrays.scala b/spark/src/main/scala/org/apache/comet/serde/arrays.scala index c82018fe6d..cc6e7eb729 100644 --- a/spark/src/main/scala/org/apache/comet/serde/arrays.scala +++ b/spark/src/main/scala/org/apache/comet/serde/arrays.scala @@ -35,12 +35,6 @@ object CometArrayRemove with CometExprShim with ArraysBase { - override def getSupportLevel(expr: ArrayRemove): SupportLevel = - Incompatible( - Some( - "Returns null when element is null instead of removing null elements" + - " (https://github.com/apache/datafusion-comet/issues/3173)")) - override def convert( expr: ArrayRemove, inputs: Seq[Attribute], @@ -55,34 +49,7 @@ object CometArrayRemove val arrayExprProto = exprToProto(expr.left, inputs, binding) val keyExprProto = exprToProto(expr.right, inputs, binding) - val arrayRemoveScalarExpr = - scalarFunctionExprToProto("array_remove_all", arrayExprProto, keyExprProto) - - val isNotNullExpr = createUnaryExpr( - expr, - expr.right, - inputs, - binding, - (builder, unaryExpr) => builder.setIsNotNull(unaryExpr)) - - val nullLiteralProto = exprToProto(Literal(null, expr.right.dataType), Seq.empty) - - if (arrayRemoveScalarExpr.isDefined && isNotNullExpr.isDefined && nullLiteralProto.isDefined) { - val caseWhenExpr = ExprOuterClass.CaseWhen - .newBuilder() - .addWhen(isNotNullExpr.get) - .addThen(arrayRemoveScalarExpr.get) - .setElseExpr(nullLiteralProto.get) - .build() - Some( - ExprOuterClass.Expr - .newBuilder() - .setCaseWhen(caseWhenExpr) - .build()) - } else { - withInfo(expr, expr.children: _*) - None - } + scalarFunctionExprToProto("array_remove_all", arrayExprProto, keyExprProto) } } diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_remove.sql b/spark/src/test/resources/sql-tests/expressions/array/array_remove.sql index aead1fa44d..a5b20c756f 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_remove.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_remove.sql @@ -24,15 +24,15 @@ CREATE TABLE test_array_remove(arr array, val int) USING parquet statement INSERT INTO test_array_remove VALUES (array(1, 2, 3, 2), 2), (array(1, 2, 3), 4), (array(), 1), (NULL, 1), (array(1, NULL, 3), NULL) -query spark_answer_only +query SELECT array_remove(arr, val) FROM test_array_remove -- column + literal -query spark_answer_only +query SELECT array_remove(arr, 2) FROM test_array_remove -- literal + column -query spark_answer_only +query SELECT array_remove(array(1, 2, 3, 2), val) FROM test_array_remove -- literal + literal diff --git a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala index 8e78f9d0b3..020759a7a6 100644 --- a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala @@ -90,7 +90,7 @@ class CometSqlFileTestSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } catch { case e: Exception => - throw new RuntimeException(s"Error executing SQL '$sql'", e) + throw new RuntimeException(s"Error executing SQL '$sql' ${e.getMessage}", e) } case SqlQuery(sql, mode, line) => try { @@ -126,7 +126,7 @@ class CometSqlFileTestSuite extends CometTestBase with AdaptiveSparkPlanHelper { } catch { case e: Exception => - throw new RuntimeException(s"Error executing SQL '$sql'", e) + throw new RuntimeException(s"Error executing SQL '$sql' ${e.getMessage}", e) } } }