From 39c3c4ec72f39cf9d1ce477ec19c62fc233f3b04 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 4 Mar 2026 06:38:47 -0500 Subject: [PATCH 01/15] stash --- native/Cargo.lock | 690 +++++++++++------- native/Cargo.toml | 14 +- native/core/Cargo.toml | 2 +- native/core/src/execution/operators/expand.rs | 8 +- .../src/execution/operators/iceberg_scan.rs | 12 +- .../src/execution/operators/parquet_writer.rs | 12 +- native/core/src/execution/operators/scan.rs | 8 +- native/core/src/execution/planner.rs | 1 + .../src/execution/shuffle/shuffle_writer.rs | 12 +- native/core/src/parquet/encryption_support.rs | 6 +- native/core/src/parquet/parquet_support.rs | 2 +- native/core/src/parquet/schema_adapter.rs | 8 +- 12 files changed, 481 insertions(+), 294 deletions(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index 78fa3fa124..0d40eb7da3 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -151,23 +151,23 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "57.3.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4754a624e5ae42081f464514be454b39711daae0458906dacde5f4c632f33a8" +checksum = "602268ce9f569f282cedb9a9f6bac569b680af47b9b077d515900c03c5d190da" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", + "arrow-arith 58.0.0", + "arrow-array 58.0.0", + "arrow-buffer 58.0.0", + "arrow-cast 58.0.0", "arrow-csv", - "arrow-data", - "arrow-ipc", + "arrow-data 58.0.0", + "arrow-ipc 58.0.0", "arrow-json", - "arrow-ord", + "arrow-ord 58.0.0", "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-schema 58.0.0", + "arrow-select 58.0.0", + "arrow-string 58.0.0", ] [[package]] @@ -176,10 +176,24 @@ version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7b3141e0ec5145a22d8694ea8b6d6f69305971c4fa1c1a13ef0195aef2d678b" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 57.3.0", + "arrow-buffer 57.3.0", + "arrow-data 57.3.0", + "arrow-schema 57.3.0", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-arith" +version = "58.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd53c6bf277dea91f136ae8e3a5d7041b44b5e489e244e637d00ae302051f56f" +dependencies = [ + "arrow-array 58.0.0", + "arrow-buffer 58.0.0", + "arrow-data 58.0.0", + "arrow-schema 58.0.0", "chrono", "num-traits", ] @@ -191,9 +205,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c8955af33b25f3b175ee10af580577280b4bd01f7e823d94c7cdef7cf8c9aef" dependencies = [ "ahash", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 57.3.0", + "arrow-data 57.3.0", + "arrow-schema 57.3.0", + "chrono", + "half", + "hashbrown 0.16.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "58.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e53796e07a6525edaf7dc28b540d477a934aff14af97967ad1d5550878969b9e" +dependencies = [ + "ahash", + "arrow-buffer 58.0.0", + "arrow-data 58.0.0", + "arrow-schema 58.0.0", "chrono", "chrono-tz", "half", @@ -215,18 +247,51 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-buffer" +version = "58.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2c1a85bb2e94ee10b76531d8bc3ce9b7b4c0d508cabfb17d477f63f2617bd20" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + [[package]] name = "arrow-cast" version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "646bbb821e86fd57189c10b4fcdaa941deaf4181924917b0daa92735baa6ada5" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-ord", - "arrow-schema", - "arrow-select", + "arrow-array 57.3.0", + "arrow-buffer 57.3.0", + "arrow-data 57.3.0", + "arrow-ord 57.3.0", + "arrow-schema 57.3.0", + "arrow-select 57.3.0", + "atoi", + "base64", + "chrono", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-cast" +version = "58.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89fb245db6b0e234ed8e15b644edb8664673fefe630575e94e62cd9d489a8a26" +dependencies = [ + "arrow-array 58.0.0", + "arrow-buffer 58.0.0", + "arrow-data 58.0.0", + "arrow-ord 58.0.0", + "arrow-schema 58.0.0", + "arrow-select 58.0.0", "atoi", "base64", "chrono", @@ -239,13 +304,13 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "57.3.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da746f4180004e3ce7b83c977daf6394d768332349d3d913998b10a120b790a" +checksum = "d374882fb465a194462527c0c15a93aa19a554cf690a6b77a26b2a02539937a7" dependencies = [ - "arrow-array", - "arrow-cast", - "arrow-schema", + "arrow-array 58.0.0", + "arrow-cast 58.0.0", + "arrow-schema 58.0.0", "chrono", "csv", "csv-core", @@ -258,8 +323,21 @@ version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fdd994a9d28e6365aa78e15da3f3950c0fdcea6b963a12fa1c391afb637b304" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 57.3.0", + "arrow-schema 57.3.0", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-data" +version = "58.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "189d210bc4244c715fa3ed9e6e22864673cccb73d5da28c2723fb2e527329b33" +dependencies = [ + "arrow-buffer 58.0.0", + "arrow-schema 58.0.0", "half", "num-integer", "num-traits", @@ -271,26 +349,40 @@ version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abf7df950701ab528bf7c0cf7eeadc0445d03ef5d6ffc151eaae6b38a58feff1" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 57.3.0", + "arrow-buffer 57.3.0", + "arrow-data 57.3.0", + "arrow-schema 57.3.0", + "arrow-select 57.3.0", + "flatbuffers", +] + +[[package]] +name = "arrow-ipc" +version = "58.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7968c2e5210c41f4909b2ef76f6e05e172b99021c2def5edf3cc48fdd39d1d6c" +dependencies = [ + "arrow-array 58.0.0", + "arrow-buffer 58.0.0", + "arrow-data 58.0.0", + "arrow-schema 58.0.0", + "arrow-select 58.0.0", "flatbuffers", "lz4_flex", ] [[package]] name = "arrow-json" -version = "57.3.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ff8357658bedc49792b13e2e862b80df908171275f8e6e075c460da5ee4bf86" +checksum = "92111dba5bf900f443488e01f00d8c4ddc2f47f5c50039d18120287b580baa22" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", + "arrow-array 58.0.0", + "arrow-buffer 58.0.0", + "arrow-cast 58.0.0", + "arrow-data 58.0.0", + "arrow-schema 58.0.0", "chrono", "half", "indexmap 2.13.0", @@ -310,23 +402,36 @@ version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d8f1870e03d4cbed632959498bcc84083b5a24bded52905ae1695bd29da45b" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 57.3.0", + "arrow-buffer 57.3.0", + "arrow-data 57.3.0", + "arrow-schema 57.3.0", + "arrow-select 57.3.0", +] + +[[package]] +name = "arrow-ord" +version = "58.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "211136cb253577ee1a6665f741a13136d4e563f64f5093ffd6fb837af90b9495" +dependencies = [ + "arrow-array 58.0.0", + "arrow-buffer 58.0.0", + "arrow-data 58.0.0", + "arrow-schema 58.0.0", + "arrow-select 58.0.0", ] [[package]] name = "arrow-row" -version = "57.3.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18228633bad92bff92a95746bbeb16e5fc318e8382b75619dec26db79e4de4c0" +checksum = "8e0f20145f9f5ea3fe383e2ba7a7487bf19be36aa9dbf5dd6a1f92f657179663" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 58.0.0", + "arrow-buffer 58.0.0", + "arrow-data 58.0.0", + "arrow-schema 58.0.0", "half", ] @@ -335,6 +440,12 @@ name = "arrow-schema" version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c872d36b7bf2a6a6a2b40de9156265f0242910791db366a2c17476ba8330d68" + +[[package]] +name = "arrow-schema" +version = "58.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b47e0ca91cc438d2c7879fe95e0bca5329fff28649e30a88c6f760b1faeddcb" dependencies = [ "bitflags 2.11.0", "serde_core", @@ -348,10 +459,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68bf3e3efbd1278f770d67e5dc410257300b161b93baedb3aae836144edcaf4b" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 57.3.0", + "arrow-buffer 57.3.0", + "arrow-data 57.3.0", + "arrow-schema 57.3.0", + "num-traits", +] + +[[package]] +name = "arrow-select" +version = "58.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "750a7d1dda177735f5e82a314485b6915c7cccdbb278262ac44090f4aba4a325" +dependencies = [ + "ahash", + "arrow-array 58.0.0", + "arrow-buffer 58.0.0", + "arrow-data 58.0.0", + "arrow-schema 58.0.0", "num-traits", ] @@ -361,11 +486,28 @@ version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85e968097061b3c0e9fe3079cf2e703e487890700546b5b0647f60fca1b5a8d8" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 57.3.0", + "arrow-buffer 57.3.0", + "arrow-data 57.3.0", + "arrow-schema 57.3.0", + "arrow-select 57.3.0", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "arrow-string" +version = "58.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eab1208bc4fe55d768cdc9b9f3d9df5a794cdb3ee2586bf89f9b30dc31ad8c" +dependencies = [ + "arrow-array 58.0.0", + "arrow-buffer 58.0.0", + "arrow-data 58.0.0", + "arrow-schema 58.0.0", + "arrow-select 58.0.0", "memchr", "num-traits", "regex", @@ -589,9 +731,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.16.0" +version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9a7b350e3bb1767102698302bc37256cbd48422809984b98d292c40e2579aa9" +checksum = "94bffc006df10ac2a68c83692d734a465f8ee6c5b384d8545a636f81d858f4bf" dependencies = [ "aws-lc-sys", "zeroize", @@ -599,9 +741,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.37.1" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b092fe214090261288111db7a2b2c2118e5a7f30dc2569f1732c4069a6840549" +checksum = "4321e568ed89bb5a7d291a7f37997c2c0df89809d7b6d12062c81ddb54aa782e" dependencies = [ "cc", "cmake", @@ -1682,12 +1824,11 @@ dependencies = [ [[package]] name = "datafusion" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503f1f4a9060ae6e650d3dff5dc7a21266fea1302d890768d45b4b28586e830f" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", - "arrow-schema", + "arrow-schema 58.0.0", "async-trait", "bytes", "chrono", @@ -1719,9 +1860,9 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.13.1", "parking_lot", - "parquet", + "parquet 58.0.0", "rand 0.9.2", "regex", "sqlparser", @@ -1733,9 +1874,8 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14417a3ee4ae3d092b56cd6c1d32e8ff3e2c9ec130ecb2276ec91c89fd599399" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", "async-trait", @@ -1751,16 +1891,15 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.13.1", "parking_lot", "tokio", ] [[package]] name = "datafusion-catalog-listing" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d0eba824adb45a4b3ac6f0251d40df3f6a9382371cad136f4f14ac9ebc6bc10" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", "async-trait", @@ -1776,7 +1915,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.13.1", ] [[package]] @@ -1812,12 +1951,12 @@ dependencies = [ "lz4_flex", "mimalloc", "num", - "object_store", + "object_store 0.13.1", "object_store_opendal", "once_cell", "opendal", "parking_lot", - "parquet", + "parquet 58.0.0", "paste", "pprof", "procfs", @@ -1863,7 +2002,7 @@ dependencies = [ "datafusion-comet-fs-hdfs3", "fs-hdfs3", "futures", - "object_store", + "object_store 0.13.1", "tokio", ] @@ -1898,22 +2037,22 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0039deefbd00c56adf5168b7ca58568fb058e4ba4c5a03b09f8be371b4e434b6" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "ahash", "arrow", - "arrow-ipc", + "arrow-ipc 58.0.0", "chrono", "half", "hashbrown 0.16.1", "hex", "indexmap 2.13.0", + "itertools 0.14.0", "libc", "log", - "object_store", - "parquet", + "object_store 0.13.1", + "parquet 58.0.0", "paste", "sqlparser", "tokio", @@ -1922,9 +2061,8 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec7e3e60b813048331f8fb9673583173e5d2dd8fef862834ee871fc98b57ca7" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "futures", "log", @@ -1933,9 +2071,8 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "802068957f620302ecf05f84ff4019601aeafd36f5f3f1334984af2e34265129" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", "async-compression", @@ -1958,7 +2095,7 @@ dependencies = [ "itertools 0.14.0", "liblzma", "log", - "object_store", + "object_store 0.13.1", "rand 0.9.2", "tokio", "tokio-util", @@ -1968,12 +2105,11 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fc387d5067c62d494a6647d29c5ad4fcdd5a6e50ab4ea1d2568caa2d66f2cc" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", - "arrow-ipc", + "arrow-ipc 58.0.0", "async-trait", "bytes", "datafusion-common", @@ -1986,15 +2122,14 @@ dependencies = [ "datafusion-session", "futures", "itertools 0.14.0", - "object_store", + "object_store 0.13.1", "tokio", ] [[package]] name = "datafusion-datasource-csv" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd5e20579bb6c8bd4e6c620253972fb723822030c280dd6aa047f660d09eeba" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", "async-trait", @@ -2008,16 +2143,15 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "object_store", + "object_store 0.13.1", "regex", "tokio", ] [[package]] name = "datafusion-datasource-json" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0788b0d48fcef31880a02013ea3cc18e5a4e0eacc3b0abdd2cd0597b99dc96e" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", "async-trait", @@ -2031,15 +2165,16 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "object_store", + "object_store 0.13.1", + "serde_json", "tokio", + "tokio-stream", ] [[package]] name = "datafusion-datasource-parquet" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66639b70f1f363f5f0950733170100e588f1acfacac90c1894e231194aa35957" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", "async-trait", @@ -2059,35 +2194,35 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.13.1", "parking_lot", - "parquet", + "parquet 58.0.0", "tokio", ] [[package]] name = "datafusion-doc" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e44b41f3e8267c6cf3eec982d63f34db9f1dd5f30abfd2e1f124f0871708952e" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" [[package]] name = "datafusion-execution" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e456f60e5d38db45335e84617006d90af14a8c8c5b8e959add708b2daaa0e2c" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", + "arrow-buffer 58.0.0", "async-trait", "chrono", "dashmap", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr-common", "futures", "log", - "object_store", + "object_store 0.13.1", "parking_lot", - "parquet", + "parquet 58.0.0", "rand 0.9.2", "tempfile", "url", @@ -2095,9 +2230,8 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6507c719804265a58043134580c1c20767e7c23ba450724393f03ec982769ad9" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", "async-trait", @@ -2117,9 +2251,8 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a413caa9c5885072b539337aed68488f0291653e8edd7d676c92df2480f6cab0" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", "datafusion-common", @@ -2130,12 +2263,11 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "189256495dc9cbbb8e20dbcf161f60422e628d201a78df8207e44bd4baefadb6" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", - "arrow-buffer", + "arrow-buffer 58.0.0", "base64", "blake2", "blake3", @@ -2151,6 +2283,7 @@ dependencies = [ "itertools 0.14.0", "log", "md-5", + "memchr", "num-traits", "rand 0.9.2", "regex", @@ -2161,9 +2294,8 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12e73dfee4cd67c4a507ffff4c5a711d39983adf544adbc09c09bf06f789f413" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "ahash", "arrow", @@ -2177,14 +2309,14 @@ dependencies = [ "datafusion-physical-expr-common", "half", "log", + "num-traits", "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87727bd9e65f4f9ac6d608c9810b7da9eaa3b18b26a4a4b76520592d49020acf" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "ahash", "arrow", @@ -2195,12 +2327,11 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5ef761359224b7c2b5a1bfad6296ac63225f8583d08ad18af9ba1a89ac3887" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", - "arrow-ord", + "arrow-ord 58.0.0", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -2211,16 +2342,17 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", + "hashbrown 0.16.1", "itertools 0.14.0", + "itoa", "log", "paste", ] [[package]] name = "datafusion-functions-table" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b17dac25dfda2d2a90ff0ad1c054a11fb1523766226bec6e9bd8c410daee2ae" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", "async-trait", @@ -2234,9 +2366,8 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c594a29ddb22cbdbce500e4d99b5b2392c5cecb4c1086298b41d1ffec14dbb77" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", "datafusion-common", @@ -2252,9 +2383,8 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aa1b15ed81c7543f62264a30dd49dec4b1b0b698053b968f53be32dfba4f729" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2262,9 +2392,8 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00c31c4795597aa25b74cab5174ac07a53051f27ce1e011ecaffa9eaeecef81" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "datafusion-doc", "quote", @@ -2273,9 +2402,8 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80ccf60767c09302b2e0fc3afebb3761a6d508d07316fab8c5e93312728a21bb" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", "chrono", @@ -2292,9 +2420,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c64b7f277556944e4edd3558da01d9e9ff9f5416f1c0aa7fee088e57bd141a7e" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "ahash", "arrow", @@ -2315,9 +2442,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7abaee372ea2d19c016ee9ef8629c4415257d291cdd152bc7f0b75f28af1b63" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", "datafusion-common", @@ -2330,9 +2456,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42237efe621f92adc22d111b531fdbc2cc38ca9b5e02327535628fb103ae2157" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "ahash", "arrow", @@ -2347,9 +2472,8 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd093498bd1319c6e5c76e9dfa905e78486f01b34579ce97f2e3a49f84c37fac" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", "datafusion-common", @@ -2365,14 +2489,13 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cbe61b12daf81a9f20ba03bd3541165d51f86e004ef37426b11881330eed261" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "ahash", "arrow", - "arrow-ord", - "arrow-schema", + "arrow-ord 58.0.0", + "arrow-schema 58.0.0", "async-trait", "datafusion-common", "datafusion-common-runtime", @@ -2389,6 +2512,7 @@ dependencies = [ "indexmap 2.13.0", "itertools 0.14.0", "log", + "num-traits", "parking_lot", "pin-project-lite", "tokio", @@ -2396,9 +2520,8 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0124331116db7f79df92ebfd2c3b11a8f90240f253555c9bb084f10b6fecf1dd" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", "datafusion-common", @@ -2413,9 +2536,8 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1673e3c58ba618a6ea0568672f00664087b8982c581e9afd5aa6c3c79c9b431f" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "async-trait", "datafusion-common", @@ -2427,38 +2549,41 @@ dependencies = [ [[package]] name = "datafusion-spark" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15d28510abfc85709578fcf9065325d43ee3303012c0ccec2dce351bdc577d00" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", "bigdecimal", "chrono", "crc32fast", + "datafusion", "datafusion-catalog", "datafusion-common", "datafusion-execution", "datafusion-expr", "datafusion-functions", + "datafusion-functions-aggregate", "datafusion-functions-nested", "log", "percent-encoding", "rand 0.9.2", + "serde_json", "sha1", + "sha2", "url", ] [[package]] name = "datafusion-sql" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5272d256dab5347bb39d2040589f45d8c6b715b27edcb5fffe88cc8b9c3909cb" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", "bigdecimal", "chrono", "datafusion-common", "datafusion-expr", + "datafusion-functions-nested", "indexmap 2.13.0", "log", "regex", @@ -2926,20 +3051,20 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 6.0.0", "rand_core 0.10.0", "wasip2", "wasip3", @@ -3254,14 +3379,14 @@ dependencies = [ "anyhow", "apache-avro", "array-init", - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-ord", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-arith 57.3.0", + "arrow-array 57.3.0", + "arrow-buffer 57.3.0", + "arrow-cast 57.3.0", + "arrow-ord 57.3.0", + "arrow-schema 57.3.0", + "arrow-select 57.3.0", + "arrow-string 57.3.0", "as-any", "async-trait", "backon", @@ -3281,7 +3406,7 @@ dependencies = [ "once_cell", "opendal", "ordered-float 4.6.0", - "parquet", + "parquet 57.3.0", "rand 0.8.5", "reqsign", "reqwest", @@ -3473,9 +3598,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.11.0" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "iri-string" @@ -3533,9 +3658,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.22" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819b44bc7c87d9117eb522f14d46e918add69ff12713c475946b0a29363ed1c2" +checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" dependencies = [ "jiff-static", "jiff-tzdb-platform", @@ -3548,9 +3673,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.22" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "470252db18ecc35fd766c0891b1e3ec6cbbcd62507e85276c01bf75d8e94d4a1" +checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" dependencies = [ "proc-macro2", "quote", @@ -3559,9 +3684,9 @@ dependencies = [ [[package]] name = "jiff-tzdb" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68971ebff725b9e2ca27a601c5eb38a4c5d64422c4cbab0c535f248087eda5c2" +checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" [[package]] name = "jiff-tzdb-platform" @@ -4080,6 +4205,30 @@ name = "object_store" version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbfbfff40aeccab00ec8a910b57ca8ecf4319b335c542f2edcd19dd25a1e2a00" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures", + "http 1.4.0", + "humantime", + "itertools 0.14.0", + "parking_lot", + "percent-encoding", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "object_store" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2858065e55c148d294a9f3aae3b0fa9458edadb41a108397094566f4e3c0dfb" dependencies = [ "async-trait", "base64", @@ -4100,7 +4249,7 @@ dependencies = [ "rand 0.9.2", "reqwest", "ring", - "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", @@ -4123,7 +4272,7 @@ dependencies = [ "bytes", "chrono", "futures", - "object_store", + "object_store 0.12.5", "opendal", "pin-project", "tokio", @@ -4247,13 +4396,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ee96b29972a257b855ff2341b37e61af5f12d6af1158b6dcdb5b31ea07bb3cb" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ipc", - "arrow-schema", - "arrow-select", + "arrow-array 57.3.0", + "arrow-buffer 57.3.0", + "arrow-cast 57.3.0", + "arrow-data 57.3.0", + "arrow-ipc 57.3.0", + "arrow-schema 57.3.0", + "arrow-select 57.3.0", "base64", "brotli", "bytes", @@ -4266,7 +4415,42 @@ dependencies = [ "num-bigint", "num-integer", "num-traits", - "object_store", + "paste", + "seq-macro", + "simdutf8", + "snap", + "thrift", + "tokio", + "twox-hash", + "zstd", +] + +[[package]] +name = "parquet" +version = "58.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f491d0ef1b510194426ee67ddc18a9b747ef3c42050c19322a2cd2e1666c29b" +dependencies = [ + "ahash", + "arrow-array 58.0.0", + "arrow-buffer 58.0.0", + "arrow-data 58.0.0", + "arrow-ipc 58.0.0", + "arrow-schema 58.0.0", + "arrow-select 58.0.0", + "base64", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.16.1", + "lz4_flex", + "num-bigint", + "num-integer", + "num-traits", + "object_store 0.13.1", "parquet-variant", "parquet-variant-compute", "parquet-variant-json", @@ -4283,11 +4467,11 @@ dependencies = [ [[package]] name = "parquet-variant" -version = "57.3.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6c31f8f9bfefb9dbf67b0807e00fd918676954a7477c889be971ac904103184" +checksum = "00ba4e5dcbc8ad65882b7337a95c12a0f9cbb6add237c53d93b803b7d7f70f02" dependencies = [ - "arrow-schema", + "arrow-schema 58.0.0", "chrono", "half", "indexmap 2.13.0", @@ -4297,27 +4481,28 @@ dependencies = [ [[package]] name = "parquet-variant-compute" -version = "57.3.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196cd9f7178fed3ac8d5e6d2b51193818e896bbc3640aea3fde3440114a8f39c" +checksum = "9ec4cfb8da15565c8d211b6bc51e8eb481ea65d19132462af3f948b150ac8efe" dependencies = [ "arrow", - "arrow-schema", + "arrow-schema 58.0.0", "chrono", "half", "indexmap 2.13.0", "parquet-variant", "parquet-variant-json", + "serde_json", "uuid", ] [[package]] name = "parquet-variant-json" -version = "57.3.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed23d7acc90ef60f7fdbcc473fa2fdaefa33542ed15b84388959346d52c839be" +checksum = "3668ff00a6aeb29d172ba15f9d8fedf1675d79bff7d1916daa333efdeaa13e46" dependencies = [ - "arrow-schema", + "arrow-schema 58.0.0", "base64", "chrono", "parquet-variant", @@ -4711,9 +4896,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.44" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -4724,6 +4909,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rand" version = "0.8.5" @@ -4752,7 +4943,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" dependencies = [ "chacha20", - "getrandom 0.4.1", + "getrandom 0.4.2", "rand_core 0.10.0", ] @@ -5079,15 +5270,6 @@ dependencies = [ "security-framework", ] -[[package]] -name = "rustls-pemfile" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "rustls-pki-types" version = "1.14.0" @@ -5439,9 +5621,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.59.0" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4591acadbcf52f0af60eafbb2c003232b2b4cd8de5f0e9437cb8b1b59046cc0f" +checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" dependencies = [ "log", "sqlparser_derive", @@ -5449,9 +5631,9 @@ dependencies = [ [[package]] name = "sqlparser_derive" -version = "0.3.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" +checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", @@ -5581,7 +5763,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0" dependencies = [ "fastrand", - "getrandom 0.4.1", + "getrandom 0.4.2", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", @@ -5756,9 +5938,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.49.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" dependencies = [ "bytes", "libc", @@ -5792,6 +5974,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -6036,7 +6230,7 @@ version = "1.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb" dependencies = [ - "getrandom 0.4.1", + "getrandom 0.4.2", "js-sys", "serde_core", "wasm-bindgen", diff --git a/native/Cargo.toml b/native/Cargo.toml index d5a6aeabc9..4067956722 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -34,14 +34,14 @@ edition = "2021" rust-version = "1.88" [workspace.dependencies] -arrow = { version = "57.3.0", features = ["prettyprint", "ffi", "chrono-tz"] } +arrow = { version = "58.0.0", features = ["prettyprint", "ffi", "chrono-tz"] } async-trait = { version = "0.1" } bytes = { version = "1.11.1" } -parquet = { version = "57.3.0", default-features = false, features = ["experimental"] } -datafusion = { version = "52.2.0", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } -datafusion-datasource = { version = "52.2.0" } -datafusion-physical-expr-adapter = { version = "52.2.0" } -datafusion-spark = { version = "52.2.0" } +parquet = { version = "58.0.0", default-features = false, features = ["experimental"] } +datafusion = { git = "https://github.com/apache/datafusion", branch = "branch-53", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } +datafusion-datasource = { git = "https://github.com/apache/datafusion", branch = "branch-53" } +datafusion-physical-expr-adapter = { git = "https://github.com/apache/datafusion", branch = "branch-53" } +datafusion-spark = { git = "https://github.com/apache/datafusion", branch = "branch-53", features = ["core"] } datafusion-comet-spark-expr = { path = "spark-expr" } datafusion-comet-proto = { path = "proto" } chrono = { version = "0.4", default-features = false, features = ["clock"] } @@ -51,7 +51,7 @@ num = "0.4" rand = "0.10" regex = "1.12.3" thiserror = "2" -object_store = { version = "0.12.3", features = ["gcp", "azure", "aws", "http"] } +object_store = { version = "0.13.1", features = ["gcp", "azure", "aws", "http"] } url = "2.2" aws-config = "1.8.14" aws-credential-types = "1.2.13" diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml index cbe397b12b..23a78aa3ee 100644 --- a/native/core/Cargo.toml +++ b/native/core/Cargo.toml @@ -96,7 +96,7 @@ jni = { version = "0.21", features = ["invocation"] } lazy_static = "1.4" assertables = "9" hex = "0.4.3" -datafusion-functions-nested = { version = "52.2.0" } +datafusion-functions-nested = { git = "https://github.com/apache/datafusion", branch = "branch-53" } [features] backtrace = ["datafusion/backtrace"] diff --git a/native/core/src/execution/operators/expand.rs b/native/core/src/execution/operators/expand.rs index 19ca204592..e06fab23ec 100644 --- a/native/core/src/execution/operators/expand.rs +++ b/native/core/src/execution/operators/expand.rs @@ -42,7 +42,7 @@ pub struct ExpandExec { projections: Vec>>, child: Arc, schema: SchemaRef, - cache: PlanProperties, + cache: Arc, } impl ExpandExec { @@ -52,12 +52,12 @@ impl ExpandExec { child: Arc, schema: SchemaRef, ) -> Self { - let cache = PlanProperties::new( + let cache = Arc::new(PlanProperties::new( EquivalenceProperties::new(Arc::clone(&schema)), Partitioning::UnknownPartitioning(1), EmissionType::Final, Boundedness::Bounded, - ); + )); Self { projections, @@ -129,7 +129,7 @@ impl ExecutionPlan for ExpandExec { Ok(Box::pin(expand_stream)) } - fn properties(&self) -> &PlanProperties { + fn properties(&self) -> &Arc { &self.cache } diff --git a/native/core/src/execution/operators/iceberg_scan.rs b/native/core/src/execution/operators/iceberg_scan.rs index 720a4c09a4..8409545763 100644 --- a/native/core/src/execution/operators/iceberg_scan.rs +++ b/native/core/src/execution/operators/iceberg_scan.rs @@ -57,7 +57,7 @@ pub struct IcebergScanExec { /// Output schema after projection output_schema: SchemaRef, /// Cached execution plan properties - plan_properties: PlanProperties, + plan_properties: Arc, /// Catalog-specific configuration for FileIO catalog_properties: HashMap, /// Pre-planned file scan tasks @@ -92,13 +92,13 @@ impl IcebergScanExec { }) } - fn compute_properties(schema: SchemaRef, num_partitions: usize) -> PlanProperties { - PlanProperties::new( + fn compute_properties(schema: SchemaRef, num_partitions: usize) -> Arc { + Arc::new(PlanProperties::new( EquivalenceProperties::new(schema), Partitioning::UnknownPartitioning(num_partitions), EmissionType::Incremental, Boundedness::Bounded, - ) + )) } } @@ -115,7 +115,7 @@ impl ExecutionPlan for IcebergScanExec { Arc::clone(&self.output_schema) } - fn properties(&self) -> &PlanProperties { + fn properties(&self) -> &Arc { &self.plan_properties } @@ -269,7 +269,7 @@ where _ => { let adapter = self .adapter_factory - .create(Arc::clone(&self.schema), Arc::clone(&file_schema)); + .create(Arc::clone(&self.schema), Arc::clone(&file_schema))?; let exprs = build_projection_expressions(&self.schema, &adapter).map_err(|e| { DataFusionError::Execution(format!( diff --git a/native/core/src/execution/operators/parquet_writer.rs b/native/core/src/execution/operators/parquet_writer.rs index 4a53ff51b8..7b53fbc4bc 100644 --- a/native/core/src/execution/operators/parquet_writer.rs +++ b/native/core/src/execution/operators/parquet_writer.rs @@ -208,7 +208,7 @@ pub struct ParquetWriterExec { /// Metrics metrics: ExecutionPlanMetricsSet, /// Cache for plan properties - cache: PlanProperties, + cache: Arc, } impl ParquetWriterExec { @@ -228,12 +228,12 @@ impl ParquetWriterExec { // Preserve the input's partitioning so each partition writes its own file let input_partitioning = input.output_partitioning().clone(); - let cache = PlanProperties::new( + let cache = Arc::new(PlanProperties::new( EquivalenceProperties::new(Arc::clone(&input.schema())), input_partitioning, EmissionType::Final, Boundedness::Bounded, - ); + )); Ok(ParquetWriterExec { input, @@ -405,11 +405,7 @@ impl ExecutionPlan for ParquetWriterExec { Some(self.metrics.clone_inner()) } - fn statistics(&self) -> Result { - self.input.partition_statistics(None) - } - - fn properties(&self) -> &PlanProperties { + fn properties(&self) -> &Arc { &self.cache } diff --git a/native/core/src/execution/operators/scan.rs b/native/core/src/execution/operators/scan.rs index 2543705fb0..dbebbe25be 100644 --- a/native/core/src/execution/operators/scan.rs +++ b/native/core/src/execution/operators/scan.rs @@ -72,7 +72,7 @@ pub struct ScanExec { /// It is also used in unit test to mock the input data from JVM. pub batch: Arc>>, /// Cache of expensive-to-compute plan properties - cache: PlanProperties, + cache: Arc, /// Metrics collector metrics: ExecutionPlanMetricsSet, /// Baseline metrics @@ -95,14 +95,14 @@ impl ScanExec { // Build schema directly from data types since get_next now always unpacks dictionaries let schema = schema_from_data_types(&data_types); - let cache = PlanProperties::new( + let cache = Arc::new(PlanProperties::new( EquivalenceProperties::new(Arc::clone(&schema)), // The partitioning is not important because we are not using DataFusion's // query planner or optimizer Partitioning::UnknownPartitioning(1), EmissionType::Final, Boundedness::Bounded, - ); + )); Ok(Self { exec_context_id, @@ -417,7 +417,7 @@ impl ExecutionPlan for ScanExec { ))) } - fn properties(&self) -> &PlanProperties { + fn properties(&self) -> &Arc { &self.cache } diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 094777e796..3eba167e41 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -1577,6 +1577,7 @@ impl PhysicalPlanner { // null doesn't equal to null in Spark join key. If the join key is // `EqualNullSafe`, Spark will rewrite it during planning. NullEquality::NullEqualsNothing, + false, )?); // If the hash join is build right, we need to swap the left and right diff --git a/native/core/src/execution/shuffle/shuffle_writer.rs b/native/core/src/execution/shuffle/shuffle_writer.rs index fe1bf0fccf..8327f04654 100644 --- a/native/core/src/execution/shuffle/shuffle_writer.rs +++ b/native/core/src/execution/shuffle/shuffle_writer.rs @@ -62,7 +62,7 @@ pub struct ShuffleWriterExec { /// Metrics metrics: ExecutionPlanMetricsSet, /// Cache for expensive-to-compute plan properties - cache: PlanProperties, + cache: Arc, /// The compression codec to use when compressing shuffle blocks codec: CompressionCodec, tracing_enabled: bool, @@ -82,12 +82,12 @@ impl ShuffleWriterExec { tracing_enabled: bool, write_buffer_size: usize, ) -> Result { - let cache = PlanProperties::new( + let cache = Arc::new(PlanProperties::new( EquivalenceProperties::new(Arc::clone(&input.schema())), Partitioning::UnknownPartitioning(1), EmissionType::Final, Boundedness::Bounded, - ); + )); Ok(ShuffleWriterExec { input, @@ -133,11 +133,7 @@ impl ExecutionPlan for ShuffleWriterExec { Some(self.metrics.clone_inner()) } - fn statistics(&self) -> Result { - self.input.partition_statistics(None) - } - - fn properties(&self) -> &PlanProperties { + fn properties(&self) -> &Arc { &self.cache } diff --git a/native/core/src/parquet/encryption_support.rs b/native/core/src/parquet/encryption_support.rs index 4540c217d5..f62c04b854 100644 --- a/native/core/src/parquet/encryption_support.rs +++ b/native/core/src/parquet/encryption_support.rs @@ -19,7 +19,7 @@ use crate::execution::operators::ExecutionError; use crate::jvm_bridge::{check_exception, JVMClasses}; use arrow::datatypes::SchemaRef; use async_trait::async_trait; -use datafusion::common::extensions_options; +use datafusion::common::{extensions_options, Result as DataFusionResult}; use datafusion::config::EncryptionFactoryOptions; use datafusion::error::DataFusionError; use datafusion::execution::parquet_encryption::EncryptionFactory; @@ -54,7 +54,7 @@ impl EncryptionFactory for CometEncryptionFactory { _options: &EncryptionFactoryOptions, _schema: &SchemaRef, _file_path: &Path, - ) -> Result>, DataFusionError> { + ) -> DataFusionResult>> { Err(DataFusionError::NotImplemented( "Comet does not support Parquet encryption yet." .parse() @@ -69,7 +69,7 @@ impl EncryptionFactory for CometEncryptionFactory { &self, options: &EncryptionFactoryOptions, file_path: &Path, - ) -> Result>, DataFusionError> { + ) -> DataFusionResult>> { let config: CometEncryptionConfig = options.to_extension_options()?; let full_path: String = config.uri_base + file_path.as_ref(); diff --git a/native/core/src/parquet/parquet_support.rs b/native/core/src/parquet/parquet_support.rs index e7ff5630f1..e1c4a1ec7c 100644 --- a/native/core/src/parquet/parquet_support.rs +++ b/native/core/src/parquet/parquet_support.rs @@ -477,7 +477,7 @@ pub(crate) fn prepare_object_store_with_configs( .map_err(|e| ExecutionError::GeneralError(e.to_string()))?; let object_store_url = ObjectStoreUrl::parse(url_key.clone())?; - runtime_env.register_object_store(&url, Arc::from(object_store)); + runtime_env.register_object_store(&url, Arc::from(object_store) as Arc); Ok((object_store_url, object_store_path)) } diff --git a/native/core/src/parquet/schema_adapter.rs b/native/core/src/parquet/schema_adapter.rs index 42f0e7fc61..e8df2c0e37 100644 --- a/native/core/src/parquet/schema_adapter.rs +++ b/native/core/src/parquet/schema_adapter.rs @@ -100,7 +100,7 @@ impl PhysicalExprAdapterFactory for SparkPhysicalExprAdapterFactory { &self, logical_file_schema: SchemaRef, physical_file_schema: SchemaRef, - ) -> Arc { + ) -> DataFusionResult> { // When case-insensitive, remap physical schema field names to match logical // field names. The DefaultPhysicalExprAdapter uses exact name matching, so // without this remapping, columns like "a" won't match logical "A" and will @@ -145,16 +145,16 @@ impl PhysicalExprAdapterFactory for SparkPhysicalExprAdapterFactory { let default_adapter = default_factory.create( Arc::clone(&logical_file_schema), Arc::clone(&adapted_physical_schema), - ); + )?; - Arc::new(SparkPhysicalExprAdapter { + Ok(Arc::new(SparkPhysicalExprAdapter { logical_file_schema, physical_file_schema: adapted_physical_schema, parquet_options: self.parquet_options.clone(), default_values: self.default_values.clone(), default_adapter, logical_to_physical_names, - }) + })) } } From 0edf710c24e417cc6f5ea2417702917c91408ded Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 4 Mar 2026 06:49:41 -0500 Subject: [PATCH 02/15] add iceberg and opendal as features --- native/core/Cargo.toml | 5 ++-- native/core/src/execution/operators/mod.rs | 2 ++ .../src/execution/operators/parquet_writer.rs | 23 ++++++++++++++----- native/core/src/execution/planner.rs | 19 +++++++++++++++ .../src/execution/shuffle/shuffle_writer.rs | 1 - 5 files changed, 41 insertions(+), 9 deletions(-) diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml index 23a78aa3ee..c44d0d65c1 100644 --- a/native/core/Cargo.toml +++ b/native/core/Cargo.toml @@ -79,7 +79,7 @@ reqwest = { version = "0.12", default-features = false, features = ["rustls-tls- object_store_opendal = {version = "0.55.0", optional = true} hdfs-sys = {version = "0.3", optional = true, features = ["hdfs_3_3"]} opendal = { version ="0.55.0", optional = true, features = ["services-hdfs"] } -iceberg = { workspace = true } +iceberg = { workspace = true, optional = true } serde_json = "1.0" uuid = "1.21.0" @@ -100,7 +100,8 @@ datafusion-functions-nested = { git = "https://github.com/apache/datafusion", br [features] backtrace = ["datafusion/backtrace"] -default = ["hdfs-opendal"] +default = [] +iceberg = ["dep:iceberg"] hdfs = ["datafusion-comet-objectstore-hdfs"] hdfs-opendal = ["opendal", "object_store_opendal", "hdfs-sys"] jemalloc = ["tikv-jemallocator", "tikv-jemalloc-ctl"] diff --git a/native/core/src/execution/operators/mod.rs b/native/core/src/execution/operators/mod.rs index 07ee995367..7d3b4dabc6 100644 --- a/native/core/src/execution/operators/mod.rs +++ b/native/core/src/execution/operators/mod.rs @@ -22,12 +22,14 @@ use std::fmt::Debug; use jni::objects::GlobalRef; pub use copy::*; +#[cfg(feature = "iceberg")] pub use iceberg_scan::*; pub use scan::*; mod copy; mod expand; pub use expand::ExpandExec; +#[cfg(feature = "iceberg")] mod iceberg_scan; mod parquet_writer; pub use parquet_writer::ParquetWriterExec; diff --git a/native/core/src/execution/operators/parquet_writer.rs b/native/core/src/execution/operators/parquet_writer.rs index 7b53fbc4bc..132ebf7be9 100644 --- a/native/core/src/execution/operators/parquet_writer.rs +++ b/native/core/src/execution/operators/parquet_writer.rs @@ -23,16 +23,18 @@ use std::{ fmt, fmt::{Debug, Formatter}, fs::File, - io::Cursor, sync::Arc, }; +#[cfg(feature = "hdfs-opendal")] +use std::io::Cursor; +#[cfg(feature = "hdfs-opendal")] use opendal::Operator; use crate::execution::shuffle::CompressionCodec; -use crate::parquet::parquet_support::{ - create_hdfs_operator, is_hdfs_scheme, prepare_object_store_with_configs, -}; +use crate::parquet::parquet_support::is_hdfs_scheme; +#[cfg(feature = "hdfs-opendal")] +use crate::parquet::parquet_support::{create_hdfs_operator, prepare_object_store_with_configs}; use arrow::datatypes::{Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use async_trait::async_trait; @@ -45,7 +47,7 @@ use datafusion::{ metrics::{ExecutionPlanMetricsSet, MetricsSet}, stream::RecordBatchStreamAdapter, DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, - SendableRecordBatchStream, Statistics, + SendableRecordBatchStream, }, }; use futures::TryStreamExt; @@ -64,6 +66,7 @@ enum ParquetWriter { /// Contains the arrow writer, HDFS operator, and destination path /// an Arrow writer writes to in-memory buffer the data converted to Parquet format /// The opendal::Writer is created lazily on first write + #[cfg(feature = "hdfs-opendal")] Remote( ArrowWriter>>, Option, @@ -80,6 +83,7 @@ impl ParquetWriter { ) -> std::result::Result<(), parquet::errors::ParquetError> { match self { ParquetWriter::LocalFile(writer) => writer.write(batch), + #[cfg(feature = "hdfs-opendal")] ParquetWriter::Remote( arrow_parquet_buffer_writer, hdfs_writer_opt, @@ -134,6 +138,7 @@ impl ParquetWriter { writer.close()?; Ok(()) } + #[cfg(feature = "hdfs-opendal")] ParquetWriter::Remote( arrow_parquet_buffer_writer, mut hdfs_writer_opt, @@ -284,7 +289,7 @@ impl ParquetWriterExec { })?; if is_hdfs_scheme(&url, object_store_options) { - // HDFS storage + #[cfg(feature = "hdfs-opendal")] { // Use prepare_object_store_with_configs to create and register the object store let (_object_store_url, object_store_path) = prepare_object_store_with_configs( @@ -324,6 +329,12 @@ impl ParquetWriterExec { object_store_path.to_string(), )) } + #[cfg(not(feature = "hdfs-opendal"))] + { + Err(DataFusionError::Execution( + "HDFS support is not enabled. Rebuild with the 'hdfs-opendal' feature.".into(), + )) + } } else if output_file_path.starts_with("file://") || output_file_path.starts_with("file:") || !output_file_path.contains("://") diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 3eba167e41..311c3b14e8 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -22,6 +22,7 @@ pub mod macros; pub mod operator_registry; use crate::execution::operators::init_csv_datasource_exec; +#[cfg(feature = "iceberg")] use crate::execution::operators::IcebergScanExec; use crate::{ errors::ExpressionError, @@ -73,6 +74,7 @@ use datafusion_comet_spark_expr::{ create_comet_physical_fun, create_comet_physical_fun_with_eval_mode, BinaryOutputStyle, BloomFilterAgg, BloomFilterMightContain, CsvWriteOptions, EvalMode, SumInteger, ToCsv, }; +#[cfg(feature = "iceberg")] use iceberg::expr::Bind; use crate::execution::operators::ExecutionError::GeneralError; @@ -1192,6 +1194,7 @@ impl PhysicalPlanner { Arc::new(SparkPlan::new(spark_plan.plan_id, Arc::new(scan), vec![])), )) } + #[cfg(feature = "iceberg")] OpStruct::IcebergScan(scan) => { // Extract common data and single partition's file tasks // Per-partition injection happens in Scala before sending to native @@ -1228,6 +1231,10 @@ impl PhysicalPlanner { )), )) } + #[cfg(not(feature = "iceberg"))] + OpStruct::IcebergScan(_) => { + Err(GeneralError("Iceberg support is not enabled. Rebuild with the 'iceberg' feature.".into()).into()) + } OpStruct::ShuffleWriter(writer) => { assert_eq!(children.len(), 1); let (scans, child) = self.create_plan(&children[0], inputs, partition_count)?; @@ -1577,6 +1584,11 @@ impl PhysicalPlanner { // null doesn't equal to null in Spark join key. If the join key is // `EqualNullSafe`, Spark will rewrite it during planning. NullEquality::NullEqualsNothing, + // null_aware is for null-aware anti joins (NOT IN subqueries). + // NullEquality controls whether NULL = NULL in join keys generally, + // while null_aware changes anti-join semantics so any NULL changes + // the entire result. Spark doesn't use this path (it rewrites + // EqualNullSafe at plan time), so false is correct. false, )?); @@ -2700,6 +2712,7 @@ fn convert_spark_types_to_arrow_schema( arrow_schema } +#[cfg(feature = "iceberg")] /// Converts a protobuf PartitionValue to an iceberg Literal. /// fn partition_value_to_literal( @@ -2785,6 +2798,7 @@ fn partition_value_to_literal( /// Uses the existing Struct::from_iter() API from iceberg-rust to construct the struct /// from the list of partition values. /// This can potentially be upstreamed to iceberg_rust +#[cfg(feature = "iceberg")] fn partition_data_to_struct( proto_partition: &spark_operator::PartitionData, ) -> Result { @@ -2804,6 +2818,7 @@ fn partition_data_to_struct( /// /// This function uses deduplication pools from the IcebergScanCommon to avoid redundant /// parsing of schemas, partition specs, partition types, name mappings, and other repeated data. +#[cfg(feature = "iceberg")] fn parse_file_scan_tasks_from_common( proto_common: &spark_operator::IcebergScanCommon, proto_tasks: &[spark_operator::IcebergFileScanTask], @@ -3252,6 +3267,7 @@ fn literal_to_array_ref( // always returns MIGHT_MATCH (never prunes row groups). These are handled by CometFilter post-scan. /// Converts a protobuf Spark expression to an Iceberg predicate for row-group filtering. +#[cfg(feature = "iceberg")] fn convert_spark_expr_to_predicate( expr: &spark_expression::Expr, ) -> Option { @@ -3383,6 +3399,7 @@ fn convert_spark_expr_to_predicate( } } +#[cfg(feature = "iceberg")] fn convert_binary_to_predicate( left: &Option>, right: &Option>, @@ -3431,6 +3448,7 @@ fn convert_binary_to_predicate( None } +#[cfg(feature = "iceberg")] fn extract_column_reference(expr: &spark_expression::Expr) -> Option { use spark_expression::expr::ExprStruct; @@ -3440,6 +3458,7 @@ fn extract_column_reference(expr: &spark_expression::Expr) -> Option { } } +#[cfg(feature = "iceberg")] fn extract_literal_as_datum(expr: &spark_expression::Expr) -> Option { use spark_expression::expr::ExprStruct; diff --git a/native/core/src/execution/shuffle/shuffle_writer.rs b/native/core/src/execution/shuffle/shuffle_writer.rs index 8327f04654..1b9433993d 100644 --- a/native/core/src/execution/shuffle/shuffle_writer.rs +++ b/native/core/src/execution/shuffle/shuffle_writer.rs @@ -36,7 +36,6 @@ use datafusion::{ metrics::{ExecutionPlanMetricsSet, MetricsSet}, stream::RecordBatchStreamAdapter, DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, SendableRecordBatchStream, - Statistics, }, }; use futures::{StreamExt, TryFutureExt, TryStreamExt}; From c96422ea292a1b40f6ce418351ccffd9467b73b7 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 4 Mar 2026 06:50:09 -0500 Subject: [PATCH 03/15] cargo fmt --- native/core/src/execution/operators/parquet_writer.rs | 4 ++-- native/core/src/execution/planner.rs | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/native/core/src/execution/operators/parquet_writer.rs b/native/core/src/execution/operators/parquet_writer.rs index 132ebf7be9..820d8b0481 100644 --- a/native/core/src/execution/operators/parquet_writer.rs +++ b/native/core/src/execution/operators/parquet_writer.rs @@ -26,10 +26,10 @@ use std::{ sync::Arc, }; -#[cfg(feature = "hdfs-opendal")] -use std::io::Cursor; #[cfg(feature = "hdfs-opendal")] use opendal::Operator; +#[cfg(feature = "hdfs-opendal")] +use std::io::Cursor; use crate::execution::shuffle::CompressionCodec; use crate::parquet::parquet_support::is_hdfs_scheme; diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 311c3b14e8..99fee145ba 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -1232,9 +1232,10 @@ impl PhysicalPlanner { )) } #[cfg(not(feature = "iceberg"))] - OpStruct::IcebergScan(_) => { - Err(GeneralError("Iceberg support is not enabled. Rebuild with the 'iceberg' feature.".into()).into()) - } + OpStruct::IcebergScan(_) => Err(GeneralError( + "Iceberg support is not enabled. Rebuild with the 'iceberg' feature.".into(), + ) + .into()), OpStruct::ShuffleWriter(writer) => { assert_eq!(children.len(), 1); let (scans, child) = self.create_plan(&children[0], inputs, partition_count)?; From eca9b33417a11c9f93c21a9332ddb8d434ef3504 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 4 Mar 2026 06:51:20 -0500 Subject: [PATCH 04/15] clippy fixes --- native/core/src/execution/planner.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 99fee145ba..b58cde0939 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -1234,8 +1234,7 @@ impl PhysicalPlanner { #[cfg(not(feature = "iceberg"))] OpStruct::IcebergScan(_) => Err(GeneralError( "Iceberg support is not enabled. Rebuild with the 'iceberg' feature.".into(), - ) - .into()), + )), OpStruct::ShuffleWriter(writer) => { assert_eq!(children.len(), 1); let (scans, child) = self.create_plan(&children[0], inputs, partition_count)?; From b5ff6411cc6683d450b598b6227609afed00417c Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 4 Mar 2026 07:33:18 -0500 Subject: [PATCH 05/15] bump to iceberg-rust df53 branch, clippy fixes --- native/Cargo.lock | 354 +++++------------- native/Cargo.toml | 2 +- native/core/Cargo.toml | 3 +- native/core/src/execution/operators/mod.rs | 2 - .../src/execution/operators/parquet_writer.rs | 4 +- native/core/src/execution/planner.rs | 62 +-- 6 files changed, 103 insertions(+), 324 deletions(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index 0d40eb7da3..a66099fc88 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -155,33 +155,19 @@ version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "602268ce9f569f282cedb9a9f6bac569b680af47b9b077d515900c03c5d190da" dependencies = [ - "arrow-arith 58.0.0", - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-cast 58.0.0", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", "arrow-csv", - "arrow-data 58.0.0", - "arrow-ipc 58.0.0", + "arrow-data", + "arrow-ipc", "arrow-json", - "arrow-ord 58.0.0", + "arrow-ord", "arrow-row", - "arrow-schema 58.0.0", - "arrow-select 58.0.0", - "arrow-string 58.0.0", -] - -[[package]] -name = "arrow-arith" -version = "57.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7b3141e0ec5145a22d8694ea8b6d6f69305971c4fa1c1a13ef0195aef2d678b" -dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "chrono", - "num-traits", + "arrow-schema", + "arrow-select", + "arrow-string", ] [[package]] @@ -190,32 +176,14 @@ version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd53c6bf277dea91f136ae8e3a5d7041b44b5e489e244e637d00ae302051f56f" dependencies = [ - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-data 58.0.0", - "arrow-schema 58.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "num-traits", ] -[[package]] -name = "arrow-array" -version = "57.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8955af33b25f3b175ee10af580577280b4bd01f7e823d94c7cdef7cf8c9aef" -dependencies = [ - "ahash", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "chrono", - "half", - "hashbrown 0.16.1", - "num-complex", - "num-integer", - "num-traits", -] - [[package]] name = "arrow-array" version = "58.0.0" @@ -223,9 +191,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e53796e07a6525edaf7dc28b540d477a934aff14af97967ad1d5550878969b9e" dependencies = [ "ahash", - "arrow-buffer 58.0.0", - "arrow-data 58.0.0", - "arrow-schema 58.0.0", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "chrono-tz", "half", @@ -235,18 +203,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "arrow-buffer" -version = "57.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c697ddca96183182f35b3a18e50b9110b11e916d7b7799cbfd4d34662f2c56c2" -dependencies = [ - "bytes", - "half", - "num-bigint", - "num-traits", -] - [[package]] name = "arrow-buffer" version = "58.0.0" @@ -259,39 +215,18 @@ dependencies = [ "num-traits", ] -[[package]] -name = "arrow-cast" -version = "57.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "646bbb821e86fd57189c10b4fcdaa941deaf4181924917b0daa92735baa6ada5" -dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-ord 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", - "atoi", - "base64", - "chrono", - "half", - "lexical-core", - "num-traits", - "ryu", -] - [[package]] name = "arrow-cast" version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89fb245db6b0e234ed8e15b644edb8664673fefe630575e94e62cd9d489a8a26" dependencies = [ - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-data 58.0.0", - "arrow-ord 58.0.0", - "arrow-schema 58.0.0", - "arrow-select 58.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", "atoi", "base64", "chrono", @@ -308,66 +243,39 @@ version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d374882fb465a194462527c0c15a93aa19a554cf690a6b77a26b2a02539937a7" dependencies = [ - "arrow-array 58.0.0", - "arrow-cast 58.0.0", - "arrow-schema 58.0.0", + "arrow-array", + "arrow-cast", + "arrow-schema", "chrono", "csv", "csv-core", "regex", ] -[[package]] -name = "arrow-data" -version = "57.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fdd994a9d28e6365aa78e15da3f3950c0fdcea6b963a12fa1c391afb637b304" -dependencies = [ - "arrow-buffer 57.3.0", - "arrow-schema 57.3.0", - "half", - "num-integer", - "num-traits", -] - [[package]] name = "arrow-data" version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "189d210bc4244c715fa3ed9e6e22864673cccb73d5da28c2723fb2e527329b33" dependencies = [ - "arrow-buffer 58.0.0", - "arrow-schema 58.0.0", + "arrow-buffer", + "arrow-schema", "half", "num-integer", "num-traits", ] -[[package]] -name = "arrow-ipc" -version = "57.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf7df950701ab528bf7c0cf7eeadc0445d03ef5d6ffc151eaae6b38a58feff1" -dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", - "flatbuffers", -] - [[package]] name = "arrow-ipc" version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7968c2e5210c41f4909b2ef76f6e05e172b99021c2def5edf3cc48fdd39d1d6c" dependencies = [ - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-data 58.0.0", - "arrow-schema 58.0.0", - "arrow-select 58.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "flatbuffers", "lz4_flex", ] @@ -378,11 +286,11 @@ version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92111dba5bf900f443488e01f00d8c4ddc2f47f5c50039d18120287b580baa22" dependencies = [ - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-cast 58.0.0", - "arrow-data 58.0.0", - "arrow-schema 58.0.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", "chrono", "half", "indexmap 2.13.0", @@ -396,30 +304,17 @@ dependencies = [ "simdutf8", ] -[[package]] -name = "arrow-ord" -version = "57.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d8f1870e03d4cbed632959498bcc84083b5a24bded52905ae1695bd29da45b" -dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", -] - [[package]] name = "arrow-ord" version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "211136cb253577ee1a6665f741a13136d4e563f64f5093ffd6fb837af90b9495" dependencies = [ - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-data 58.0.0", - "arrow-schema 58.0.0", - "arrow-select 58.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", ] [[package]] @@ -428,19 +323,13 @@ version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e0f20145f9f5ea3fe383e2ba7a7487bf19be36aa9dbf5dd6a1f92f657179663" dependencies = [ - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-data 58.0.0", - "arrow-schema 58.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "half", ] -[[package]] -name = "arrow-schema" -version = "57.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c872d36b7bf2a6a6a2b40de9156265f0242910791db366a2c17476ba8330d68" - [[package]] name = "arrow-schema" version = "58.0.0" @@ -452,20 +341,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "arrow-select" -version = "57.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68bf3e3efbd1278f770d67e5dc410257300b161b93baedb3aae836144edcaf4b" -dependencies = [ - "ahash", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "num-traits", -] - [[package]] name = "arrow-select" version = "58.0.0" @@ -473,41 +348,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "750a7d1dda177735f5e82a314485b6915c7cccdbb278262ac44090f4aba4a325" dependencies = [ "ahash", - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-data 58.0.0", - "arrow-schema 58.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "num-traits", ] -[[package]] -name = "arrow-string" -version = "57.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85e968097061b3c0e9fe3079cf2e703e487890700546b5b0647f60fca1b5a8d8" -dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", - "memchr", - "num-traits", - "regex", - "regex-syntax", -] - [[package]] name = "arrow-string" version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1eab1208bc4fe55d768cdc9b9f3d9df5a794cdb3ee2586bf89f9b30dc31ad8c" dependencies = [ - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-data 58.0.0", - "arrow-schema 58.0.0", - "arrow-select 58.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "memchr", "num-traits", "regex", @@ -1828,7 +1686,7 @@ version = "53.0.0" source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", - "arrow-schema 58.0.0", + "arrow-schema", "async-trait", "bytes", "chrono", @@ -1862,7 +1720,7 @@ dependencies = [ "log", "object_store 0.13.1", "parking_lot", - "parquet 58.0.0", + "parquet", "rand 0.9.2", "regex", "sqlparser", @@ -1956,7 +1814,7 @@ dependencies = [ "once_cell", "opendal", "parking_lot", - "parquet 58.0.0", + "parquet", "paste", "pprof", "procfs", @@ -2042,7 +1900,7 @@ source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e7 dependencies = [ "ahash", "arrow", - "arrow-ipc 58.0.0", + "arrow-ipc", "chrono", "half", "hashbrown 0.16.1", @@ -2052,7 +1910,7 @@ dependencies = [ "libc", "log", "object_store 0.13.1", - "parquet 58.0.0", + "parquet", "paste", "sqlparser", "tokio", @@ -2109,7 +1967,7 @@ version = "53.0.0" source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", - "arrow-ipc 58.0.0", + "arrow-ipc", "async-trait", "bytes", "datafusion-common", @@ -2196,7 +2054,7 @@ dependencies = [ "log", "object_store 0.13.1", "parking_lot", - "parquet 58.0.0", + "parquet", "tokio", ] @@ -2211,7 +2069,7 @@ version = "53.0.0" source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", - "arrow-buffer 58.0.0", + "arrow-buffer", "async-trait", "chrono", "dashmap", @@ -2222,7 +2080,7 @@ dependencies = [ "log", "object_store 0.13.1", "parking_lot", - "parquet 58.0.0", + "parquet", "rand 0.9.2", "tempfile", "url", @@ -2267,7 +2125,7 @@ version = "53.0.0" source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", - "arrow-buffer 58.0.0", + "arrow-buffer", "base64", "blake2", "blake3", @@ -2331,7 +2189,7 @@ version = "53.0.0" source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" dependencies = [ "arrow", - "arrow-ord 58.0.0", + "arrow-ord", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -2494,8 +2352,8 @@ source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e7 dependencies = [ "ahash", "arrow", - "arrow-ord 58.0.0", - "arrow-schema 58.0.0", + "arrow-ord", + "arrow-schema", "async-trait", "datafusion-common", "datafusion-common-runtime", @@ -3374,19 +3232,19 @@ dependencies = [ [[package]] name = "iceberg" version = "0.8.0" -source = "git+https://github.com/apache/iceberg-rust?rev=b24ab63#b24ab6310235f71907f4b6b6dc14a8e5d9291acc" +source = "git+https://github.com/mbutrovich/iceberg-rust?branch=df53-upgrade#71bc1ab8189860d00a0805db405431d806c03a26" dependencies = [ "anyhow", "apache-avro", "array-init", - "arrow-arith 57.3.0", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-cast 57.3.0", - "arrow-ord 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", - "arrow-string 57.3.0", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "arrow-string", "as-any", "async-trait", "backon", @@ -3406,7 +3264,7 @@ dependencies = [ "once_cell", "opendal", "ordered-float 4.6.0", - "parquet 57.3.0", + "parquet", "rand 0.8.5", "reqsign", "reqwest", @@ -4389,42 +4247,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "parquet" -version = "57.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ee96b29972a257b855ff2341b37e61af5f12d6af1158b6dcdb5b31ea07bb3cb" -dependencies = [ - "ahash", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-cast 57.3.0", - "arrow-data 57.3.0", - "arrow-ipc 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", - "base64", - "brotli", - "bytes", - "chrono", - "flate2", - "futures", - "half", - "hashbrown 0.16.1", - "lz4_flex", - "num-bigint", - "num-integer", - "num-traits", - "paste", - "seq-macro", - "simdutf8", - "snap", - "thrift", - "tokio", - "twox-hash", - "zstd", -] - [[package]] name = "parquet" version = "58.0.0" @@ -4432,12 +4254,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f491d0ef1b510194426ee67ddc18a9b747ef3c42050c19322a2cd2e1666c29b" dependencies = [ "ahash", - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-data 58.0.0", - "arrow-ipc 58.0.0", - "arrow-schema 58.0.0", - "arrow-select 58.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", "base64", "brotli", "bytes", @@ -4471,7 +4293,7 @@ version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00ba4e5dcbc8ad65882b7337a95c12a0f9cbb6add237c53d93b803b7d7f70f02" dependencies = [ - "arrow-schema 58.0.0", + "arrow-schema", "chrono", "half", "indexmap 2.13.0", @@ -4486,7 +4308,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ec4cfb8da15565c8d211b6bc51e8eb481ea65d19132462af3f948b150ac8efe" dependencies = [ "arrow", - "arrow-schema 58.0.0", + "arrow-schema", "chrono", "half", "indexmap 2.13.0", @@ -4502,7 +4324,7 @@ version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3668ff00a6aeb29d172ba15f9d8fedf1675d79bff7d1916daa333efdeaa13e46" dependencies = [ - "arrow-schema 58.0.0", + "arrow-schema", "base64", "chrono", "parquet-variant", diff --git a/native/Cargo.toml b/native/Cargo.toml index 4067956722..abf8a3bf5e 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -55,7 +55,7 @@ object_store = { version = "0.13.1", features = ["gcp", "azure", "aws", "http"] url = "2.2" aws-config = "1.8.14" aws-credential-types = "1.2.13" -iceberg = { git = "https://github.com/apache/iceberg-rust", rev = "b24ab63" } +iceberg = { git = "https://github.com/mbutrovich/iceberg-rust", branch = "df53-upgrade" } [profile.release] debug = true diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml index c44d0d65c1..2233aa8855 100644 --- a/native/core/Cargo.toml +++ b/native/core/Cargo.toml @@ -79,7 +79,7 @@ reqwest = { version = "0.12", default-features = false, features = ["rustls-tls- object_store_opendal = {version = "0.55.0", optional = true} hdfs-sys = {version = "0.3", optional = true, features = ["hdfs_3_3"]} opendal = { version ="0.55.0", optional = true, features = ["services-hdfs"] } -iceberg = { workspace = true, optional = true } +iceberg = { workspace = true } serde_json = "1.0" uuid = "1.21.0" @@ -101,7 +101,6 @@ datafusion-functions-nested = { git = "https://github.com/apache/datafusion", br [features] backtrace = ["datafusion/backtrace"] default = [] -iceberg = ["dep:iceberg"] hdfs = ["datafusion-comet-objectstore-hdfs"] hdfs-opendal = ["opendal", "object_store_opendal", "hdfs-sys"] jemalloc = ["tikv-jemallocator", "tikv-jemalloc-ctl"] diff --git a/native/core/src/execution/operators/mod.rs b/native/core/src/execution/operators/mod.rs index 7d3b4dabc6..07ee995367 100644 --- a/native/core/src/execution/operators/mod.rs +++ b/native/core/src/execution/operators/mod.rs @@ -22,14 +22,12 @@ use std::fmt::Debug; use jni::objects::GlobalRef; pub use copy::*; -#[cfg(feature = "iceberg")] pub use iceberg_scan::*; pub use scan::*; mod copy; mod expand; pub use expand::ExpandExec; -#[cfg(feature = "iceberg")] mod iceberg_scan; mod parquet_writer; pub use parquet_writer::ParquetWriterExec; diff --git a/native/core/src/execution/operators/parquet_writer.rs b/native/core/src/execution/operators/parquet_writer.rs index 820d8b0481..bb2f54b2a6 100644 --- a/native/core/src/execution/operators/parquet_writer.rs +++ b/native/core/src/execution/operators/parquet_writer.rs @@ -280,7 +280,7 @@ impl ParquetWriterExec { output_file_path: &str, schema: SchemaRef, props: WriterProperties, - runtime_env: Arc, + _runtime_env: Arc, object_store_options: &HashMap, ) -> Result { // Parse URL and match on storage scheme directly @@ -293,7 +293,7 @@ impl ParquetWriterExec { { // Use prepare_object_store_with_configs to create and register the object store let (_object_store_url, object_store_path) = prepare_object_store_with_configs( - runtime_env, + _runtime_env, output_file_path.to_string(), object_store_options, ) diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index b58cde0939..46d00a3e67 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -22,7 +22,6 @@ pub mod macros; pub mod operator_registry; use crate::execution::operators::init_csv_datasource_exec; -#[cfg(feature = "iceberg")] use crate::execution::operators::IcebergScanExec; use crate::{ errors::ExpressionError, @@ -74,7 +73,6 @@ use datafusion_comet_spark_expr::{ create_comet_physical_fun, create_comet_physical_fun_with_eval_mode, BinaryOutputStyle, BloomFilterAgg, BloomFilterMightContain, CsvWriteOptions, EvalMode, SumInteger, ToCsv, }; -#[cfg(feature = "iceberg")] use iceberg::expr::Bind; use crate::execution::operators::ExecutionError::GeneralError; @@ -107,7 +105,6 @@ use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer}; use arrow::row::{OwnedRow, RowConverter, SortField}; use datafusion::common::utils::SingleRowListArrayBuilder; use datafusion::common::UnnestOptions; -use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec; use datafusion::physical_plan::filter::FilterExec; use datafusion::physical_plan::limit::GlobalLimitExec; use datafusion::physical_plan::unnest::{ListUnnest, UnnestExec}; @@ -1194,7 +1191,6 @@ impl PhysicalPlanner { Arc::new(SparkPlan::new(spark_plan.plan_id, Arc::new(scan), vec![])), )) } - #[cfg(feature = "iceberg")] OpStruct::IcebergScan(scan) => { // Extract common data and single partition's file tasks // Per-partition injection happens in Scala before sending to native @@ -1231,10 +1227,6 @@ impl PhysicalPlanner { )), )) } - #[cfg(not(feature = "iceberg"))] - OpStruct::IcebergScan(_) => Err(GeneralError( - "Iceberg support is not enabled. Rebuild with the 'iceberg' feature.".into(), - )), OpStruct::ShuffleWriter(writer) => { assert_eq!(children.len(), 1); let (scans, child) = self.create_plan(&children[0], inputs, partition_count)?; @@ -1522,42 +1514,17 @@ impl PhysicalPlanner { NullEquality::NullEqualsNothing, )?); - if join.filter.is_some() { - // SMJ with join filter produces lots of tiny batches - let coalesce_batches: Arc = - Arc::new(CoalesceBatchesExec::new( - Arc::::clone(&join), - self.session_ctx - .state() - .config_options() - .execution - .batch_size, - )); - Ok(( - scans, - Arc::new(SparkPlan::new_with_additional( - spark_plan.plan_id, - coalesce_batches, - vec![ - Arc::clone(&join_params.left), - Arc::clone(&join_params.right), - ], - vec![join], - )), - )) - } else { - Ok(( - scans, - Arc::new(SparkPlan::new( - spark_plan.plan_id, - join, - vec![ - Arc::clone(&join_params.left), - Arc::clone(&join_params.right), - ], - )), - )) - } + Ok(( + scans, + Arc::new(SparkPlan::new( + spark_plan.plan_id, + join, + vec![ + Arc::clone(&join_params.left), + Arc::clone(&join_params.right), + ], + )), + )) } OpStruct::HashJoin(join) => { let (join_params, scans) = self.parse_join_parameters( @@ -2712,7 +2679,6 @@ fn convert_spark_types_to_arrow_schema( arrow_schema } -#[cfg(feature = "iceberg")] /// Converts a protobuf PartitionValue to an iceberg Literal. /// fn partition_value_to_literal( @@ -2798,7 +2764,6 @@ fn partition_value_to_literal( /// Uses the existing Struct::from_iter() API from iceberg-rust to construct the struct /// from the list of partition values. /// This can potentially be upstreamed to iceberg_rust -#[cfg(feature = "iceberg")] fn partition_data_to_struct( proto_partition: &spark_operator::PartitionData, ) -> Result { @@ -2818,7 +2783,6 @@ fn partition_data_to_struct( /// /// This function uses deduplication pools from the IcebergScanCommon to avoid redundant /// parsing of schemas, partition specs, partition types, name mappings, and other repeated data. -#[cfg(feature = "iceberg")] fn parse_file_scan_tasks_from_common( proto_common: &spark_operator::IcebergScanCommon, proto_tasks: &[spark_operator::IcebergFileScanTask], @@ -3267,7 +3231,6 @@ fn literal_to_array_ref( // always returns MIGHT_MATCH (never prunes row groups). These are handled by CometFilter post-scan. /// Converts a protobuf Spark expression to an Iceberg predicate for row-group filtering. -#[cfg(feature = "iceberg")] fn convert_spark_expr_to_predicate( expr: &spark_expression::Expr, ) -> Option { @@ -3399,7 +3362,6 @@ fn convert_spark_expr_to_predicate( } } -#[cfg(feature = "iceberg")] fn convert_binary_to_predicate( left: &Option>, right: &Option>, @@ -3448,7 +3410,6 @@ fn convert_binary_to_predicate( None } -#[cfg(feature = "iceberg")] fn extract_column_reference(expr: &spark_expression::Expr) -> Option { use spark_expression::expr::ExprStruct; @@ -3458,7 +3419,6 @@ fn extract_column_reference(expr: &spark_expression::Expr) -> Option { } } -#[cfg(feature = "iceberg")] fn extract_literal_as_datum(expr: &spark_expression::Expr) -> Option { use spark_expression::expr::ExprStruct; From 42583001b3bdb0ac4b164918e831b685a8a364a9 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 4 Mar 2026 07:47:16 -0500 Subject: [PATCH 06/15] bump to iceberg-rust df53 branch, clippy fixes --- native/Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index a66099fc88..fc08f2b485 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -3232,7 +3232,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.8.0" -source = "git+https://github.com/mbutrovich/iceberg-rust?branch=df53-upgrade#71bc1ab8189860d00a0805db405431d806c03a26" +source = "git+https://github.com/mbutrovich/iceberg-rust?branch=df53-upgrade#72fbebc4b0b9f5363accfb4707c52635306e5271" dependencies = [ "anyhow", "apache-avro", From efa94372d6bcdcc1209a4c6f112e795c547140ac Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 4 Mar 2026 09:11:42 -0500 Subject: [PATCH 07/15] fix fileIO construction --- .../src/execution/operators/iceberg_scan.rs | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/native/core/src/execution/operators/iceberg_scan.rs b/native/core/src/execution/operators/iceberg_scan.rs index 8409545763..3ecc3b2c5c 100644 --- a/native/core/src/execution/operators/iceberg_scan.rs +++ b/native/core/src/execution/operators/iceberg_scan.rs @@ -38,7 +38,7 @@ use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, }; use futures::{Stream, StreamExt, TryStreamExt}; -use iceberg::io::FileIO; +use iceberg::io::{FileIO, FileIOBuilder, OpenDalStorageFactory, StorageFactory}; use crate::execution::operators::ExecutionError; use crate::parquet::parquet_support::SparkParquetOptions; @@ -191,20 +191,32 @@ impl IcebergScanExec { Ok(Box::pin(wrapped_stream)) } + fn storage_factory_for(path: &str) -> Result, DataFusionError> { + let scheme = path.split("://").next().unwrap_or("file"); + match scheme { + "file" | "" => Ok(Arc::new(OpenDalStorageFactory::Fs)), + "s3" | "s3a" => Ok(Arc::new(OpenDalStorageFactory::S3 { + configured_scheme: scheme.to_string(), + customized_credential_load: None, + })), + _ => Err(DataFusionError::Execution(format!( + "Unsupported storage scheme: {scheme}" + ))), + } + } + fn load_file_io( catalog_properties: &HashMap, metadata_location: &str, ) -> Result { - let mut file_io_builder = FileIO::from_path(metadata_location) - .map_err(|e| DataFusionError::Execution(format!("Failed to create FileIO: {}", e)))?; + let factory = Self::storage_factory_for(metadata_location)?; + let mut file_io_builder = FileIOBuilder::new(factory); for (key, value) in catalog_properties { file_io_builder = file_io_builder.with_prop(key, value); } - file_io_builder - .build() - .map_err(|e| DataFusionError::Execution(format!("Failed to build FileIO: {}", e))) + Ok(file_io_builder.build()) } } From a548871c09b7d9e4a03cd4a1807ffd6cfab37d04 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 4 Mar 2026 11:43:52 -0500 Subject: [PATCH 08/15] update deps --- native/Cargo.lock | 66 +++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index fc08f2b485..8e2ed37f69 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -1683,7 +1683,7 @@ dependencies = [ [[package]] name = "datafusion" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", "arrow-schema", @@ -1733,7 +1733,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", "async-trait", @@ -1757,7 +1757,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", "async-trait", @@ -1896,7 +1896,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "ahash", "arrow", @@ -1920,7 +1920,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "futures", "log", @@ -1930,7 +1930,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", "async-compression", @@ -1964,7 +1964,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", "arrow-ipc", @@ -1987,7 +1987,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", "async-trait", @@ -2009,7 +2009,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", "async-trait", @@ -2032,7 +2032,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", "async-trait", @@ -2061,12 +2061,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" [[package]] name = "datafusion-execution" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", "arrow-buffer", @@ -2089,7 +2089,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", "async-trait", @@ -2110,7 +2110,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", "datafusion-common", @@ -2122,7 +2122,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", "arrow-buffer", @@ -2153,7 +2153,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "ahash", "arrow", @@ -2174,7 +2174,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "ahash", "arrow", @@ -2186,7 +2186,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", "arrow-ord", @@ -2210,7 +2210,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", "async-trait", @@ -2225,7 +2225,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", "datafusion-common", @@ -2242,7 +2242,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2251,7 +2251,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "datafusion-doc", "quote", @@ -2261,7 +2261,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", "chrono", @@ -2279,7 +2279,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "ahash", "arrow", @@ -2301,7 +2301,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", "datafusion-common", @@ -2315,7 +2315,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "ahash", "arrow", @@ -2331,7 +2331,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", "datafusion-common", @@ -2348,7 +2348,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "ahash", "arrow", @@ -2379,7 +2379,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", "datafusion-common", @@ -2395,7 +2395,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "async-trait", "datafusion-common", @@ -2408,7 +2408,7 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", "bigdecimal", @@ -2434,7 +2434,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#23e7a19186e71044a610e5bfcc4e647598bf557a" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", "bigdecimal", @@ -3232,7 +3232,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.8.0" -source = "git+https://github.com/mbutrovich/iceberg-rust?branch=df53-upgrade#72fbebc4b0b9f5363accfb4707c52635306e5271" +source = "git+https://github.com/mbutrovich/iceberg-rust?branch=df53-upgrade#646ab672a649322f3b140b0cf3dc76e26bb540a1" dependencies = [ "anyhow", "apache-avro", From faf1c56054747ba2cc7f508a199fd5b649824223 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Tue, 17 Mar 2026 07:30:54 -0400 Subject: [PATCH 09/15] Bump to 53.0.0-rc2. --- native/Cargo.lock | 645 ++++++++++++++++++++++++++++------------------ native/Cargo.toml | 8 +- 2 files changed, 393 insertions(+), 260 deletions(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index 8e2ed37f69..35de5ed68d 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -1363,12 +1363,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - [[package]] name = "const-random" version = "0.1.18" @@ -1438,15 +1432,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc32c" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" -dependencies = [ - "rustc_version", -] - [[package]] name = "crc32fast" version = "1.5.0" @@ -1683,7 +1668,7 @@ dependencies = [ [[package]] name = "datafusion" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", "arrow-schema", @@ -1692,25 +1677,25 @@ dependencies = [ "chrono", "datafusion-catalog", "datafusion-catalog-listing", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-common-runtime", "datafusion-datasource", "datafusion-datasource-arrow", "datafusion-datasource-csv", "datafusion-datasource-json", "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", + "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-functions 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-functions-aggregate 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-functions-nested 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-functions-table", "datafusion-functions-window", "datafusion-optimizer", - "datafusion-physical-expr", + "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-physical-optimizer", "datafusion-physical-plan", "datafusion-session", @@ -1733,17 +1718,17 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", + "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-physical-plan", "datafusion-session", "futures", @@ -1757,18 +1742,18 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", "async-trait", "datafusion-catalog", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", + "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-physical-plan", "futures", "itertools 0.14.0", @@ -1793,7 +1778,7 @@ dependencies = [ "datafusion-comet-proto", "datafusion-comet-spark-expr", "datafusion-datasource", - "datafusion-functions-nested", + "datafusion-functions-nested 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", "datafusion-physical-expr-adapter", "datafusion-spark", "futures", @@ -1896,7 +1881,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "ahash", "arrow", @@ -1918,9 +1903,29 @@ dependencies = [ ] [[package]] -name = "datafusion-common-runtime" +name = "datafusion-common" version = "53.0.0" source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +dependencies = [ + "ahash", + "arrow", + "arrow-ipc", + "chrono", + "half", + "hashbrown 0.16.1", + "indexmap 2.13.0", + "itertools 0.14.0", + "libc", + "log", + "paste", + "tokio", + "web-time", +] + +[[package]] +name = "datafusion-common-runtime" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "futures", "log", @@ -1930,7 +1935,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", "async-compression", @@ -1938,13 +1943,13 @@ dependencies = [ "bytes", "bzip2", "chrono", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", + "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-physical-plan", "datafusion-session", "flate2", @@ -1964,18 +1969,18 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", "arrow-ipc", "async-trait", "bytes", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", + "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-physical-plan", "datafusion-session", "futures", @@ -1987,17 +1992,17 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", + "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-physical-plan", "datafusion-session", "futures", @@ -2009,17 +2014,17 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", + "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-physical-plan", "datafusion-session", "futures", @@ -2032,20 +2037,20 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr", + "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-physical-plan", "datafusion-pruning", "datafusion-session", @@ -2058,6 +2063,11 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-doc" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" + [[package]] name = "datafusion-doc" version = "53.0.0" @@ -2066,16 +2076,16 @@ source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c [[package]] name = "datafusion-execution" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", "arrow-buffer", "async-trait", "chrono", "dashmap", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "futures", "log", "object_store 0.13.1", @@ -2086,6 +2096,49 @@ dependencies = [ "url", ] +[[package]] +name = "datafusion-execution" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +dependencies = [ + "arrow", + "arrow-buffer", + "async-trait", + "chrono", + "dashmap", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "futures", + "log", + "object_store 0.13.1", + "parking_lot", + "rand 0.9.2", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" +dependencies = [ + "arrow", + "async-trait", + "chrono", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-functions-window-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "indexmap 2.13.0", + "itertools 0.14.0", + "paste", + "serde_json", + "sqlparser", +] + [[package]] name = "datafusion-expr" version = "53.0.0" @@ -2094,12 +2147,12 @@ dependencies = [ "arrow", "async-trait", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-functions-window-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", "indexmap 2.13.0", "itertools 0.14.0", "paste", @@ -2107,13 +2160,25 @@ dependencies = [ "sqlparser", ] +[[package]] +name = "datafusion-expr-common" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" +dependencies = [ + "arrow", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "indexmap 2.13.0", + "itertools 0.14.0", + "paste", +] + [[package]] name = "datafusion-expr-common" version = "53.0.0" source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ "arrow", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", "indexmap 2.13.0", "itertools 0.14.0", "paste", @@ -2122,7 +2187,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", "arrow-buffer", @@ -2131,12 +2196,12 @@ dependencies = [ "blake3", "chrono", "chrono-tz", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-macros", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-macros 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "hex", "itertools 0.14.0", "log", @@ -2150,6 +2215,54 @@ dependencies = [ "uuid", ] +[[package]] +name = "datafusion-functions" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "chrono", + "chrono-tz", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-macros 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "hex", + "itertools 0.14.0", + "log", + "memchr", + "num-traits", + "rand 0.9.2", + "regex", + "unicode-segmentation", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" +dependencies = [ + "ahash", + "arrow", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-macros 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "half", + "log", + "num-traits", + "paste", +] + [[package]] name = "datafusion-functions-aggregate" version = "53.0.0" @@ -2157,20 +2270,32 @@ source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c dependencies = [ "ahash", "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-macros 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", "half", "log", "num-traits", "paste", ] +[[package]] +name = "datafusion-functions-aggregate-common" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" +dependencies = [ + "ahash", + "arrow", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", +] + [[package]] name = "datafusion-functions-aggregate-common" version = "53.0.0" @@ -2178,9 +2303,33 @@ source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c dependencies = [ "ahash", "arrow", - "datafusion-common", - "datafusion-expr-common", - "datafusion-physical-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", +] + +[[package]] +name = "datafusion-functions-nested" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" +dependencies = [ + "arrow", + "arrow-ord", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-functions 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-functions-aggregate 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-macros 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "hashbrown 0.16.1", + "itertools 0.14.0", + "itoa", + "log", + "paste", ] [[package]] @@ -2190,16 +2339,16 @@ source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c dependencies = [ "arrow", "arrow-ord", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-functions 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-functions-aggregate 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-macros 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", "hashbrown 0.16.1", "itertools 0.14.0", "itoa", @@ -2210,13 +2359,13 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", "async-trait", "datafusion-catalog", - "datafusion-common", - "datafusion-expr", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-physical-plan", "parking_lot", "paste", @@ -2225,27 +2374,46 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-functions-window-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-macros 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "log", "paste", ] +[[package]] +name = "datafusion-functions-window-common" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" +dependencies = [ + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", +] + [[package]] name = "datafusion-functions-window-common" version = "53.0.0" source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ - "datafusion-common", - "datafusion-physical-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", +] + +[[package]] +name = "datafusion-macros" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" +dependencies = [ + "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "quote", + "syn 2.0.117", ] [[package]] @@ -2253,7 +2421,7 @@ name = "datafusion-macros" version = "53.0.0" source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" dependencies = [ - "datafusion-doc", + "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", "quote", "syn 2.0.117", ] @@ -2261,14 +2429,14 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "indexmap 2.13.0", "itertools 0.14.0", "log", @@ -2276,6 +2444,28 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "datafusion-physical-expr" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" +dependencies = [ + "ahash", + "arrow", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "half", + "hashbrown 0.16.1", + "indexmap 2.13.0", + "itertools 0.14.0", + "parking_lot", + "paste", + "petgraph", + "tokio", +] + [[package]] name = "datafusion-physical-expr" version = "53.0.0" @@ -2283,11 +2473,11 @@ source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c dependencies = [ "ahash", "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", "half", "hashbrown 0.16.1", "indexmap 2.13.0", @@ -2301,17 +2491,33 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-functions 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "itertools 0.14.0", ] +[[package]] +name = "datafusion-physical-expr-common" +version = "53.0.0" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" +dependencies = [ + "ahash", + "arrow", + "chrono", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "hashbrown 0.16.1", + "indexmap 2.13.0", + "itertools 0.14.0", + "parking_lot", +] + [[package]] name = "datafusion-physical-expr-common" version = "53.0.0" @@ -2320,8 +2526,8 @@ dependencies = [ "ahash", "arrow", "chrono", - "datafusion-common", - "datafusion-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", "hashbrown 0.16.1", "indexmap 2.13.0", "itertools 0.14.0", @@ -2331,15 +2537,15 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-physical-plan", "datafusion-pruning", "itertools 0.14.0", @@ -2348,22 +2554,22 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "ahash", "arrow", "arrow-ord", "arrow-schema", "async-trait", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-functions 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-functions-window-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "futures", "half", "hashbrown 0.16.1", @@ -2379,14 +2585,14 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-datasource", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-physical-plan", "itertools 0.14.0", "log", @@ -2395,12 +2601,12 @@ dependencies = [ [[package]] name = "datafusion-session" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "async-trait", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "datafusion-physical-plan", "parking_lot", ] @@ -2408,7 +2614,7 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", "bigdecimal", @@ -2416,12 +2622,12 @@ dependencies = [ "crc32fast", "datafusion", "datafusion-catalog", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-functions 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-functions-aggregate 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-functions-nested 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "log", "percent-encoding", "rand 0.9.2", @@ -2434,14 +2640,14 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", "bigdecimal", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-functions-nested", + "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-functions-nested 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", "indexmap 2.13.0", "log", "regex", @@ -2533,7 +2739,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", - "const-oid", "crypto-common", "subtle", ] @@ -2555,15 +2760,6 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8975ffdaa0ef3661bfe02dbdcc06c9f829dfafe6a3c474de366a8d5e44276921" -[[package]] -name = "dlv-list" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" -dependencies = [ - "const-random", -] - [[package]] name = "dunce" version = "1.0.5" @@ -2626,7 +2822,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3231,8 +3427,8 @@ dependencies = [ [[package]] name = "iceberg" -version = "0.8.0" -source = "git+https://github.com/mbutrovich/iceberg-rust?branch=df53-upgrade#646ab672a649322f3b140b0cf3dc76e26bb540a1" +version = "0.9.0" +source = "git+https://github.com/mbutrovich/iceberg-rust?branch=df53-upgrade#934d88a6c21e4389c380397485a35365c77a8a02" dependencies = [ "anyhow", "apache-avro", @@ -3262,11 +3458,9 @@ dependencies = [ "moka", "murmur3", "once_cell", - "opendal", "ordered-float 4.6.0", "parquet", "rand 0.8.5", - "reqsign", "reqwest", "roaring", "serde", @@ -3478,7 +3672,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3526,7 +3720,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4158,7 +4352,6 @@ dependencies = [ "backon", "base64", "bytes", - "crc32c", "futures", "getrandom 0.2.17", "hdrs", @@ -4169,7 +4362,6 @@ dependencies = [ "md-5", "percent-encoding", "quick-xml 0.38.4", - "reqsign", "reqwest", "serde", "serde_json", @@ -4202,16 +4394,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "ordered-multimap" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" -dependencies = [ - "dlv-list", - "hashbrown 0.14.5", -] - [[package]] name = "outref" version = "0.5.2" @@ -4641,16 +4823,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "quick-xml" -version = "0.37.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "quick-xml" version = "0.38.4" @@ -4713,7 +4885,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -4897,35 +5069,6 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" -[[package]] -name = "reqsign" -version = "0.16.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43451dbf3590a7590684c25fb8d12ecdcc90ed3ac123433e500447c7d77ed701" -dependencies = [ - "anyhow", - "async-trait", - "base64", - "chrono", - "form_urlencoded", - "getrandom 0.2.17", - "hex", - "hmac", - "home", - "http 1.4.0", - "log", - "percent-encoding", - "quick-xml 0.37.5", - "rand 0.8.5", - "reqwest", - "rust-ini", - "serde", - "serde_json", - "sha1", - "sha2", - "tokio", -] - [[package]] name = "reqwest" version = "0.12.28" @@ -5002,16 +5145,6 @@ dependencies = [ "byteorder", ] -[[package]] -name = "rust-ini" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" -dependencies = [ - "cfg-if", - "ordered-multimap", -] - [[package]] name = "rustc-demangle" version = "0.1.27" @@ -5062,7 +5195,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5588,7 +5721,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6288,7 +6421,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] diff --git a/native/Cargo.toml b/native/Cargo.toml index abf8a3bf5e..d4118c89a3 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -38,10 +38,10 @@ arrow = { version = "58.0.0", features = ["prettyprint", "ffi", "chrono-tz"] } async-trait = { version = "0.1" } bytes = { version = "1.11.1" } parquet = { version = "58.0.0", default-features = false, features = ["experimental"] } -datafusion = { git = "https://github.com/apache/datafusion", branch = "branch-53", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } -datafusion-datasource = { git = "https://github.com/apache/datafusion", branch = "branch-53" } -datafusion-physical-expr-adapter = { git = "https://github.com/apache/datafusion", branch = "branch-53" } -datafusion-spark = { git = "https://github.com/apache/datafusion", branch = "branch-53", features = ["core"] } +datafusion = { git = "https://github.com/apache/datafusion", tag = "53.0.0-rc2", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } +datafusion-datasource = { git = "https://github.com/apache/datafusion", tag = "53.0.0-rc2" } +datafusion-physical-expr-adapter = { git = "https://github.com/apache/datafusion", tag = "53.0.0-rc2" } +datafusion-spark = { git = "https://github.com/apache/datafusion", tag = "53.0.0-rc2", features = ["core"] } datafusion-comet-spark-expr = { path = "spark-expr" } datafusion-comet-proto = { path = "proto" } chrono = { version = "0.4", default-features = false, features = ["clock"] } From 5a70cc9ab958f9dfcaf1f0bf83a05a4c301ff23e Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Tue, 17 Mar 2026 07:35:07 -0400 Subject: [PATCH 10/15] Merge in upstream/main. --- native/Cargo.lock | 563 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 442 insertions(+), 121 deletions(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index 35de5ed68d..b7be88a62c 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -17,6 +17,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + [[package]] name = "ahash" version = "0.8.12" @@ -87,9 +98,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anyhow" @@ -547,9 +558,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-config" -version = "1.8.14" +version = "1.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a8fc176d53d6fe85017f230405e3255cedb4a02221cb55ed6d76dccbbb099b2" +checksum = "11493b0bad143270fb8ad284a096dd529ba91924c5409adeac856cc1bf047dbc" dependencies = [ "aws-credential-types", "aws-runtime", @@ -567,7 +578,7 @@ dependencies = [ "fastrand", "hex", "http 1.4.0", - "ring", + "sha1", "time", "tokio", "tracing", @@ -577,9 +588,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.13" +version = "1.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d203b0bf2626dcba8665f5cd0871d7c2c0930223d6b6be9097592fea21242d0" +checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -611,9 +622,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede2ddc593e6c8acc6ce3358c28d6677a6dc49b65ba4b37a2befe14a11297e75" +checksum = "5fc0651c57e384202e47153c1260b84a9936e19803d747615edf199dc3b98d17" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -636,9 +647,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.95.0" +version = "1.96.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00c5ff27c6ba2cbd95e6e26e2e736676fdf6bcf96495b187733f521cfe4ce448" +checksum = "f64a6eded248c6b453966e915d32aeddb48ea63ad17932682774eb026fbef5b1" dependencies = [ "aws-credential-types", "aws-runtime", @@ -660,9 +671,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.97.0" +version = "1.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d186f1e5a3694a188e5a0640b3115ccc6e084d104e16fd6ba968dca072ffef8" +checksum = "db96d720d3c622fcbe08bae1c4b04a72ce6257d8b0584cb5418da00ae20a344f" dependencies = [ "aws-credential-types", "aws-runtime", @@ -684,9 +695,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.99.0" +version = "1.100.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9acba7c62f3d4e2408fa998a3a8caacd8b9a5b5549cf36e2372fbdae329d5449" +checksum = "fafbdda43b93f57f699c5dfe8328db590b967b8a820a13ccdd6687355dfcc7ca" dependencies = [ "aws-credential-types", "aws-runtime", @@ -709,9 +720,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37411f8e0f4bea0c3ca0958ce7f18f6439db24d555dbd809787262cd00926aa9" +checksum = "b0b660013a6683ab23797778e21f1f854744fdf05f68204b4cca4c8c04b5d1f4" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -857,9 +868,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.4.6" +version = "1.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b1117b3b2bbe166d11199b540ceed0d0f7676e36e7b962b5a437a9971eac75" +checksum = "9d73dbfbaa8e4bc57b9045137680b958d274823509a360abfd8e1d514d40c95c" dependencies = [ "base64-simd", "bytes", @@ -889,9 +900,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.13" +version = "1.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0470cc047657c6e286346bdf10a8719d26efd6a91626992e0e64481e44323e96" +checksum = "47c8323699dd9b3c8d5b3c13051ae9cdef58fd179957c882f8374dd8725962d9" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -943,6 +954,12 @@ dependencies = [ "vsimd", ] +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bigdecimal" version = "0.4.10" @@ -1049,6 +1066,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "blocking" version = "1.6.2" @@ -1074,9 +1100,9 @@ dependencies = [ [[package]] name = "bon" -version = "3.9.0" +version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d13a61f2963b88eef9c1be03df65d42f6996dfeac1054870d950fcf66686f83" +checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe" dependencies = [ "bon-macros", "rustversion", @@ -1084,9 +1110,9 @@ dependencies = [ [[package]] name = "bon-macros" -version = "3.9.0" +version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d314cc62af2b6b0c65780555abb4d02a03dd3b799cd42419044f0c38d99738c0" +checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" dependencies = [ "darling 0.23.0", "ident_case", @@ -1167,11 +1193,20 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" -version = "1.2.56" +version = "1.2.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" +checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" dependencies = [ "find-msvc-tools", "jobserver", @@ -1268,6 +1303,16 @@ dependencies = [ "half", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clang-sys" version = "1.8.1" @@ -1281,18 +1326,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.60" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a" +checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.5.60" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstyle", "clap_lex", @@ -1300,9 +1345,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "cmake" @@ -1363,6 +1408,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "const-random" version = "0.1.18" @@ -1432,6 +1483,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -1557,16 +1617,6 @@ dependencies = [ "darling_macro 0.20.11", ] -[[package]] -name = "darling" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" -dependencies = [ - "darling_core 0.21.3", - "darling_macro 0.21.3", -] - [[package]] name = "darling" version = "0.23.0" @@ -1591,20 +1641,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "darling_core" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 2.0.117", -] - [[package]] name = "darling_core" version = "0.23.0" @@ -1629,17 +1665,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "darling_macro" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" -dependencies = [ - "darling_core 0.21.3", - "quote", - "syn 2.0.117", -] - [[package]] name = "darling_macro" version = "0.23.0" @@ -1786,6 +1811,7 @@ dependencies = [ "hdrs", "hex", "iceberg", + "iceberg-storage-opendal", "itertools 0.14.0", "jni", "lazy_static", @@ -1872,6 +1898,7 @@ dependencies = [ "num", "rand 0.10.0", "regex", + "serde", "serde_json", "thiserror 2.0.18", "tokio", @@ -1905,7 +1932,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "ahash", "arrow", @@ -2071,7 +2098,7 @@ source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ed [[package]] name = "datafusion-doc" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" [[package]] name = "datafusion-execution" @@ -2099,7 +2126,7 @@ dependencies = [ [[package]] name = "datafusion-execution" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", "arrow-buffer", @@ -2142,7 +2169,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", "async-trait", @@ -2175,7 +2202,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", @@ -2218,7 +2245,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", "arrow-buffer", @@ -2266,7 +2293,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "ahash", "arrow", @@ -2299,7 +2326,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "ahash", "arrow", @@ -2335,7 +2362,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", "arrow-ord", @@ -2400,7 +2427,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", @@ -2419,7 +2446,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", "quote", @@ -2469,7 +2496,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "ahash", "arrow", @@ -2521,7 +2548,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#c466f820c66c08ccc8c7a7d1eaf39de39b4cbd61" +source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "ahash", "arrow", @@ -2663,6 +2690,17 @@ dependencies = [ "uuid", ] +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" version = "0.5.8" @@ -2739,6 +2777,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", "subtle", ] @@ -2756,9 +2795,18 @@ dependencies = [ [[package]] name = "dissimilar" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8975ffdaa0ef3661bfe02dbdcc06c9f829dfafe6a3c474de366a8d5e44276921" +checksum = "aeda16ab4059c5fd2a83f2b9c9e9c981327b18aa8e3b313f7e6563799d4f093e" + +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] [[package]] name = "dunce" @@ -2822,7 +2870,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3478,6 +3526,25 @@ dependencies = [ "zstd", ] +[[package]] +name = "iceberg-storage-opendal" +version = "0.9.0" +source = "git+https://github.com/mbutrovich/iceberg-rust?branch=df53-upgrade#934d88a6c21e4389c380397485a35365c77a8a02" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "cfg-if", + "futures", + "iceberg", + "opendal", + "reqsign", + "reqwest", + "serde", + "typetag", + "url", +] + [[package]] name = "icu_collections" version = "2.1.1" @@ -3633,6 +3700,16 @@ dependencies = [ "str_stack", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "integer-encoding" version = "3.0.4" @@ -3672,7 +3749,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3720,7 +3797,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3793,6 +3870,21 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonwebtoken" +version = "9.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" +dependencies = [ + "base64", + "js-sys", + "pem", + "ring", + "serde", + "serde_json", + "simple_asn1", +] + [[package]] name = "kv-log-macro" version = "1.0.7" @@ -3807,6 +3899,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin 0.9.8", +] [[package]] name = "lazycell" @@ -3885,9 +3980,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.182" +version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" [[package]] name = "libloading" @@ -4025,9 +4120,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lz4_flex" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e" +checksum = "98c23545df7ecf1b16c303910a69b079e8e251d60f7dd2cc9b4177f2afaf1746" dependencies = [ "twox-hash", ] @@ -4177,6 +4272,22 @@ dependencies = [ "serde", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -4332,9 +4443,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oorandom" @@ -4352,6 +4463,7 @@ dependencies = [ "backon", "base64", "bytes", + "crc32c", "futures", "getrandom 0.2.17", "hdrs", @@ -4362,6 +4474,7 @@ dependencies = [ "md-5", "percent-encoding", "quick-xml 0.38.4", + "reqsign", "reqwest", "serde", "serde_json", @@ -4394,6 +4507,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown 0.14.5", +] + [[package]] name = "outref" version = "0.5.2" @@ -4520,12 +4643,41 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + [[package]] name = "peeking_take_while" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -4605,6 +4757,44 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der", + "pbkdf2", + "scrypt", + "sha2", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "pkcs5", + "rand_core 0.6.4", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.32" @@ -4661,9 +4851,9 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "portable-atomic-util" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a9db96d7fa8782dd8c15ce32ffe8680bbd1e978a43bf51a34d39483540495f5" +checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" dependencies = [ "portable-atomic", ] @@ -4699,7 +4889,7 @@ dependencies = [ "nix", "once_cell", "smallvec", - "spin", + "spin 0.10.0", "symbolic-demangle", "tempfile", "thiserror 2.0.18", @@ -4823,6 +5013,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quick-xml" version = "0.38.4" @@ -4855,9 +5055,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.13" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ "bytes", "getrandom 0.3.4", @@ -4885,7 +5085,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -5069,6 +5269,38 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "reqsign" +version = "0.16.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43451dbf3590a7590684c25fb8d12ecdcc90ed3ac123433e500447c7d77ed701" +dependencies = [ + "anyhow", + "async-trait", + "base64", + "chrono", + "form_urlencoded", + "getrandom 0.2.17", + "hex", + "hmac", + "home", + "http 1.4.0", + "jsonwebtoken", + "log", + "once_cell", + "percent-encoding", + "quick-xml 0.37.5", + "rand 0.8.5", + "reqwest", + "rsa", + "rust-ini", + "serde", + "serde_json", + "sha1", + "sha2", + "tokio", +] + [[package]] name = "reqwest" version = "0.12.28" @@ -5145,6 +5377,37 @@ dependencies = [ "byteorder", ] +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "sha2", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rust-ini" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + [[package]] name = "rustc-demangle" version = "0.1.27" @@ -5195,7 +5458,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5259,6 +5522,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + [[package]] name = "same-file" version = "1.0.6" @@ -5270,9 +5542,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] @@ -5307,6 +5579,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -5439,9 +5722,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.17.0" +version = "3.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "381b283ce7bc6b476d903296fb59d0d36633652b633b27f64db4fb46dcbfc3b9" +checksum = "dd5414fad8e6907dbdd5bc441a50ae8d6e26151a03b1de04d89a5576de61d01f" dependencies = [ "base64", "chrono", @@ -5458,11 +5741,11 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.17.0" +version = "3.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6d4e30573c8cb306ed6ab1dca8423eec9a463ea0e155f45399455e0368b27e0" +checksum = "d3db8978e608f1fe7357e211969fd9abdcae80bac1ba7a3369bb7eb6b404eb65" dependencies = [ - "darling 0.21.3", + "darling 0.23.0", "proc-macro2", "quote", "syn 2.0.117", @@ -5519,6 +5802,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + [[package]] name = "simd-adler32" version = "0.3.8" @@ -5531,6 +5824,18 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "simple_asn1" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" +dependencies = [ + "num-bigint", + "num-traits", + "thiserror 2.0.18", + "time", +] + [[package]] name = "siphasher" version = "1.0.2" @@ -5557,14 +5862,20 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "spin" version = "0.10.0" @@ -5574,6 +5885,16 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "sqlparser" version = "0.61.0" @@ -5713,15 +6034,15 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "tempfile" -version = "3.26.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5878,9 +6199,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -6181,9 +6502,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.21.0" +version = "1.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb" +checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -6421,7 +6742,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6845,18 +7166,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.40" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5" +checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.40" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953" +checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" dependencies = [ "proc-macro2", "quote", From eb3198fc14753880db51841972e1a5fd82384b7d Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Tue, 17 Mar 2026 07:47:06 -0400 Subject: [PATCH 11/15] Fix native/core/Cargo.toml. --- native/Cargo.lock | 589 ++++++++++------------------------------- native/core/Cargo.toml | 2 +- 2 files changed, 148 insertions(+), 443 deletions(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index b7be88a62c..70e59fe96d 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -1702,25 +1702,25 @@ dependencies = [ "chrono", "datafusion-catalog", "datafusion-catalog-listing", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", "datafusion-datasource-arrow", "datafusion-datasource-csv", "datafusion-datasource-json", "datafusion-datasource-parquet", - "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-functions 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-functions-aggregate 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-functions-nested 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", "datafusion-functions-table", "datafusion-functions-window", "datafusion-optimizer", - "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr", "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr-common", "datafusion-physical-optimizer", "datafusion-physical-plan", "datafusion-session", @@ -1748,12 +1748,12 @@ dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-plan", "datafusion-session", "futures", @@ -1772,13 +1772,13 @@ dependencies = [ "arrow", "async-trait", "datafusion-catalog", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-common", "datafusion-datasource", - "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr-common", "datafusion-physical-plan", "futures", "itertools 0.14.0", @@ -1803,7 +1803,7 @@ dependencies = [ "datafusion-comet-proto", "datafusion-comet-spark-expr", "datafusion-datasource", - "datafusion-functions-nested 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-functions-nested", "datafusion-physical-expr-adapter", "datafusion-spark", "futures", @@ -1929,26 +1929,6 @@ dependencies = [ "web-time", ] -[[package]] -name = "datafusion-common" -version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" -dependencies = [ - "ahash", - "arrow", - "arrow-ipc", - "chrono", - "half", - "hashbrown 0.16.1", - "indexmap 2.13.0", - "itertools 0.14.0", - "libc", - "log", - "paste", - "tokio", - "web-time", -] - [[package]] name = "datafusion-common-runtime" version = "53.0.0" @@ -1970,13 +1950,13 @@ dependencies = [ "bytes", "bzip2", "chrono", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-common", "datafusion-common-runtime", - "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", "flate2", @@ -2002,12 +1982,12 @@ dependencies = [ "arrow-ipc", "async-trait", "bytes", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", "futures", @@ -2024,12 +2004,12 @@ dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", "futures", @@ -2046,12 +2026,12 @@ dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", "futures", @@ -2069,15 +2049,15 @@ dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr", "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", "datafusion-session", @@ -2095,11 +2075,6 @@ name = "datafusion-doc" version = "53.0.0" source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" -[[package]] -name = "datafusion-doc" -version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" - [[package]] name = "datafusion-execution" version = "53.0.0" @@ -2110,9 +2085,9 @@ dependencies = [ "async-trait", "chrono", "dashmap", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr-common", "futures", "log", "object_store 0.13.1", @@ -2123,28 +2098,6 @@ dependencies = [ "url", ] -[[package]] -name = "datafusion-execution" -version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" -dependencies = [ - "arrow", - "arrow-buffer", - "async-trait", - "chrono", - "dashmap", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "futures", - "log", - "object_store 0.13.1", - "parking_lot", - "rand 0.9.2", - "tempfile", - "url", -] - [[package]] name = "datafusion-expr" version = "53.0.0" @@ -2153,33 +2106,12 @@ dependencies = [ "arrow", "async-trait", "chrono", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-functions-window-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "indexmap 2.13.0", - "itertools 0.14.0", - "paste", - "serde_json", - "sqlparser", -] - -[[package]] -name = "datafusion-expr" -version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" -dependencies = [ - "arrow", - "async-trait", - "chrono", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-functions-window-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-common", + "datafusion-doc", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", "indexmap 2.13.0", "itertools 0.14.0", "paste", @@ -2193,19 +2125,7 @@ version = "53.0.0" source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "indexmap 2.13.0", - "itertools 0.14.0", - "paste", -] - -[[package]] -name = "datafusion-expr-common" -version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" -dependencies = [ - "arrow", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-common", "indexmap 2.13.0", "itertools 0.14.0", "paste", @@ -2223,12 +2143,12 @@ dependencies = [ "blake3", "chrono", "chrono-tz", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-macros 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", "hex", "itertools 0.14.0", "log", @@ -2242,33 +2162,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "datafusion-functions" -version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" -dependencies = [ - "arrow", - "arrow-buffer", - "base64", - "chrono", - "chrono-tz", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-macros 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "hex", - "itertools 0.14.0", - "log", - "memchr", - "num-traits", - "rand 0.9.2", - "regex", - "unicode-segmentation", - "uuid", -] - [[package]] name = "datafusion-functions-aggregate" version = "53.0.0" @@ -2276,35 +2169,14 @@ source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ed dependencies = [ "ahash", "arrow", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-macros 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "half", - "log", - "num-traits", - "paste", -] - -[[package]] -name = "datafusion-functions-aggregate" -version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" -dependencies = [ - "ahash", - "arrow", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-macros 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "half", "log", "num-traits", @@ -2318,21 +2190,9 @@ source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ed dependencies = [ "ahash", "arrow", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", -] - -[[package]] -name = "datafusion-functions-aggregate-common" -version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" -dependencies = [ - "ahash", - "arrow", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", ] [[package]] @@ -2342,40 +2202,16 @@ source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ed dependencies = [ "arrow", "arrow-ord", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-functions 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-functions-aggregate 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-macros 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "hashbrown 0.16.1", - "itertools 0.14.0", - "itoa", - "log", - "paste", -] - -[[package]] -name = "datafusion-functions-nested" -version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" -dependencies = [ - "arrow", - "arrow-ord", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-functions 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-functions-aggregate 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-macros 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr-common", "hashbrown 0.16.1", "itertools 0.14.0", "itoa", @@ -2391,8 +2227,8 @@ dependencies = [ "arrow", "async-trait", "datafusion-catalog", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-common", + "datafusion-expr", "datafusion-physical-plan", "parking_lot", "paste", @@ -2404,13 +2240,13 @@ version = "53.0.0" source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-functions-window-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-macros 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-common", + "datafusion-doc", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "log", "paste", ] @@ -2420,17 +2256,8 @@ name = "datafusion-functions-window-common" version = "53.0.0" source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", -] - -[[package]] -name = "datafusion-functions-window-common" -version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" -dependencies = [ - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-common", + "datafusion-physical-expr-common", ] [[package]] @@ -2438,17 +2265,7 @@ name = "datafusion-macros" version = "53.0.0" source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ - "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "datafusion-macros" -version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" -dependencies = [ - "datafusion-doc 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-doc", "quote", "syn 2.0.117", ] @@ -2460,10 +2277,10 @@ source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ed dependencies = [ "arrow", "chrono", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", "indexmap 2.13.0", "itertools 0.14.0", "log", @@ -2478,33 +2295,11 @@ source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ed dependencies = [ "ahash", "arrow", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "half", - "hashbrown 0.16.1", - "indexmap 2.13.0", - "itertools 0.14.0", - "parking_lot", - "paste", - "petgraph", - "tokio", -] - -[[package]] -name = "datafusion-physical-expr" -version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" -dependencies = [ - "ahash", - "arrow", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", "half", "hashbrown 0.16.1", "indexmap 2.13.0", @@ -2521,11 +2316,11 @@ version = "53.0.0" source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-functions 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "itertools 0.14.0", ] @@ -2537,24 +2332,8 @@ dependencies = [ "ahash", "arrow", "chrono", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "hashbrown 0.16.1", - "indexmap 2.13.0", - "itertools 0.14.0", - "parking_lot", -] - -[[package]] -name = "datafusion-physical-expr-common" -version = "53.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-53#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" -dependencies = [ - "ahash", - "arrow", - "chrono", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", - "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?branch=branch-53)", + "datafusion-common", + "datafusion-expr-common", "hashbrown 0.16.1", "indexmap 2.13.0", "itertools 0.14.0", @@ -2567,12 +2346,12 @@ version = "53.0.0" source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", "itertools 0.14.0", @@ -2588,15 +2367,15 @@ dependencies = [ "arrow-ord", "arrow-schema", "async-trait", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-common", "datafusion-common-runtime", - "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-functions 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-functions-window-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "futures", "half", "hashbrown 0.16.1", @@ -2615,11 +2394,11 @@ version = "53.0.0" source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "arrow", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-common", "datafusion-datasource", - "datafusion-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "datafusion-physical-plan", "itertools 0.14.0", "log", @@ -2631,9 +2410,9 @@ version = "53.0.0" source = "git+https://github.com/apache/datafusion?tag=53.0.0-rc2#28fc91a2a5a8ede5797fde04527f76f351c3cd8b" dependencies = [ "async-trait", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", "datafusion-physical-plan", "parking_lot", ] @@ -2649,12 +2428,12 @@ dependencies = [ "crc32fast", "datafusion", "datafusion-catalog", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-execution 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-functions 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-functions-aggregate 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-functions-nested 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", "log", "percent-encoding", "rand 0.9.2", @@ -2672,9 +2451,9 @@ dependencies = [ "arrow", "bigdecimal", "chrono", - "datafusion-common 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-expr 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", - "datafusion-functions-nested 53.0.0 (git+https://github.com/apache/datafusion?tag=53.0.0-rc2)", + "datafusion-common", + "datafusion-expr", + "datafusion-functions-nested", "indexmap 2.13.0", "log", "regex", @@ -2870,7 +2649,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3749,7 +3528,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3797,7 +3576,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4964,7 +4743,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.13.0", "log", "multimap", "petgraph", @@ -4983,7 +4762,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.117", @@ -5085,7 +4864,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -5458,7 +5237,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6042,7 +5821,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6742,7 +6521,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6837,15 +6616,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -6879,30 +6649,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -6915,12 +6668,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -6933,12 +6680,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -6951,24 +6692,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -6981,12 +6710,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -6999,12 +6722,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -7017,12 +6734,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -7035,12 +6746,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "wit-bindgen" version = "0.51.0" diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml index 2eb3c182cc..6b2b81a79f 100644 --- a/native/core/Cargo.toml +++ b/native/core/Cargo.toml @@ -97,7 +97,7 @@ jni = { version = "0.21", features = ["invocation"] } lazy_static = "1.4" assertables = "9" hex = "0.4.3" -datafusion-functions-nested = { git = "https://github.com/apache/datafusion", branch = "branch-53" } +datafusion-functions-nested = { git = "https://github.com/apache/datafusion", tag = "53.0.0-rc2" } [features] backtrace = ["datafusion/backtrace"] From 0de138170a614e6696d007686d3284c32167eea1 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Tue, 17 Mar 2026 08:10:32 -0400 Subject: [PATCH 12/15] Fix native test failures, clippy. --- native/core/src/execution/jni_api.rs | 2 ++ native/core/src/execution/planner.rs | 17 ++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index 361deae182..858639b025 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -40,6 +40,7 @@ use datafusion::{ prelude::{SessionConfig, SessionContext}, }; use datafusion_comet_proto::spark_operator::Operator; +use datafusion_spark::function::array::repeat::SparkArrayRepeat; use datafusion_spark::function::bitwise::bit_count::SparkBitCount; use datafusion_spark::function::bitwise::bit_get::SparkBitGet; use datafusion_spark::function::bitwise::bitwise_not::SparkBitwiseNot; @@ -389,6 +390,7 @@ fn prepare_datafusion_session_context( // register UDFs from datafusion-spark crate fn register_datafusion_spark_function(session_ctx: &SessionContext) { + session_ctx.register_udf(ScalarUDF::new_from_impl(SparkArrayRepeat::default())); session_ctx.register_udf(ScalarUDF::new_from_impl(SparkExpm1::default())); session_ctx.register_udf(ScalarUDF::new_from_impl(SparkSha2::default())); session_ctx.register_udf(ScalarUDF::new_from_impl(CharFunc::default())); diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 3f1245b05e..3b52d0e954 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -85,6 +85,7 @@ use datafusion::common::{ JoinType as DFJoinType, NullEquality, ScalarValue, }; use datafusion::datasource::listing::PartitionedFile; +use datafusion::logical_expr::type_coercion::functions::fields_with_udf; use datafusion::logical_expr::type_coercion::other::get_coerce_type_for_case_expression; use datafusion::logical_expr::{ AggregateUDF, ReturnFieldArgs, ScalarUDF, WindowFrame, WindowFrameBound, WindowFrameUnits, @@ -2548,15 +2549,14 @@ impl PhysicalPlanner { other => other, }; let func = self.session_ctx.udf(fun_name)?; - let coerced_types = func - .coerce_types(&input_expr_types) - .unwrap_or_else(|_| input_expr_types.clone()); - - let arg_fields = coerced_types + let input_fields: Vec<_> = input_expr_types .iter() .enumerate() .map(|(i, dt)| Arc::new(Field::new(format!("arg{i}"), dt.clone(), true))) - .collect::>(); + .collect(); + let arg_fields = fields_with_udf(&input_fields, func.as_ref())?; + let coerced_types: Vec<_> = + arg_fields.iter().map(|f| f.data_type().clone()).collect(); // TODO this should try and find scalar let arguments = args @@ -4058,6 +4058,9 @@ mod tests { #[test] fn test_array_repeat() { let session_ctx = SessionContext::new(); + session_ctx.register_udf(ScalarUDF::new_from_impl( + datafusion_spark::function::array::repeat::SparkArrayRepeat::default(), + )); let task_ctx = session_ctx.task_ctx(); let planner = PhysicalPlanner::new(Arc::from(session_ctx), 0); @@ -4175,7 +4178,7 @@ mod tests { "+--------------+", "| [0] |", "| [3, 3, 3, 3] |", - "| [] |", + "| |", "+--------------+", ]; assert_batches_eq!(expected, &[batch]); From 7257a24dc25cf81f97fde52ace6349b6967023b8 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Tue, 17 Mar 2026 08:38:52 -0400 Subject: [PATCH 13/15] Update to use object_store 0.13 in hdfs.rs. --- .../src/execution/operators/parquet_writer.rs | 1 + native/hdfs/src/object_store/hdfs.rs | 176 ++++-------------- 2 files changed, 42 insertions(+), 135 deletions(-) diff --git a/native/core/src/execution/operators/parquet_writer.rs b/native/core/src/execution/operators/parquet_writer.rs index bb2f54b2a6..8ba79098d4 100644 --- a/native/core/src/execution/operators/parquet_writer.rs +++ b/native/core/src/execution/operators/parquet_writer.rs @@ -583,6 +583,7 @@ mod tests { /// Helper function to create a test RecordBatch with 1000 rows of (int, string) data /// Example batch_id 1 -> 0..1000, 2 -> 1001..2000 + #[allow(dead_code)] fn create_test_record_batch(batch_id: i32) -> Result { assert!(batch_id > 0, "batch_id must be greater than 0"); let num_rows = batch_id * 1000; diff --git a/native/hdfs/src/object_store/hdfs.rs b/native/hdfs/src/object_store/hdfs.rs index a93774cffe..cb5a2fa5c2 100644 --- a/native/hdfs/src/object_store/hdfs.rs +++ b/native/hdfs/src/object_store/hdfs.rs @@ -31,8 +31,9 @@ use fs_hdfs::walkdir::HdfsWalkDir; use futures::{stream::BoxStream, StreamExt, TryStreamExt}; use object_store::{ path::{self, Path}, - Error, GetOptions, GetRange, GetResult, GetResultPayload, ListResult, MultipartUpload, - ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, + CopyMode, CopyOptions, Error, GetOptions, GetRange, GetResult, GetResultPayload, ListResult, + MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, PutPayload, + PutResult, Result, }; /// scheme for HDFS File System @@ -144,62 +145,6 @@ impl ObjectStore for HadoopFileSystem { unimplemented!() } - async fn get(&self, location: &Path) -> Result { - let hdfs = self.hdfs.clone(); - let hdfs_root = self.hdfs.url().to_owned(); - let location = HadoopFileSystem::path_to_filesystem(location); - - let (blob, object_metadata, range) = maybe_spawn_blocking(move || { - let file = hdfs.open(&location).map_err(to_error)?; - - let file_status = file.get_file_status().map_err(to_error)?; - - let to_read = file_status.len(); - let mut total_read = 0; - let mut buf = vec![0; to_read]; - while total_read < to_read { - let read = file.read(buf.as_mut_slice()).map_err(to_error)?; - if read <= 0 { - break; - } - total_read += read as usize; - } - - if total_read != to_read { - return Err(Error::Generic { - store: "HadoopFileSystem", - source: Box::new(HdfsErr::Generic(format!( - "Error reading path {} with expected size {} and actual size {}", - file.path(), - to_read, - total_read - ))), - }); - } - - file.close().map_err(to_error)?; - - let object_metadata = convert_metadata(file_status.clone(), &hdfs_root); - - let range = Range { - start: 0, - end: file_status.len() as u64, - }; - - Ok((buf.into(), object_metadata, range)) - }) - .await?; - - Ok(GetResult { - payload: GetResultPayload::Stream( - futures::stream::once(async move { Ok(blob) }).boxed(), - ), - meta: object_metadata, - range, - attributes: Default::default(), - }) - } - async fn get_opts(&self, location: &Path, options: GetOptions) -> Result { if options.if_match.is_some() || options.if_none_match.is_some() { return Err(Error::Generic { @@ -249,51 +194,40 @@ impl ObjectStore for HadoopFileSystem { }) } - async fn get_range(&self, location: &Path, range: Range) -> Result { + async fn get_ranges(&self, location: &Path, ranges: &[Range]) -> Result> { let hdfs = self.hdfs.clone(); let location = HadoopFileSystem::path_to_filesystem(location); + let ranges = ranges.to_vec(); maybe_spawn_blocking(move || { let file = hdfs.open(&location).map_err(to_error)?; - let buf = Self::read_range(&range, &file)?; + let result = ranges + .iter() + .map(|range| Self::read_range(range, &file)) + .collect::>>()?; file.close().map_err(to_error)?; - - Ok(buf) + Ok(result) }) .await } - async fn get_ranges(&self, location: &Path, ranges: &[Range]) -> Result> { - coalesce_ranges( - ranges, - |range| self.get_range(location, range), - HDFS_COALESCE_DEFAULT, - ) - .await - } - - async fn head(&self, location: &Path) -> Result { - let hdfs = self.hdfs.clone(); - let hdfs_root = self.hdfs.url().to_owned(); - let location = HadoopFileSystem::path_to_filesystem(location); - - maybe_spawn_blocking(move || { - let file_status = hdfs.get_file_status(&location).map_err(to_error)?; - Ok(convert_metadata(file_status, &hdfs_root)) - }) - .await - } - - async fn delete(&self, location: &Path) -> Result<()> { + fn delete_stream( + &self, + locations: BoxStream<'static, Result>, + ) -> BoxStream<'static, Result> { let hdfs = self.hdfs.clone(); - let location = HadoopFileSystem::path_to_filesystem(location); - - maybe_spawn_blocking(move || { - hdfs.delete(&location, false).map_err(to_error)?; - - Ok(()) - }) - .await + locations + .map(move |location| { + let hdfs = hdfs.clone(); + maybe_spawn_blocking(move || { + let location = location?; + let fs_path = HadoopFileSystem::path_to_filesystem(&location); + hdfs.delete(&fs_path, false).map_err(to_error)?; + Ok(location) + }) + }) + .buffered(10) + .boxed() } /// List all of the leaf files under the prefix path. @@ -402,61 +336,33 @@ impl ObjectStore for HadoopFileSystem { .await } - /// Copy an object from one path to another. - /// If there exists an object at the destination, it will be overwritten. - async fn copy(&self, from: &Path, to: &Path) -> Result<()> { + async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> { let hdfs = self.hdfs.clone(); let from = HadoopFileSystem::path_to_filesystem(from); let to = HadoopFileSystem::path_to_filesystem(to); maybe_spawn_blocking(move || { - // We need to make sure the source exist if !hdfs.exist(&from) { return Err(Error::NotFound { path: from.clone(), source: Box::new(HdfsErr::FileNotFound(from)), }); } - // Delete destination if exists - if hdfs.exist(&to) { - hdfs.delete(&to, false).map_err(to_error)?; - } - - fs_hdfs::util::HdfsUtil::copy(hdfs.as_ref(), &from, hdfs.as_ref(), &to) - .map_err(to_error)?; - - Ok(()) - }) - .await - } - - /// It's only allowed for the same HDFS - async fn rename(&self, from: &Path, to: &Path) -> Result<()> { - let hdfs = self.hdfs.clone(); - let from = HadoopFileSystem::path_to_filesystem(from); - let to = HadoopFileSystem::path_to_filesystem(to); - - maybe_spawn_blocking(move || { - hdfs.rename(&from, &to, true).map_err(to_error)?; - - Ok(()) - }) - .await - } - - /// Copy an object from one path to another, only if destination is empty. - /// Will return an error if the destination already has an object. - async fn copy_if_not_exists(&self, from: &Path, to: &Path) -> Result<()> { - let hdfs = self.hdfs.clone(); - let from = HadoopFileSystem::path_to_filesystem(from); - let to = HadoopFileSystem::path_to_filesystem(to); - maybe_spawn_blocking(move || { - if hdfs.exist(&to) { - return Err(Error::AlreadyExists { - path: from, - source: Box::new(HdfsErr::FileAlreadyExists(to)), - }); + match options.mode { + CopyMode::Overwrite => { + if hdfs.exist(&to) { + hdfs.delete(&to, false).map_err(to_error)?; + } + } + CopyMode::Create => { + if hdfs.exist(&to) { + return Err(Error::AlreadyExists { + path: from, + source: Box::new(HdfsErr::FileAlreadyExists(to)), + }); + } + } } fs_hdfs::util::HdfsUtil::copy(hdfs.as_ref(), &from, hdfs.as_ref(), &to) From 43059fdf7e4b6bca53b31fcd13100f8195e3496d Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Tue, 17 Mar 2026 09:52:45 -0400 Subject: [PATCH 14/15] Fix memory pool issues. --- .../src/execution/memory_pools/fair_pool.rs | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/native/core/src/execution/memory_pools/fair_pool.rs b/native/core/src/execution/memory_pools/fair_pool.rs index 1a98f91e49..e1f686fdc8 100644 --- a/native/core/src/execution/memory_pools/fair_pool.rs +++ b/native/core/src/execution/memory_pools/fair_pool.rs @@ -108,16 +108,21 @@ impl MemoryPool for CometFairMemoryPool { .expect("unexpected amount of unregister happened"); } - fn grow(&self, reservation: &MemoryReservation, additional: usize) { - self.try_grow(reservation, additional).unwrap(); + fn grow(&self, _reservation: &MemoryReservation, additional: usize) { + self.try_grow(_reservation, additional).unwrap(); } - fn shrink(&self, reservation: &MemoryReservation, subtractive: usize) { + fn shrink(&self, _reservation: &MemoryReservation, subtractive: usize) { if subtractive > 0 { let mut state = self.state.lock(); - let size = reservation.size(); - if size < subtractive { - panic!("Failed to release {subtractive} bytes where only {size} bytes reserved") + // We don't use reservation.size() here because DataFusion 53+ decrements + // the reservation's atomic size before calling pool.shrink(), so it would + // reflect the post-shrink value rather than the pre-shrink value. + if state.used < subtractive { + panic!( + "Failed to release {subtractive} bytes where only {} bytes tracked by pool", + state.used + ) } self.release(subtractive) .unwrap_or_else(|_| panic!("Failed to release {subtractive} bytes")); @@ -127,7 +132,7 @@ impl MemoryPool for CometFairMemoryPool { fn try_grow( &self, - reservation: &MemoryReservation, + _reservation: &MemoryReservation, additional: usize, ) -> Result<(), DataFusionError> { if additional > 0 { @@ -137,10 +142,13 @@ impl MemoryPool for CometFairMemoryPool { .pool_size .checked_div(num) .expect("overflow in checked_div"); - let size = reservation.size(); - if limit < size + additional { + // We use state.used instead of reservation.size() because DataFusion 53+ + // calls pool.try_grow() before incrementing the reservation's atomic size, + // so reservation.size() would not include prior grows. + let used = state.used; + if limit < used + additional { return resources_err!( - "Failed to acquire {additional} bytes where {size} bytes already reserved and the fair limit is {limit} bytes, {num} registered" + "Failed to acquire {additional} bytes where {used} bytes already reserved and the fair limit is {limit} bytes, {num} registered" ); } From 1794cfda25cdd1f0d1060ae2bbede2d65a0ef0dc Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Tue, 17 Mar 2026 14:24:37 -0400 Subject: [PATCH 15/15] Fix type coercion for Utf8View stuff. --- native/core/src/execution/planner.rs | 85 +++++++++++++++++++++++++--- 1 file changed, 78 insertions(+), 7 deletions(-) diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 3b52d0e954..11fd7be3f6 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -88,8 +88,8 @@ use datafusion::datasource::listing::PartitionedFile; use datafusion::logical_expr::type_coercion::functions::fields_with_udf; use datafusion::logical_expr::type_coercion::other::get_coerce_type_for_case_expression; use datafusion::logical_expr::{ - AggregateUDF, ReturnFieldArgs, ScalarUDF, WindowFrame, WindowFrameBound, WindowFrameUnits, - WindowFunctionDefinition, + AggregateUDF, ReturnFieldArgs, ScalarUDF, TypeSignature, WindowFrame, WindowFrameBound, + WindowFrameUnits, WindowFunctionDefinition, }; use datafusion::physical_expr::expressions::{Literal, StatsType}; use datafusion::physical_expr::window::WindowExpr; @@ -2549,14 +2549,44 @@ impl PhysicalPlanner { other => other, }; let func = self.session_ctx.udf(fun_name)?; - let input_fields: Vec<_> = input_expr_types + + // Type coercion strategy: + // + // In DF52, Comet used coerce_types() which returns NotImplemented + // for most UDFs, so input types were kept unchanged. In DF53, + // fields_with_udf() runs full coercion which aggressively promotes + // types (e.g. Utf8 to Utf8View via Variadic signatures, Int32 to Int64 + // via Exact signatures). This breaks Comet's native implementations. + // + // Strategy: + // 1. Try coerce_types() — only UDFs that explicitly implement it + // will return Ok. Same as DF52 behavior. + // 2. For "well-supported" signatures (Coercible, String, Numeric, + // Comparable), use fields_with_udf(). These preserve input types + // (e.g. Utf8 stays Utf8, not promoted to Utf8View). + // 3. For all other signatures (Variadic, Exact, etc.), keep original + // types unchanged. Same as DF52 behavior. + let coerced_types = match func.coerce_types(&input_expr_types) { + Ok(types) => types, + Err(_) if needs_fields_coercion(&func.signature().type_signature) => { + let input_fields: Vec<_> = input_expr_types + .iter() + .enumerate() + .map(|(i, dt)| { + Arc::new(Field::new(format!("arg{i}"), dt.clone(), true)) + }) + .collect(); + let arg_fields = fields_with_udf(&input_fields, func.as_ref())?; + arg_fields.iter().map(|f| f.data_type().clone()).collect() + } + Err(_) => input_expr_types.clone(), + }; + + let arg_fields: Vec<_> = coerced_types .iter() .enumerate() .map(|(i, dt)| Arc::new(Field::new(format!("arg{i}"), dt.clone(), true))) .collect(); - let arg_fields = fields_with_udf(&input_fields, func.as_ref())?; - let coerced_types: Vec<_> = - arg_fields.iter().map(|f| f.data_type().clone()).collect(); // TODO this should try and find scalar let arguments = args @@ -2612,10 +2642,33 @@ impl PhysicalPlanner { fun_name, fun_expr, args.to_vec(), - Arc::new(Field::new(fun_name, data_type, true)), + Arc::new(Field::new(fun_name, data_type.clone(), true)), Arc::new(ConfigOptions::default()), )); + // DF53 changed some UDFs (e.g. md5) to return StringViewArray at execution + // time (apache/datafusion#20045). Comet does not yet support view types, so + // cast the result back to the non-view variant. + let scalar_expr = match data_type { + DataType::Utf8View => Arc::new(CastExpr::new( + scalar_expr, + DataType::Utf8, + Some(CastOptions { + safe: false, + ..Default::default() + }), + )) as Arc, + DataType::BinaryView => Arc::new(CastExpr::new( + scalar_expr, + DataType::Binary, + Some(CastOptions { + safe: false, + ..Default::default() + }), + )) as Arc, + _ => scalar_expr, + }; + Ok(scalar_expr) } @@ -3594,6 +3647,24 @@ fn extract_literal_as_datum(expr: &spark_expression::Expr) -> Option bool { + match sig { + TypeSignature::Coercible(_) + | TypeSignature::String(_) + | TypeSignature::Numeric(_) + | TypeSignature::Comparable(_) => true, + TypeSignature::OneOf(sigs) => sigs.iter().any(needs_fields_coercion), + _ => false, + } +} + #[cfg(test)] mod tests { use futures::{poll, StreamExt};