Repository: spiraldb/vortex
Pull request #2417: Datafusion 46 support
Author: login: AdamGS
Created: 2025-02-18T17:54:03Z | Merged: 2025-03-09T17:18:04Z

I mostly just like to see changes in Github.
We can merge this once Datafusion 46 released, assuming it includes the following PRs:
- apache/datafusion#14754
- https://github.com/apache/datafusion/pull/14671
- also hoping for https://github.com/apache/datafusion/pull/14798

Seems like everything we wanted to get in made it, this branch now compiles AND passes all checks, so we're only waiting for the actual release in https://github.com/apache/datafusion/issues/14123


Diff:
diff --git a/Cargo.lock b/Cargo.lock
index 488a809de51..fe226709fcc 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1307,26 +1307,29 @@ dependencies = [
 
 [[package]]
 name = "datafusion"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eae420e7a5b0b7f1c39364cc76cbcd0f5fdc416b2514ae3847c2676bbd60702a"
+checksum = "46b879c1aa3a85ecbfa376704f0fe4bfebae1a44a5d35faa4466bf85469b6a0e"
 dependencies = [
  "arrow",
- "arrow-array",
  "arrow-ipc",
  "arrow-schema",
  "async-trait",
  "bytes",
  "chrono",
  "datafusion-catalog",
+ "datafusion-catalog-listing",
  "datafusion-common",
  "datafusion-common-runtime",
+ "datafusion-datasource",
  "datafusion-execution",
  "datafusion-expr",
+ "datafusion-expr-common",
  "datafusion-functions",
  "datafusion-functions-aggregate",
  "datafusion-functions-table",
  "datafusion-functions-window",
+ "datafusion-macros",
  "datafusion-optimizer",
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
@@ -1334,7 +1337,6 @@ dependencies = [
  "datafusion-physical-plan",
  "datafusion-sql",
  "futures",
- "glob",
  "itertools 0.14.0",
  "log",
  "object_store",
@@ -1351,9 +1353,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f27987bc22b810939e8dfecc55571e9d50355d6ea8ec1c47af8383a76a6d0e1"
+checksum = "9e42f516243fe30137f2b7d5712611286baf8d1d758a46157bada7c35fdf38df"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1367,21 +1369,39 @@ dependencies = [
  "itertools 0.14.0",
  "log",
  "parking_lot",
- "sqlparser",
+]
+
+[[package]]
+name = "datafusion-catalog-listing"
+version = "46.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e495290c231d617f0a940860a885cb2f4c3efe46c1983c30d3fa12faf1ccb208"
+dependencies = [
+ "arrow",
+ "async-trait",
+ "datafusion-catalog",
+ "datafusion-common",
+ "datafusion-datasource",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
+ "futures",
+ "log",
+ "object_store",
+ "tokio",
 ]
 
 [[package]]
 name = "datafusion-common"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3f6d5b8c9408cc692f7c194b8aa0c0f9b253e065a8d960ad9cdc2a13e697602"
+checksum = "af67ddc82e1c8e6843c326ca13aa20e5420cce9f886b4e1ee39ea43defae3145"
 dependencies = [
  "ahash",
  "arrow",
- "arrow-array",
- "arrow-buffer",
  "arrow-ipc",
- "arrow-schema",
  "base64",
  "half",
  "hashbrown 0.14.5",
@@ -1398,25 +1418,53 @@ dependencies = [
 
 [[package]]
 name = "datafusion-common-runtime"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d4603c8e8a4baf77660ab7074cc66fc15cc8a18f2ce9dfadb755fc6ee294e48"
+checksum = "36ee9403a2ec39183437825d232f556a5dee89f13f6fd78f8c7f8f999489e4ca"
 dependencies = [
  "log",
  "tokio",
 ]
 
+[[package]]
+name = "datafusion-datasource"
+version = "46.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8c8b7568b638dd309bcc1cdeb66776f233b110d44bdc6fd67ef1919f9ec9803"
+dependencies = [
+ "arrow",
+ "async-trait",
+ "bytes",
+ "chrono",
+ "datafusion-catalog",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
+ "futures",
+ "glob",
+ "itertools 0.14.0",
+ "log",
+ "object_store",
+ "rand 0.8.5",
+ "tokio",
+ "url",
+]
+
 [[package]]
 name = "datafusion-doc"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5bf4bc68623a5cf231eed601ed6eb41f46a37c4d15d11a0bff24cbc8396cd66"
+checksum = "8612c81304578a2e2b82d31caf8173312cb086a7a23a23556b9fff3ac7c18221"
 
 [[package]]
 name = "datafusion-execution"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88b491c012cdf8e051053426013429a76f74ee3c2db68496c79c323ca1084d27"
+checksum = "3591e6d4900e57bad7f861f14f5c763f716da76553b0d037ec91c192c876f09c"
 dependencies = [
  "arrow",
  "dashmap",
@@ -1433,9 +1481,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-expr"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5a181408d4fc5dc22f9252781a8f39f2d0e5d1b33ec9bde242844980a2689c1"
+checksum = "5033d0f6198d177f50a7721d80db141af15dd12f45ad6dce34e2cdbb6538e39d"
 dependencies = [
  "arrow",
  "chrono",
@@ -1453,21 +1501,22 @@ dependencies = [
 
 [[package]]
 name = "datafusion-expr-common"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1129b48e8534d8c03c6543bcdccef0b55c8ac0c1272a15a56c67068b6eb1885"
+checksum = "56def48a7dfb9f92aa18e18dfdffaca79b5383f03c59bb0107959c1698634557"
 dependencies = [
  "arrow",
  "datafusion-common",
+ "indexmap",
  "itertools 0.14.0",
  "paste",
 ]
 
 [[package]]
 name = "datafusion-functions"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6125874e4856dfb09b59886784fcb74cde5cfc5930b3a80a1a728ef7a010df6b"
+checksum = "9a79b703b42b0aac97485b84c6810c78114b0974a75a33514840ba0bbe0de38f"
 dependencies = [
  "arrow",
  "arrow-buffer",
@@ -1479,7 +1528,6 @@ dependencies = [
  "datafusion-expr",
  "datafusion-expr-common",
  "datafusion-macros",
- "hashbrown 0.14.5",
  "hex",
  "itertools 0.14.0",
  "log",
@@ -1491,14 +1539,12 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-aggregate"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3add7b1d3888e05e7c95f2b281af900ca69ebdcb21069ba679b33bde8b3b9d6"
+checksum = "fdad20375e85365ed262b5583955c308840efc6ff9271ff463cf86789adfb686"
 dependencies = [
  "ahash",
  "arrow",
- "arrow-buffer",
- "arrow-schema",
  "datafusion-common",
  "datafusion-doc",
  "datafusion-execution",
@@ -1514,9 +1560,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-aggregate-common"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e18baa4cfc3d2f144f74148ed68a1f92337f5072b6dde204a0dbbdf3324989c"
+checksum = "ff73249ee3cdc81ad04317d3b4231fc02a8c03a3a1b4b13953244e6443f6b498"
 dependencies = [
  "ahash",
  "arrow",
@@ -1527,9 +1573,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-table"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c403ddd473bbb0952ba880008428b3c7febf0ed3ce1eec35a205db20efb2a36"
+checksum = "ac12628c3e43461118e95d5772f729e1cc39db883d8ee52e4b80038b0f614bbf"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1543,9 +1589,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-window"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ab18c2fb835614d06a75f24a9e09136d3a8c12a92d97c95a6af316a1787a9c5"
+checksum = "03eb449555c7cc03bb61d43d90edef70d070d34bc4a0d8f7e358d157232f3220"
 dependencies = [
  "datafusion-common",
  "datafusion-doc",
@@ -1560,9 +1606,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-window-common"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a77b73bc15e7d1967121fdc7a55d819bfb9d6c03766a6c322247dce9094a53a4"
+checksum = "6a0c7606e568ee6a15d33a2532eb0d18e7769bb88af55f6b70be4db9fd937d18"
 dependencies = [
  "datafusion-common",
  "datafusion-physical-expr-common",
@@ -1570,9 +1616,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-macros"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09369b8d962291e808977cf94d495fd8b5b38647232d7ef562c27ac0f495b0af"
+checksum = "64030e805d3d257e3012e4378500d4ac90b1ebacd03f1110e8ec927b77f09486"
 dependencies = [
  "datafusion-expr",
  "quote",
@@ -1581,9 +1627,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-optimizer"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2403a7e4a84637f3de7d8d4d7a9ccc0cc4be92d89b0161ba3ee5be82f0531c54"
+checksum = "ae6af7bdae7565aa7a4cb1deb7fe18d89c63c5d93b5203b473ca1dbe02a1cd3d"
 dependencies = [
  "arrow",
  "chrono",
@@ -1599,15 +1645,12 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86ff72ac702b62dbf2650c4e1d715ebd3e4aab14e3885e72e8549e250307347c"
+checksum = "3f68601feda90c255c9023a881e833efca9d7539bab0565ac1355b0249326e91"
 dependencies = [
  "ahash",
  "arrow",
- "arrow-array",
- "arrow-buffer",
- "arrow-schema",
  "datafusion-common",
  "datafusion-expr",
  "datafusion-expr-common",
@@ -1624,13 +1667,12 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr-common"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60982b7d684e25579ee29754b4333057ed62e2cc925383c5f0bd8cab7962f435"
+checksum = "00c1a08b00d340ca3bc1cd2f094ecaeaf6f099a2980e11255976660fa0409182"
 dependencies = [
  "ahash",
  "arrow",
- "arrow-buffer",
  "datafusion-common",
  "datafusion-expr-common",
  "hashbrown 0.14.5",
@@ -1639,12 +1681,11 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-optimizer"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac5e85c189d5238a5cf181a624e450c4cd4c66ac77ca551d6f3ff9080bac90bb"
+checksum = "7cd34f3438cf9629ea0e3425027582334fb6671a05ee43671ca3c47896b75dda"
 dependencies = [
  "arrow",
- "arrow-schema",
  "datafusion-common",
  "datafusion-execution",
  "datafusion-expr",
@@ -1652,22 +1693,18 @@ dependencies = [
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
  "datafusion-physical-plan",
- "futures",
  "itertools 0.14.0",
  "log",
- "url",
 ]
 
 [[package]]
 name = "datafusion-physical-plan"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c36bf163956d7e2542657c78b3383fdc78f791317ef358a359feffcdb968106f"
+checksum = "7624484ada341d30ef465eae61f760e779f080c621bbc3dc0335a75fa78e8dec"
 dependencies = [
  "ahash",
  "arrow",
- "arrow-array",
- "arrow-buffer",
  "arrow-ord",
  "arrow-schema",
  "async-trait",
@@ -1692,13 +1729,11 @@ dependencies = [
 
 [[package]]
 name = "datafusion-sql"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e13caa4daede211ecec53c78b13c503b592794d125f9a3cc3afe992edf9e7f43"
+checksum = "1e717736a394ed92d9dcf2d74439c655474dd39aa65a064a6bae697b6d20e5fe"
 dependencies = [
  "arrow",
- "arrow-array",
- "arrow-schema",
  "bigdecimal",
  "datafusion-common",
  "datafusion-expr",
@@ -3625,6 +3660,15 @@ dependencies = [
  "prost",
 ]
 
+[[package]]
+name = "psm"
+version = "0.1.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f58e5423e24c18cc840e1c98370b3993c6649cd1678b4d24318bcf0a083cbe88"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "ptr_meta"
 version = "0.3.0"
@@ -3950,6 +3994,26 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "recursive"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e"
+dependencies = [
+ "recursive-proc-macro-impl",
+ "stacker",
+]
+
+[[package]]
+name = "recursive-proc-macro-impl"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b"
+dependencies = [
+ "quote",
+ "syn 2.0.98",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.5.9"
@@ -4561,11 +4625,12 @@ dependencies = [
 
 [[package]]
 name = "sqlparser"
-version = "0.53.0"
+version = "0.54.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05a528114c392209b3264855ad491fcce534b94a38771b0a0b97a79379275ce8"
+checksum = "c66e3b7374ad4a6af849b08b3e7a6eda0edbd82f0fd59b57e22671bf16979899"
 dependencies = [
  "log",
+ "recursive",
  "sqlparser_derive",
 ]
 
@@ -4586,6 +4651,19 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
 
+[[package]]
+name = "stacker"
+version = "0.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d08feb8f695b465baed819b03c128dc23f57a694510ab1f06c77f763975685e"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "libc",
+ "psm",
+ "windows-sys 0.59.0",
+]
+
 [[package]]
 name = "static_assertions"
 version = "1.1.0"
@@ -5377,6 +5455,7 @@ dependencies = [
  "rand 0.9.0",
  "tempfile",
  "tokio",
+ "tokio-stream",
  "tracing",
  "tracing-futures",
  "url",
diff --git a/Cargo.toml b/Cargo.toml
index df0bac73ed5..d0f695caa07 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -53,32 +53,32 @@ version = "0.25.2"
 anyhow = "1.0.95"
 arbitrary = "1.3.2"
 arrayref = "0.3.7"
-arrow = "54.2"
-arrow-arith = "54.2"
-arrow-array = "54.2"
-arrow-buffer = "54.2"
-arrow-cast = "54.2"
-arrow-ord = "54.2"
-arrow-schema = "54.2"
-arrow-select = "54.2"
-arrow-string = "54.2"
+arrow = "54.2.1"
+arrow-arith = "54.2.1"
+arrow-array = "54.2.1"
+arrow-buffer = "54.2.1"
+arrow-cast = "54.2.1"
+arrow-ord = "54.2.1"
+arrow-schema = "54.2.1"
+arrow-select = "54.2.1"
+arrow-string = "54.2.1"
 async-once-cell = "0.5.4"
 async-trait = "0.1.86"
 backtrace = "0.3.74"
 bit-vec = "0.8.0"
-bytes = "1.9"
+bytes = "1.10"
 bzip2 = "0.5.0"
 cfg-if = "1"
 chrono = "0.4.38"
 clap = "4"
 compio = { version = "0.13", features = ["io-uring"], default-features = false }
 crossterm = "0.28"
-datafusion = { version = "45", default-features = false }
-datafusion-common = { version = "45", default-features = false }
-datafusion-execution = "45"
-datafusion-expr = "45"
-datafusion-physical-expr = "45"
-datafusion-physical-plan = "45"
+datafusion = { version = "46", default-features = false }
+datafusion-common = { version = "46" }
+datafusion-execution = { version = "46" }
+datafusion-expr = { version = "46" }
+datafusion-physical-expr = { version = "46" }
+datafusion-physical-plan = { version = "46" }
 divan = { package = "codspeed-divan-compat", version = "2.8.0" }
 dyn-hash = "0.2.0"
 enum-iterator = "2.0.0"
@@ -113,7 +113,7 @@ opentelemetry = "0.28.0"
 opentelemetry-otlp = "0.28.0"
 opentelemetry_sdk = "0.28.0"
 parking_lot = "0.12.3"
-parquet = "54.2"
+parquet = "54.2.1"
 paste = "1.0.15"
 pin-project = "1.1.5"
 pin-project-lite = "0.2.15"
@@ -153,13 +153,14 @@ tar = "0.4"
 tempfile = "3"
 thiserror = "2.0.3"
 tikv-jemallocator = "0.6"
-tokio = "1.36"
+tokio = "1.43"
+tokio-stream = "0.1.17"
 tracing = { version = "0.1.41" }
 tracing-chrome = "0.7.2"
 tracing-futures = "0.2.5"
 tracing-subscriber = "0.3.19"
 url = "2.5.4"
-uuid = { version = "1.14", features = ["js"] }
+uuid = { version = "1.15", features = ["js"] }
 wasm-bindgen-futures = "0.4.39"
 witchcraft-metrics = "1.0.1"
 
diff --git a/vortex-datafusion/Cargo.toml b/vortex-datafusion/Cargo.toml
index 6d8634d0307..1a212b43dbc 100644
--- a/vortex-datafusion/Cargo.toml
+++ b/vortex-datafusion/Cargo.toml
@@ -32,6 +32,7 @@ object_store = { workspace = true }
 pin-project = { workspace = true }
 rand = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "fs"] }
+tokio-stream = { workspace = true }
 tracing = { workspace = true, optional = true }
 tracing-futures = { workspace = true, features = [
     "futures-03",
diff --git a/vortex-datafusion/src/lib.rs b/vortex-datafusion/src/lib.rs
index 1dd41b245b2..2eae23a5921 100644
--- a/vortex-datafusion/src/lib.rs
+++ b/vortex-datafusion/src/lib.rs
@@ -1,6 +1,5 @@
 //! Connectors to enable DataFusion to read Vortex data.
 #![deny(missing_docs)]
-#![allow(clippy::nonminimal_bool)]
 #![allow(clippy::cast_possible_truncation)]
 
 use std::fmt::Debug;
diff --git a/vortex-datafusion/src/memory/provider.rs b/vortex-datafusion/src/memory/provider.rs
index d68ac1f2d49..35cf563a408 100644
--- a/vortex-datafusion/src/memory/provider.rs
+++ b/vortex-datafusion/src/memory/provider.rs
@@ -185,7 +185,7 @@ mod test {
     use arrow_schema::{DataType, Field, Schema};
     use datafusion::functions_aggregate::count::count_distinct;
     use datafusion::prelude::SessionContext;
-    use datafusion_common::{Column, TableReference};
+    use datafusion_common::{Column, Spans, TableReference};
     use datafusion_expr::{BinaryExpr, Expr, Operator, and, col, lit};
     use vortex_array::arrays::{PrimitiveArray, StructArray, VarBinViewArray};
     use vortex_array::{Array, ArrayRef};
@@ -249,6 +249,7 @@ mod test {
                         table: "orders".into(),
                     }),
                     name: "o_orderstatus".to_string(),
+                    spans: Spans::new(),
                 }
                 .into(),
             ),
diff --git a/vortex-datafusion/src/persistent/config.rs b/vortex-datafusion/src/persistent/config.rs
index 60bdb7cb6f4..bcb9586e40c 100644
--- a/vortex-datafusion/src/persistent/config.rs
+++ b/vortex-datafusion/src/persistent/config.rs
@@ -35,6 +35,7 @@ pub struct ConfigProjection {
     pub arrow_schema: SchemaRef,
     pub constraints: Constraints,
     pub statistics: Statistics,
+    #[allow(dead_code)]
     pub orderings: Vec<LexOrdering>,
     pub projection_expr: Arc<dyn VortexExpr>,
 }
diff --git a/vortex-datafusion/src/persistent/execution.rs b/vortex-datafusion/src/persistent/execution.rs
deleted file mode 100644
index 57d5f5278ac..00000000000
--- a/vortex-datafusion/src/persistent/execution.rs
+++ /dev/null
@@ -1,319 +0,0 @@
-use std::collections::VecDeque;
-use std::fmt;
-use std::sync::Arc;
-
-use arrow_schema::SchemaRef;
-use datafusion::config::ConfigOptions;
-use datafusion::datasource::listing::PartitionedFile;
-use datafusion::datasource::physical_plan::{FileScanConfig, FileStream};
-use datafusion_common::{Result as DFResult, Statistics};
-use datafusion_execution::{SendableRecordBatchStream, TaskContext};
-use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr};
-use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};
-use datafusion_physical_plan::metrics::MetricsSet;
-use datafusion_physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties};
-use itertools::Itertools;
-use object_store::ObjectStoreScheme;
-use vortex_error::VortexExpect;
-use vortex_expr::datafusion::convert_expr_to_vortex;
-use vortex_expr::{VortexExpr, and};
-
-use super::cache::FooterCache;
-use super::config::{ConfigProjection, FileScanConfigExt};
-use super::metrics::VortexExecMetrics;
-use crate::persistent::metrics::PARTITION_LABEL;
-use crate::persistent::opener::VortexFileOpener;
-
-#[derive(Debug, Clone)]
-pub(crate) struct VortexExec {
-    file_scan_config: FileScanConfig,
-    metrics: VortexExecMetrics,
-    predicate: Option<Arc<dyn VortexExpr>>,
-    plan_properties: PlanProperties,
-    projected_statistics: Statistics,
-    initial_read_cache: FooterCache,
-    projected_arrow_schema: SchemaRef,
-    projection: Arc<dyn VortexExpr>,
-}
-
-impl VortexExec {
-    pub fn try_new(
-        file_scan_config: FileScanConfig,
-        metrics: VortexExecMetrics,
-        predicate: Option<Arc<dyn PhysicalExpr>>,
-        initial_read_cache: FooterCache,
-    ) -> DFResult<Self> {
-        let ConfigProjection {
-            arrow_schema,
-            constraints: _constraints,
-            mut statistics,
-            orderings,
-            projection_expr,
-        } = file_scan_config.project_for_vortex();
-
-        let predicate = make_vortex_predicate(predicate);
-
-        // We must take care to report in-exact statistics if we have any form of filter
-        // push-down.
-        if predicate.is_some() {
-            statistics = statistics.to_inexact();
-        }
-
-        let plan_properties = PlanProperties::new(
-            EquivalenceProperties::new_with_orderings(arrow_schema.clone(), &orderings),
-            Partitioning::UnknownPartitioning(file_scan_config.file_groups.len()),
-            EmissionType::Incremental,
-            Boundedness::Bounded,
-        );
-
-        Ok(Self {
-            file_scan_config,
-            metrics,
-            predicate,
-            plan_properties,
-            initial_read_cache,
-            projected_statistics: statistics,
-            projected_arrow_schema: arrow_schema,
-            projection: projection_expr,
-        })
-    }
-
-    pub(crate) fn into_arc(self) -> Arc<dyn ExecutionPlan> {
-        Arc::new(self) as _
-    }
-}
-
-impl DisplayAs for VortexExec {
-    fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "VortexExec: ")?;
-        self.file_scan_config.fmt_as(t, f)?;
-
-        Ok(())
-    }
-}
-
-impl ExecutionPlan for VortexExec {
-    fn name(&self) -> &str {
-        "VortexExec"
-    }
-
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
-    fn properties(&self) -> &PlanProperties {
-        &self.plan_properties
-    }
-
-    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
-        vec![]
-    }
-
-    fn with_new_children(
-        self: Arc<Self>,
-        _children: Vec<Arc<dyn ExecutionPlan>>,
-    ) -> DFResult<Arc<dyn ExecutionPlan>> {
-        Ok(self)
-    }
-
-    fn execute(
-        &self,
-        partition: usize,
-        context: Arc<TaskContext>,
-    ) -> DFResult<SendableRecordBatchStream> {
-        log::debug!("Executing partition {partition}");
-        let partition_metrics = self
-            .metrics
-            .child_with_tags([(PARTITION_LABEL, partition.to_string())].into_iter());
-        let object_store = context
-            .runtime_env()
-            .object_store(&self.file_scan_config.object_store_url)?;
-        let (scheme, _) = ObjectStoreScheme::parse(self.file_scan_config.object_store_url.as_ref())
-            .map_err(object_store::Error::from)?;
-
-        let opener = VortexFileOpener::new(
-            scheme,
-            object_store,
-            self.projection.clone(),
-            self.predicate.clone(),
-            self.initial_read_cache.clone(),
-            self.projected_arrow_schema.clone(),
-            context.session_config().batch_size(),
-            partition_metrics,
-        )?;
-        let stream = FileStream::new(
-            &self.file_scan_config,
-            partition,
-            opener,
-            &self.metrics.execution_plan,
-        )?;
-
-        Ok(Box::pin(stream))
-    }
-
-    fn statistics(&self) -> DFResult<Statistics> {
-        Ok(self.projected_statistics.clone())
-    }
-
-    fn metrics(&self) -> Option<MetricsSet> {
-        Some(self.metrics.metrics_set())
-    }
-
-    fn repartitioned(
-        &self,
-        target_partitions: usize,
-        config: &ConfigOptions,
-    ) -> DFResult<Option<Arc<dyn ExecutionPlan>>> {
-        let all_files = self
-            .file_scan_config
-            .file_groups
-            .iter()
-            .flatten()
-            .collect::<Vec<_>>();
-
-        let total_size = all_files.iter().map(|f| f.object_meta.size).sum::<usize>();
-
-        // If there's one file or less total files in the scan, we can't repartition it
-        if all_files.len() <= 1 {
-            return Ok(None);
-        }
-
-        if total_size < config.optimizer.repartition_file_min_size {
-            return Ok(None);
-        }
-
-        let repartitioned_file_groups = repartition_by_size(all_files, target_partitions);
-        let mut new_plan = self.clone();
-        let num_partitions = repartitioned_file_groups.len();
-
-        log::debug!("VortexExec repartitioned to {num_partitions} partitions");
-        new_plan.file_scan_config.file_groups = repartitioned_file_groups;
-        new_plan.plan_properties.partitioning = Partitioning::UnknownPartitioning(num_partitions);
-
-        Ok(Some(Arc::new(new_plan)))
-    }
-}
-
-fn make_vortex_predicate(predicate: Option<Arc<dyn PhysicalExpr>>) -> Option<Arc<dyn VortexExpr>> {
-    predicate
-        .as_ref()
-        // If we cannot convert an expr to a vortex expr, we run no filter, since datafusion
-        // will rerun the filter expression anyway.
-        .and_then(|expr| {
-            // This splits expressions into conjunctions and converts them to vortex expressions.
-            // Any inconvertible expressions are dropped since true /\ a == a.
-            datafusion_physical_expr::split_conjunction(expr)
-                .into_iter()
-                .filter_map(|e| convert_expr_to_vortex(e.clone()).ok())
-                .reduce(and)
-        })
-}
-
-fn repartition_by_size(
-    all_files: Vec<&PartitionedFile>,
-    desired_partitions: usize,
-) -> Vec<Vec<PartitionedFile>> {
-    let total_file_count = all_files.len();
-    let total_size = all_files.iter().map(|f| f.object_meta.size).sum::<usize>();
-    let target_partition_size = total_size / desired_partitions;
-
-    let mut partitions = Vec::with_capacity(desired_partitions);
-
-    let mut curr_partition_size = 0;
-    let mut curr_partition = Vec::default();
-
-    let mut all_files = VecDeque::from_iter(
-        all_files
-            .into_iter()
-            .sorted_unstable_by_key(|f| f.object_meta.size),
-    );
-
-    while !all_files.is_empty() && partitions.len() < desired_partitions {
-        // If the current partition is empty, we want to bootstrap it with the biggest file we have leftover.
-        let file = if curr_partition.is_empty() {
-            all_files.pop_back()
-        // If we already have files in the partition, we try and fill it up.
-        } else {
-            // Peak at the biggest file left
-            let biggest_file_size = all_files
-                .back()
-                .vortex_expect("We must have at least one item")
-                .object_meta
-                .size;
-
-            let smallest_file_size = all_files
-                .front()
-                .vortex_expect("We must have at least one item")
-                .object_meta
-                .size;
-
-            // We try and find a file on either end that fits in the partition
-            if curr_partition_size + biggest_file_size >= target_partition_size {
-                all_files.pop_front()
-            } else if curr_partition_size + smallest_file_size >= target_partition_size {
-                all_files.pop_back()
-            } else {
-                None
-            }
-        };
-
-        // Add a file to the partition
-        if let Some(file) = file {
-            curr_partition_size += file.object_meta.size;
-            curr_partition.push(file.clone());
-        }
-
-        // If the partition is full, move on to the next one
-        if curr_partition_size >= target_partition_size || file.is_none() {
-            curr_partition_size = 0;
-            partitions.push(std::mem::take(&mut curr_partition));
-        }
-    }
-
-    // If we we're still missing the last partition
-    if !curr_partition.is_empty() && partitions.len() != desired_partitions {
-        partitions.push(std::mem::take(&mut curr_partition));
-    } else if !curr_partition.is_empty() {
-        for (idx, file) in curr_partition.into_iter().enumerate() {
-            let new_part_idx = idx % partitions.len();
-            partitions[new_part_idx].push(file.clone());
-        }
-    }
-
-    for (idx, file) in all_files.into_iter().enumerate() {
-        let new_part_idx = idx % partitions.len();
-        partitions[new_part_idx].push(file.clone());
-    }
-
-    // Assert that we have the correct number of partitions and that the total number of files is right
-    assert_eq!(total_file_count, partitions.iter().flatten().count());
-
-    partitions
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn basic_repartition_test() {
-        let file_groups = vec![
-            PartitionedFile::new("a", 100),
-            PartitionedFile::new("b", 25),
-            PartitionedFile::new("c", 25),
-            PartitionedFile::new("d", 25),
-            PartitionedFile::new("e", 50),
-        ];
-
-        let groups = repartition_by_size(file_groups.iter().collect(), 2);
-
-        assert_eq!(groups.len(), 2);
-
-        let file_groups = (0..100)
-            .map(|idx| PartitionedFile::new(format!("{idx}"), idx))
-            .collect::<Vec<_>>();
-
-        let groups = repartition_by_size(file_groups.iter().collect(), 16);
-        assert_eq!(groups.len(), 16);
-    }
-}
diff --git a/vortex-datafusion/src/persistent/format.rs b/vortex-datafusion/src/persistent/format.rs
index 6d7ac0c79ff..92dd5c6b3ca 100644
--- a/vortex-datafusion/src/persistent/format.rs
+++ b/vortex-datafusion/src/persistent/format.rs
@@ -3,10 +3,10 @@ use std::sync::Arc;
 
 use arrow_schema::{Schema, SchemaRef};
 use async_trait::async_trait;
+use datafusion::catalog::Session;
 use datafusion::datasource::file_format::file_compression_type::FileCompressionType;
 use datafusion::datasource::file_format::{FileFormat, FileFormatFactory, FilePushdownSupport};
-use datafusion::datasource::physical_plan::{FileScanConfig, FileSinkConfig};
-use datafusion::execution::SessionState;
+use datafusion::datasource::physical_plan::{FileScanConfig, FileSinkConfig, FileSource};
 use datafusion_common::parsers::CompressionTypeVariant;
 use datafusion_common::stats::Precision;
 use datafusion_common::{
@@ -26,13 +26,16 @@ use vortex_array::{ArrayRegistry, stats};
 use vortex_dtype::DType;
 use vortex_dtype::arrow::FromArrowType;
 use vortex_error::{VortexExpect, VortexResult, vortex_err};
+use vortex_expr::datafusion::convert_expr_to_vortex;
+use vortex_expr::{VortexExpr, and};
 use vortex_file::{VORTEX_FILE_EXTENSION, VortexOpenOptions};
 use vortex_io::ObjectStoreReadAt;
 use vortex_layout::{LayoutRegistry, LayoutRegistryExt};
 
 use super::cache::FooterCache;
-use super::execution::VortexExec;
+use super::metrics::VortexSourceMetrics;
 use super::sink::VortexSink;
+use super::source::VortexSource;
 use crate::{PrecisionExt as _, can_be_pushed_down};
 
 /// Vortex implementation of a DataFusion [`FileFormat`].
@@ -88,7 +91,7 @@ impl FileFormatFactory for VortexFormatFactory {
     #[allow(clippy::disallowed_types)]
     fn create(
         &self,
-        _state: &SessionState,
+        _state: &dyn Session,
         format_options: &std::collections::HashMap<String, String>,
     ) -> DFResult<Arc<dyn FileFormat>> {
         if !format_options.is_empty() {
@@ -159,9 +162,16 @@ impl FileFormat for VortexFormat {
         }
     }
 
+    fn file_source(&self) -> Arc<dyn FileSource> {
+        Arc::new(VortexSource::new(
+            self.footer_cache.clone(),
+            VortexSourceMetrics::default(),
+        ))
+    }
+
     async fn infer_schema(
         &self,
-        state: &SessionState,
+        state: &dyn Session,
         store: &Arc<dyn ObjectStore>,
         objects: &[ObjectMeta],
     ) -> DFResult<SchemaRef> {
@@ -194,7 +204,7 @@ impl FileFormat for VortexFormat {
     #[cfg_attr(feature = "tracing", tracing::instrument(skip_all, fields(location = object.location.as_ref())))]
     async fn infer_stats(
         &self,
-        _state: &SessionState,
+        _state: &dyn Session,
         store: &Arc<dyn ObjectStore>,
         table_schema: SchemaRef,
         object: &ObjectMeta,
@@ -294,7 +304,7 @@ impl FileFormat for VortexFormat {
 
     async fn create_physical_plan(
         &self,
-        _state: &SessionState,
+        _state: &dyn Session,
         file_scan_config: FileScanConfig,
         filters: Option<&Arc<dyn PhysicalExpr>>,
     ) -> DFResult<Arc<dyn ExecutionPlan>> {
@@ -319,21 +329,20 @@ impl FileFormat for VortexFormat {
             return not_impl_err!("Vortex doesn't support output ordering");
         }
 
-        let exec = VortexExec::try_new(
-            file_scan_config,
-            Default::default(),
-            filters.cloned(),
-            self.footer_cache.clone(),
-        )?
-        .into_arc();
+        let mut source =
+            VortexSource::new(self.footer_cache.clone(), VortexSourceMetrics::default());
 
-        Ok(exec)
+        if let Some(predicate) = make_vortex_predicate(filters) {
+            source = source.with_predicate(predicate);
+        }
+
+        Ok(file_scan_config.with_source(Arc::new(source)).build())
     }
 
     async fn create_writer_physical_plan(
         &self,
         input: Arc<dyn ExecutionPlan>,
-        _state: &SessionState,
+        _state: &dyn Session,
         conf: FileSinkConfig,
         order_requirements: Option<LexRequirement>,
     ) -> DFResult<Arc<dyn ExecutionPlan>> {
@@ -369,6 +378,22 @@ impl FileFormat for VortexFormat {
     }
 }
 
+pub(crate) fn make_vortex_predicate(
+    predicate: Option<&Arc<dyn PhysicalExpr>>,
+) -> Option<Arc<dyn VortexExpr>> {
+    predicate
+        // If we cannot convert an expr to a vortex expr, we run no filter, since datafusion
+        // will rerun the filter expression anyway.
+        .and_then(|expr| {
+            // This splits expressions into conjunctions and converts them to vortex expressions.
+            // Any inconvertible expressions are dropped since true /\ a == a.
+            datafusion_physical_expr::split_conjunction(expr)
+                .into_iter()
+                .filter_map(|e| convert_expr_to_vortex(e.clone()).ok())
+                .reduce(and)
+        })
+}
+
 #[cfg(test)]
 mod tests {
     use datafusion::execution::SessionStateBuilder;
diff --git a/vortex-datafusion/src/persistent/metrics.rs b/vortex-datafusion/src/persistent/metrics.rs
index 23ec193d6e8..86cda3d3e94 100644
--- a/vortex-datafusion/src/persistent/metrics.rs
+++ b/vortex-datafusion/src/persistent/metrics.rs
@@ -1,6 +1,7 @@
 //! Vortex table provider metrics.
 use std::sync::Arc;
 
+use datafusion::datasource::source::DataSourceExec;
 use datafusion_physical_plan::metrics::{
     Count, ExecutionPlanMetricsSet, Gauge, Label as DatafusionLabel,
     MetricValue as DatafusionMetricValue, MetricsSet,
@@ -10,8 +11,6 @@ use datafusion_physical_plan::{
 };
 use vortex_metrics::{DefaultTags, Metric, MetricId, Tags, VortexMetrics};
 
-use super::execution::VortexExec;
-
 pub(crate) static PARTITION_LABEL: &str = "partition";
 
 /// Extracts datafusion metrics from all VortexExec instances in
@@ -35,7 +34,7 @@ impl ExecutionPlanVisitor for VortexMetricsFinder {
     fn pre_visit(&mut self, plan: &dyn ExecutionPlan) -> Result<bool, Self::Error> {
         if let Some(metrics) = plan
             .as_any()
-            .downcast_ref::<VortexExec>()
+            .downcast_ref::<DataSourceExec>()
             .and_then(|exec| exec.metrics())
         {
             self.0.push(metrics);
@@ -45,25 +44,25 @@ impl ExecutionPlanVisitor for VortexMetricsFinder {
 }
 
 #[derive(Clone, Debug, Default)]
-pub(crate) struct VortexExecMetrics {
+pub(crate) struct VortexSourceMetrics {
     pub vortex: VortexMetrics,
     pub execution_plan: ExecutionPlanMetricsSet,
 }
 
-impl VortexExecMetrics {
+impl VortexSourceMetrics {
     pub fn child_with_tags(&self, additional_tags: impl Into<DefaultTags>) -> VortexMetrics {
         self.vortex.child_with_tags(additional_tags)
     }
 
-    pub fn metrics_set(&self) -> MetricsSet {
-        let mut base = self.execution_plan.clone_inner();
+    pub fn report_to_datafusion(&self) -> &ExecutionPlanMetricsSet {
+        let base = &self.execution_plan;
         for metric in self
             .vortex
             .snapshot()
             .iter()
             .flat_map(|(id, metric)| metric_to_datafusion(id, metric))
         {
-            base.push(Arc::new(metric));
+            base.register(Arc::new(metric));
         }
         base
     }
diff --git a/vortex-datafusion/src/persistent/mod.rs b/vortex-datafusion/src/persistent/mod.rs
index e093c7f92df..8e99bf214b1 100644
--- a/vortex-datafusion/src/persistent/mod.rs
+++ b/vortex-datafusion/src/persistent/mod.rs
@@ -1,11 +1,11 @@
 //! Persistent implementation of a Vortex table provider.
 mod cache;
 mod config;
-mod execution;
 mod format;
 pub mod metrics;
 mod opener;
 mod sink;
+mod source;
 
 pub use format::{VortexFormat, VortexFormatFactory, VortexFormatOptions};
 
diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs
index 738b894e0fe..eec8956d60d 100644
--- a/vortex-datafusion/src/persistent/opener.rs
+++ b/vortex-datafusion/src/persistent/opener.rs
@@ -7,7 +7,6 @@ use futures::{FutureExt as _, StreamExt};
 use object_store::{ObjectStore, ObjectStoreScheme};
 use tokio::runtime::Handle;
 use vortex_array::ToCanonical;
-use vortex_error::VortexResult;
 use vortex_expr::{ExprRef, VortexExpr};
 use vortex_file::executor::{TaskExecutor, TokioExecutor};
 use vortex_file::{SplitBy, VortexOpenOptions};
@@ -39,8 +38,8 @@ impl VortexFileOpener {
         projected_arrow_schema: SchemaRef,
         batch_size: usize,
         metrics: VortexMetrics,
-    ) -> VortexResult<Self> {
-        Ok(Self {
+    ) -> Self {
+        Self {
             scheme,
             object_store,
             projection,
@@ -49,7 +48,7 @@ impl VortexFileOpener {
             projected_arrow_schema,
             batch_size,
             metrics,
-        })
+        }
     }
 }
 
@@ -88,8 +87,8 @@ impl FileOpener for VortexFileOpener {
 
             Ok(vxf
                 .scan()
-                .with_projection(projection.clone())
-                .with_some_filter(filter.clone())
+                .with_projection(projection)
+                .with_some_filter(filter)
                 .with_prefetch_conjuncts(true)
                 .with_canonicalize(true)
                 // DataFusion likes ~8k row batches. Ideally we would respect the config,
diff --git a/vortex-datafusion/src/persistent/sink.rs b/vortex-datafusion/src/persistent/sink.rs
index 9413a190dc4..61eab3fe802 100644
--- a/vortex-datafusion/src/persistent/sink.rs
+++ b/vortex-datafusion/src/persistent/sink.rs
@@ -4,19 +4,22 @@ use std::sync::atomic::{AtomicU64, Ordering};
 
 use arrow_schema::SchemaRef;
 use async_trait::async_trait;
-use datafusion::datasource::physical_plan::FileSinkConfig;
+use datafusion::common::runtime::SpawnedTask;
+use datafusion::datasource::file_format::write::demux::DemuxedStreamReceiver;
+use datafusion::datasource::physical_plan::{FileSink, FileSinkConfig};
+use datafusion_common::DataFusionError;
 use datafusion_execution::{SendableRecordBatchStream, TaskContext};
 use datafusion_physical_plan::insert::DataSink;
 use datafusion_physical_plan::metrics::MetricsSet;
 use datafusion_physical_plan::{DisplayAs, DisplayFormatType};
-use futures::{StreamExt, TryStreamExt};
-use rand::distr::{Alphanumeric, SampleString};
+use futures::StreamExt;
+use object_store::ObjectStore;
+use tokio_stream::wrappers::ReceiverStream;
 use vortex_array::TryIntoArray;
 use vortex_array::stream::ArrayStreamAdapter;
 use vortex_dtype::DType;
 use vortex_dtype::arrow::FromArrowType;
-use vortex_error::VortexError;
-use vortex_file::{VORTEX_FILE_EXTENSION, VortexWriteOptions};
+use vortex_file::VortexWriteOptions;
 use vortex_io::{ObjectStoreWriter, VortexWrite};
 
 pub struct VortexSink {
@@ -65,47 +68,52 @@ impl DataSink for VortexSink {
         &self,
         data: SendableRecordBatchStream,
         context: &Arc<TaskContext>,
-    ) -> datafusion_common::error::Result<u64> {
-        let object_store = context
-            .runtime_env()
-            .object_store(&self.config.object_store_url)?;
-
-        let base_output_path = &self.config.table_paths[0];
-
-        let single_file_output =
-            !base_output_path.is_collection() && base_output_path.file_extension().is_some();
-
-        let path = if single_file_output {
-            base_output_path.prefix().to_owned()
-        } else {
-            let filename = Alphanumeric.sample_string(&mut rand::rng(), 16);
-            base_output_path
-                .prefix()
-                .child(format!("{filename}.{}", VORTEX_FILE_EXTENSION))
-        };
+    ) -> datafusion_common::Result<u64> {
+        FileSink::write_all(self, data, context).await
+    }
+}
 
-        let vortex_writer = ObjectStoreWriter::new(object_store, path).await?;
+#[async_trait]
+impl FileSink for VortexSink {
+    fn config(&self) -> &FileSinkConfig {
+        &self.config
+    }
 
-        // TODO(adam): This is a temporary hack
+    async fn spawn_writer_tasks_and_join(
+        &self,
+        _context: &Arc<TaskContext>,
+        demux_task: SpawnedTask<datafusion_common::Result<()>>,
+        mut file_stream_rx: DemuxedStreamReceiver,
+        object_store: Arc<dyn ObjectStore>,
+    ) -> datafusion_common::Result<u64> {
+        // This is a hack
         let row_counter = Arc::new(AtomicU64::new(0));
 
-        let dtype = DType::from_arrow(data.schema());
-        let stream = data
-            .map_err(VortexError::from)
-            .map(|rb| rb.and_then(|rb| rb.try_into_array()))
-            .map_ok(|rb| {
-                row_counter.fetch_add(rb.len() as u64, Ordering::SeqCst);
-                rb
+        // TODO(adamg):
+        // 1. We only write only file at a time
+        // 2. We can probably be better at signaling how much memory we're consuming (potentially when reading too), see ParquetSink::spawn_writer_tasks_and_join.
+        while let Some((path, rx)) = file_stream_rx.recv().await {
+            let writer = ObjectStoreWriter::new(object_store.clone(), path).await?;
+
+            let stream = ReceiverStream::new(rx).map(|rb| {
+                row_counter.fetch_add(rb.num_rows() as u64, Ordering::Relaxed);
+                rb.try_into_array()
             });
+            let dtype = DType::from_arrow(self.config.output_schema.as_ref());
+            let stream_adapter = ArrayStreamAdapter::new(dtype, stream);
 
-        let stream = ArrayStreamAdapter::new(dtype, stream);
+            let mut writer = VortexWriteOptions::default()
+                .write(writer, stream_adapter)
+                .await?;
 
-        let mut writer = VortexWriteOptions::default()
-            .write(vortex_writer, stream)
-            .await?;
+            writer.flush().await?;
+            writer.shutdown().await?;
+        }
 
-        writer.flush().await?;
-        writer.shutdown().await?;
+        demux_task
+            .join_unwind()
+            .await
+            .map_err(DataFusionError::ExecutionJoin)??;
 
         Ok(row_counter.load(Ordering::SeqCst))
     }
@@ -115,6 +123,7 @@ impl DataSink for VortexSink {
 mod tests {
     use std::sync::Arc;
 
+    use datafusion::datasource::DefaultTableSource;
     use datafusion::execution::SessionStateBuilder;
     use datafusion::prelude::SessionContext;
     use datafusion_expr::{Expr, LogicalPlan, LogicalPlanBuilder, Values};
@@ -131,7 +140,7 @@ mod tests {
         register_vortex_format_factory(factory, &mut session_state_builder);
         let session = SessionContext::new_with_state(session_state_builder.build());
 
-        let df = session
+        session
             .sql(&format!(
                 "CREATE EXTERNAL TABLE my_tbl \
                     (c1 VARCHAR NOT NULL, c2 INT NOT NULL) \
@@ -142,7 +151,6 @@ mod tests {
             .await
             .unwrap();
 
-        assert_eq!(df.clone().count().await.unwrap(), 0);
         let my_tbl = session.table("my_tbl").await.unwrap();
 
         // It's valuable to have two insert code paths because they actually behave slightly differently
@@ -154,10 +162,12 @@ mod tests {
             ]],
         };
 
+        let tbl_provider = session.table_provider("my_tbl").await.unwrap();
+
         let logical_plan = LogicalPlanBuilder::insert_into(
             LogicalPlan::Values(values.clone()),
             "my_tbl",
-            my_tbl.schema().as_arrow(),
+            Arc::new(DefaultTableSource::new(tbl_provider)),
             datafusion_expr::dml::InsertOp::Append,
         )
         .unwrap()
@@ -173,7 +183,7 @@ mod tests {
             .unwrap();
 
         session
-            .sql("INSERT INTO my_tbl VALUES ('hello', 42::INT);")
+            .sql("INSERT INTO my_tbl VALUES ('world', 24);")
             .await
             .unwrap()
             .collect()
diff --git a/vortex-datafusion/src/persistent/source.rs b/vortex-datafusion/src/persistent/source.rs
new file mode 100644
index 00000000000..6cd93feb5e5
--- /dev/null
+++ b/vortex-datafusion/src/persistent/source.rs
@@ -0,0 +1,280 @@
+use std::any::Any;
+use std::collections::VecDeque;
+use std::sync::Arc;
+
+use arrow_schema::SchemaRef;
+use datafusion::datasource::listing::PartitionedFile;
+use datafusion::datasource::physical_plan::{FileOpener, FileScanConfig, FileSource};
+use datafusion_common::{Result as DFResult, Statistics};
+use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
+use itertools::Itertools as _;
+use object_store::{ObjectStore, ObjectStoreScheme};
+use vortex_error::VortexExpect as _;
+use vortex_expr::{Identity, VortexExpr};
+use vortex_file::VORTEX_FILE_EXTENSION;
+
+use super::cache::FooterCache;
+use super::config::{ConfigProjection, FileScanConfigExt};
+use super::metrics::{PARTITION_LABEL, VortexSourceMetrics};
+use super::opener::VortexFileOpener;
+
+/// A config for [`VortexFileOpener`]. Used to create [`DataSourceExec`] based physical plans.
+///
+/// [`DataSourceExec`]: datafusion_physical_plan::source::DataSourceExec
+#[derive(Clone)]
+pub struct VortexSource {
+    pub(crate) footer_cache: FooterCache,
+    pub(crate) predicate: Option<Arc<dyn VortexExpr>>,
+    pub(crate) projection: Option<Arc<dyn VortexExpr>>,
+    pub(crate) batch_size: Option<usize>,
+    pub(crate) projected_statistics: Option<Statistics>,
+    pub(crate) arrow_schema: Option<SchemaRef>,
+    pub(crate) metrics: VortexSourceMetrics,
+}
+
+impl VortexSource {
+    pub(crate) fn new(footer_cache: FooterCache, metrics: VortexSourceMetrics) -> Self {
+        Self {
+            footer_cache,
+            metrics,
+            projection: None,
+            batch_size: None,
+            projected_statistics: None,
+            arrow_schema: None,
+            predicate: None,
+        }
+    }
+
+    /// Sets a [`VortexExpr`] as a predicate
+    pub fn with_predicate(&self, predicate: Arc<dyn VortexExpr>) -> Self {
+        let mut source = self.clone();
+        source.predicate = Some(predicate);
+        source
+    }
+}
+
+impl FileSource for VortexSource {
+    fn create_file_opener(
+        &self,
+        object_store: Arc<dyn ObjectStore>,
+        base_config: &FileScanConfig,
+        partition: usize,
+    ) -> Arc<dyn FileOpener> {
+        let (scheme, _) = ObjectStoreScheme::parse(base_config.object_store_url.as_ref())
+            .ok()
+            .vortex_expect("Couldn't parse object store URL");
+
+        let partition_metrics = self
+            .metrics
+            .child_with_tags([(PARTITION_LABEL, partition.to_string())].into_iter());
+
+        let batch_size = self
+            .batch_size
+            .vortex_expect("batch_size must be supplied to VortexSource");
+
+        let opener = VortexFileOpener::new(
+            scheme,
+            object_store,
+            self.projection.clone().unwrap_or_else(Identity::new_expr),
+            self.predicate.clone(),
+            self.footer_cache.clone(),
+            self.arrow_schema
+                .clone()
+                .vortex_expect("We should have a schema here"),
+            batch_size,
+            partition_metrics,
+        );
+
+        Arc::new(opener)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn with_batch_size(&self, batch_size: usize) -> Arc<dyn FileSource> {
+        let mut source = self.clone();
+        source.batch_size = Some(batch_size);
+        Arc::new(source)
+    }
+
+    fn with_schema(&self, schema: SchemaRef) -> Arc<dyn FileSource> {
+        // todo(adam): does this need to the same as `with_projection`?
+        let mut source = self.clone();
+        source.arrow_schema = Some(schema);
+        Arc::new(source)
+    }
+
+    fn with_projection(&self, config: &FileScanConfig) -> Arc<dyn FileSource> {
+        let ConfigProjection {
+            arrow_schema,
+            constraints: _constraints,
+            statistics,
+            orderings: _,
+            projection_expr,
+        } = config.project_for_vortex();
+
+        let statistics = if self.predicate.is_some() {
+            statistics.to_inexact()
+        } else {
+            statistics
+        };
+
+        let mut source = self.clone();
+        source.projection = Some(projection_expr);
+        source.arrow_schema = Some(arrow_schema);
+        source.projected_statistics = Some(statistics);
+
+        Arc::new(source)
+    }
+
+    fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> {
+        let mut source = self.clone();
+        source.projected_statistics = Some(statistics);
+        Arc::new(source)
+    }
+
+    fn metrics(&self) -> &ExecutionPlanMetricsSet {
+        self.metrics.report_to_datafusion()
+    }
+
+    fn statistics(&self) -> DFResult<Statistics> {
+        let statistics = self
+            .projected_statistics
+            .clone()
+            .vortex_expect("projected_statistics must be set");
+
+        if self.predicate.is_some() {
+            Ok(statistics.to_inexact())
+        } else {
+            Ok(statistics)
+        }
+    }
+
+    fn file_type(&self) -> &str {
+        VORTEX_FILE_EXTENSION
+    }
+
+    fn repartitioned(
+        &self,
+        target_partitions: usize,
+        _repartition_file_min_size: usize,
+        _output_ordering: Option<datafusion_physical_expr::LexOrdering>,
+        config: &FileScanConfig,
+    ) -> DFResult<Option<FileScanConfig>> {
+        let mut new_config = config.clone();
+        let file_groups = std::mem::take(&mut new_config.file_groups);
+        new_config.file_groups = repartition_by_size(file_groups, target_partitions);
+
+        Ok(Some(new_config))
+    }
+}
+
+pub(crate) fn repartition_by_size(
+    file_groups: Vec<Vec<PartitionedFile>>,
+    desired_partitions: usize,
+) -> Vec<Vec<PartitionedFile>> {
+    let all_files = file_groups.iter().flatten().collect::<Vec<_>>();
+    let total_file_count = all_files.len();
+    let total_size = all_files.iter().map(|f| f.object_meta.size).sum::<usize>();
+    let target_partition_size = total_size / desired_partitions;
+
+    let mut partitions = Vec::with_capacity(desired_partitions);
+
+    let mut curr_partition_size = 0;
+    let mut curr_partition = Vec::default();
+
+    let mut all_files = VecDeque::from_iter(
+        all_files
+            .into_iter()
+            .sorted_unstable_by_key(|f| f.object_meta.size),
+    );
+
+    while !all_files.is_empty() && partitions.len() < desired_partitions {
+        // If the current partition is empty, we want to bootstrap it with the biggest file we have leftover.
+        let file = if curr_partition.is_empty() {
+            all_files.pop_back()
+        // If we already have files in the partition, we try and fill it up.
+        } else {
+            // Peak at the biggest file left
+            let biggest_file_size = all_files
+                .back()
+                .vortex_expect("We must have at least one item")
+                .object_meta
+                .size;
+
+            let smallest_file_size = all_files
+                .front()
+                .vortex_expect("We must have at least one item")
+                .object_meta
+                .size;
+
+            // We try and find a file on either end that fits in the partition
+            if curr_partition_size + biggest_file_size >= target_partition_size {
+                all_files.pop_front()
+            } else if curr_partition_size + smallest_file_size >= target_partition_size {
+                all_files.pop_back()
+            } else {
+                None
+            }
+        };
+
+        // Add a file to the partition
+        if let Some(file) = file {
+            curr_partition_size += file.object_meta.size;
+            curr_partition.push(file.clone());
+        }
+
+        // If the partition is full, move on to the next one
+        if curr_partition_size >= target_partition_size || file.is_none() {
+            curr_partition_size = 0;
+            partitions.push(std::mem::take(&mut curr_partition));
+        }
+    }
+
+    // If we we're still missing the last partition
+    if !curr_partition.is_empty() && partitions.len() != desired_partitions {
+        partitions.push(std::mem::take(&mut curr_partition));
+    } else if !curr_partition.is_empty() {
+        for (idx, file) in curr_partition.into_iter().enumerate() {
+            let new_part_idx = idx % partitions.len();
+            partitions[new_part_idx].push(file.clone());
+        }
+    }
+
+    for (idx, file) in all_files.into_iter().enumerate() {
+        let new_part_idx = idx % partitions.len();
+        partitions[new_part_idx].push(file.clone());
+    }
+
+    // Assert that we have the correct number of partitions and that the total number of files is right
+    assert_eq!(total_file_count, partitions.iter().flatten().count());
+
+    partitions
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn basic_repartition_test() {
+        let file_groups = vec![vec![
+            PartitionedFile::new("a", 100),
+            PartitionedFile::new("b", 25),
+            PartitionedFile::new("c", 25),
+            PartitionedFile::new("d", 25),
+            PartitionedFile::new("e", 50),
+        ]];
+
+        repartition_by_size(file_groups, 2);
+
+        let file_groups = vec![
+            (0..100)
+                .map(|idx| PartitionedFile::new(format!("{idx}"), idx))
+                .collect(),
+        ];
+
+        repartition_by_size(file_groups, 16);
+    }
+}


Reviews:
user: login: AdamGS
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-02-19T12:05:03Z
user: login: onursatici
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-03T15:38:05Z
user: login: AdamGS
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-03T18:30:01Z

Review comments:
diff_hunk: @@ -1,12 +1,13 @@
 //! Persistent implementation of a Vortex table provider.
 mod cache;
 mod config;
-mod execution;
 mod format;
 mod opener;
 mod sink;
+mod source;
 
 pub use format::{VortexFormat, VortexFormatFactory, VortexFormatOptions};
+pub use source::VortexSource;
path: vortex-datafusion/src/persistent/mod.rs
user: login: AdamGS
body: I don't think it actually needs to be public
created_at: 2025-02-19T12:05:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -26,13 +26,16 @@ use vortex_array::stats::{Stat, StatsProviderExt, StatsSet};
 use vortex_array::{ArrayRegistry, stats};
 use vortex_dtype::DType;
 use vortex_error::{VortexExpect, VortexResult, vortex_err};
+use vortex_expr::datafusion::convert_expr_to_vortex;
+use vortex_expr::{VortexExpr, and};
 use vortex_file::{VORTEX_FILE_EXTENSION, VortexOpenOptions};
 use vortex_io::ObjectStoreReadAt;
 use vortex_layout::{LayoutRegistry, LayoutRegistryExt};
 
 use super::cache::FooterCache;
-use super::execution::VortexExec;
+use super::metrics::VortexExecMetrics;
path: vortex-datafusion/src/persistent/format.rs
user: login: onursatici
body: nit: rename to VortexSourceMetrics
created_at: 2025-03-03T15:35:57Z
author_association: CONTRIBUTOR
diff_hunk: @@ -159,9 +162,16 @@ impl FileFormat for VortexFormat {
         }
     }
 
+    fn file_source(&self) -> Arc<dyn FileSource> {
+        Arc::new(VortexSource::new(
+            self.footer_cache.clone(),
+            VortexExecMetrics::default(),
path: vortex-datafusion/src/persistent/format.rs
user: login: onursatici
body: we are creating a new metrics container per call but as far as I see this is only used by datafusion for downcasting, so it should be fine
created_at: 2025-03-03T15:37:59Z
author_association: CONTRIBUTOR
diff_hunk: @@ -26,13 +26,16 @@ use vortex_array::stats::{Stat, StatsProviderExt, StatsSet};
 use vortex_array::{ArrayRegistry, stats};
 use vortex_dtype::DType;
 use vortex_error::{VortexExpect, VortexResult, vortex_err};
+use vortex_expr::datafusion::convert_expr_to_vortex;
+use vortex_expr::{VortexExpr, and};
 use vortex_file::{VORTEX_FILE_EXTENSION, VortexOpenOptions};
 use vortex_io::ObjectStoreReadAt;
 use vortex_layout::{LayoutRegistry, LayoutRegistryExt};
 
 use super::cache::FooterCache;
-use super::execution::VortexExec;
+use super::metrics::VortexExecMetrics;
path: vortex-datafusion/src/persistent/format.rs
user: login: AdamGS
body: done
created_at: 2025-03-03T18:30:01Z
author_association: CONTRIBUTOR

PR comments:
user: login: github-actions[bot]
created_at: 2025-02-19T14:39:43Z
body: # Benchmarks: TPC-H on NVME
<details>
<summary>Table of Results</summary>

| name                            |   PR b97b9f2c |   base 9f331e15 |   ratio (PR/base) | unit   |
|:--------------------------------|--------------:|----------------:|------------------:|:-------|
| tpch_q01/arrow                  |      46814764 |     4.08447e+07 |          1.14617  | ns     |
| tpch_q02/arrow                  |      48973737 |     4.6717e+07  |          1.04831  | ns     |
| tpch_q03/arrow                  |      30766041 |     2.91328e+07 |          1.05606  | ns     |
| tpch_q04/arrow                  |      23304113 |     2.0694e+07  |          1.12613  | ns     |
| tpch_q05/arrow                  |      48195454 |     4.60596e+07 |          1.04637  | ns     |
| tpch_q06/arrow                  |      10343090 |     8.52471e+06 |          1.21331  | ns     |
| tpch_q07/arrow                  |      79271919 |     6.97104e+07 |          1.13716  | ns     |
| tpch_q08/arrow                  |      58796600 |     5.39776e+07 |          1.08928  | ns     |
| tpch_q09/arrow                  |      73187245 |     6.64376e+07 |          1.10159  | ns     |
| tpch_q10/arrow                  |      48567260 |     4.20389e+07 |          1.15529  | ns     |
| tpch_q11/arrow                  |      25206092 |     2.22554e+07 |          1.13258  | ns     |
| tpch_q12/arrow                  |      29349650 |     2.589e+07   |          1.13363  | ns     |
| tpch_q13/arrow                  |      17323939 |     1.54277e+07 |          1.12291  | ns     |
| tpch_q14/arrow                  |      15513910 |     1.23542e+07 |          1.25576  | ns     |
| tpch_q15/arrow                  |      31051360 |     2.264e+07   |          1.37153  | ns     |
| tpch_q16/arrow                  |      23337823 |     1.97894e+07 |          1.17931  | ns     |
| tpch_q17/arrow                  |      61250816 |     5.85785e+07 |          1.04562  | ns     |
| tpch_q18/arrow                  |     103425701 |     9.32697e+07 |          1.10889  | ns     |
| tpch_q19/arrow                  |      30350457 |     2.53769e+07 |          1.19599  | ns     |
| tpch_q20/arrow                  |      37540004 |     3.01326e+07 |          1.24583  | ns     |
| tpch_q21/arrow                  |     117939389 |     1.08658e+08 |          1.08542  | ns     |
| tpch_q22/arrow                  |      15957822 |     1.35142e+07 |          1.18082  | ns     |
| tpch_q01/parquet                |     116625506 |     1.12174e+08 |          1.03969  | ns     |
| tpch_q02/parquet                |     118449296 |     1.11287e+08 |          1.06436  | ns     |
| tpch_q03/parquet                |     107499112 |     9.90065e+07 |          1.08578  | ns     |
| tpch_q04/parquet                |      63227101 |     5.47861e+07 |          1.15407  | ns     |
| tpch_q05/parquet                |     125470068 |     1.138e+08   |          1.10255  | ns     |
| tpch_q06/parquet                |      26903619 |     2.43043e+07 |          1.10695  | ns     |
| tpch_q07/parquet                |     134657840 |     1.23816e+08 |          1.08756  | ns     |
| tpch_q08/parquet                |     165129586 |     1.50245e+08 |          1.09907  | ns     |
| tpch_q09/parquet                |     215404776 |     2.01786e+08 |          1.06749  | ns     |
| tpch_q10/parquet                |     132651231 |     1.24476e+08 |          1.06568  | ns     |
| tpch_q11/parquet                |      53823206 |     5.15499e+07 |          1.0441   | ns     |
| tpch_q12/parquet                |      97547382 |     8.82884e+07 |          1.10487  | ns     |
| tpch_q13/parquet                |     151779872 |     1.43811e+08 |          1.05541  | ns     |
| tpch_q14/parquet                |      49731367 |     4.32881e+07 |          1.14885  | ns     |
| tpch_q15/parquet                |      66476861 |     6.10372e+07 |          1.08912  | ns     |
| tpch_q16/parquet                |      55233680 |     4.73471e+07 |          1.16657  | ns     |
| tpch_q17/parquet                |     135906590 |     1.25096e+08 |          1.08642  | ns     |
| tpch_q18/parquet                |     198121554 |     1.8296e+08  |          1.08287  | ns     |
| tpch_q19/parquet                |      76526929 |     7.19709e+07 |          1.0633   | ns     |
| tpch_q20/parquet                |     101077350 |     9.16694e+07 |          1.10263  | ns     |
| tpch_q21/parquet                |     191549858 |     1.74484e+08 |          1.09781  | ns     |
| tpch_q22/parquet                |      52292197 |     4.72311e+07 |          1.10716  | ns     |
| tpch_q01/vortex-file-compressed |      37410820 |     3.2454e+07  |          1.15273  | ns     |
| tpch_q02/vortex-file-compressed |      65463877 |     5.2561e+07  |          1.24548  | ns     |
| tpch_q03/vortex-file-compressed |      35470417 |     3.10933e+07 |          1.14077  | ns     |
| tpch_q04/vortex-file-compressed |      22536876 |     1.89111e+07 |          1.19173  | ns     |
| tpch_q05/vortex-file-compressed |      51843094 |     4.38517e+07 |          1.18224  | ns     |
| tpch_q06/vortex-file-compressed |       9912669 |     8.84167e+06 |          1.12113  | ns     |
| tpch_q07/vortex-file-compressed |      75159849 |     6.40491e+07 |          1.17347  | ns     |
| tpch_q08/vortex-file-compressed |      60266602 |     5.23924e+07 |          1.15029  | ns     |
| tpch_q09/vortex-file-compressed |      71360755 |     6.48479e+07 |          1.10043  | ns     |
| tpch_q10/vortex-file-compressed |      58357352 |     5.12777e+07 |          1.13807  | ns     |
| tpch_q11/vortex-file-compressed |      29945096 |     2.30017e+07 |          1.30187  | ns     |
| tpch_q12/vortex-file-compressed |      23459399 |     2.38949e+07 |          0.981772 | ns     |
| tpch_q13/vortex-file-compressed |      28364620 |     2.51642e+07 |          1.12718  | ns     |
| tpch_q14/vortex-file-compressed |      15973074 |     1.5311e+07  |          1.04324  | ns     |
| tpch_q15/vortex-file-compressed |      34100890 |     2.76424e+07 |          1.23364  | ns     |
| tpch_q16/vortex-file-compressed |      32229223 |     2.58082e+07 |          1.2488   | ns     |
| tpch_q17/vortex-file-compressed |      58383602 |     4.90882e+07 |          1.18936  | ns     |
| tpch_q18/vortex-file-compressed |      90671415 |     8.15663e+07 |          1.11163  | ns     |
| tpch_q19/vortex-file-compressed |      32544099 |     3.06786e+07 |          1.06081  | ns     |
| tpch_q20/vortex-file-compressed |      42858326 |     3.48887e+07 |          1.22843  | ns     |
| tpch_q21/vortex-file-compressed |      98374579 |     8.19945e+07 |          1.19977  | ns     |
| tpch_q22/vortex-file-compressed |      30265138 |     2.6364e+07  |          1.14797  | ns     |
</details>

<!-- thollander/actions-comment-pull-request "bench-pr-comment-tpch-nvme" -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-02-19T14:40:46Z
body: # Benchmarks: TPC-H on S3
<details>
<summary>Table of Results</summary>

| name                            |   PR b97b9f2c |   base 9f331e15 |   ratio (PR/base) | unit   |
|:--------------------------------|--------------:|----------------:|------------------:|:-------|
| tpch_q01/parquet                |     260708968 |     2.66292e+08 |          0.979035 | ns     |
| tpch_q02/parquet                |     678727230 |     6.69487e+08 |          1.0138   | ns     |
| tpch_q03/parquet                |     411388517 |     4.29232e+08 |          0.95843  | ns     |
| tpch_q04/parquet                |     216431000 |     2.27385e+08 |          0.951824 | ns     |
| tpch_q05/parquet                |     552127397 |     5.80036e+08 |          0.951884 | ns     |
| tpch_q06/parquet                |     182282351 |     1.81014e+08 |          1.007    | ns     |
| tpch_q07/parquet                |     611844490 |     6.28693e+08 |          0.973201 | ns     |
| tpch_q08/parquet                |     774012355 |     8.14386e+08 |          0.950424 | ns     |
| tpch_q09/parquet                |     705467847 |     6.61229e+08 |          1.0669   | ns     |
| tpch_q10/parquet                |     554948955 |     5.23239e+08 |          1.0606   | ns     |
| tpch_q11/parquet                |     275690421 |     2.87372e+08 |          0.959351 | ns     |
| tpch_q12/parquet                |     278217071 |     2.79256e+08 |          0.996278 | ns     |
| tpch_q13/parquet                |     381604593 |     3.89036e+08 |          0.980897 | ns     |
| tpch_q14/parquet                |     245971146 |     2.52142e+08 |          0.975525 | ns     |
| tpch_q15/parquet                |     457070181 |     4.63399e+08 |          0.986342 | ns     |
| tpch_q16/parquet                |     257782409 |     2.63092e+08 |          0.97982  | ns     |
| tpch_q17/parquet                |     387588015 |     3.92452e+08 |          0.987606 | ns     |
| tpch_q18/parquet                |     519680170 |     5.31858e+08 |          0.977102 | ns     |
| tpch_q19/parquet                |     277595773 |     2.96374e+08 |          0.936639 | ns     |
| tpch_q20/parquet                |     520019788 |     5.12223e+08 |          1.01522  | ns     |
| tpch_q21/parquet                |     599233430 |     5.98214e+08 |          1.0017   | ns     |
| tpch_q22/parquet                |     257725737 |     2.70901e+08 |          0.951365 | ns     |
| tpch_q01/vortex-file-compressed |     142731476 |     1.43859e+08 |          0.992159 | ns     |
| tpch_q02/vortex-file-compressed |     422206783 |     4.18795e+08 |          1.00815  | ns     |
| tpch_q03/vortex-file-compressed |     281118634 |     2.76606e+08 |          1.01631  | ns     |
| tpch_q04/vortex-file-compressed |     197566856 |     1.9735e+08  |          1.0011   | ns     |
| tpch_q05/vortex-file-compressed |     351263799 |     3.36328e+08 |          1.04441  | ns     |
| tpch_q06/vortex-file-compressed |     116929262 |     1.19863e+08 |          0.975526 | ns     |
| tpch_q07/vortex-file-compressed |     383040718 |     3.88968e+08 |          0.984762 | ns     |
| tpch_q08/vortex-file-compressed |     446356878 |     4.40612e+08 |          1.01304  | ns     |
| tpch_q09/vortex-file-compressed |     391595164 |     3.89547e+08 |          1.00526  | ns     |
| tpch_q10/vortex-file-compressed |     384176687 |     3.98045e+08 |          0.965158 | ns     |
| tpch_q11/vortex-file-compressed |     160052228 |     1.56294e+08 |          1.02404  | ns     |
| tpch_q12/vortex-file-compressed |     205903109 |     2.12569e+08 |          0.968642 | ns     |
| tpch_q13/vortex-file-compressed |     178549495 |     1.73468e+08 |          1.02929  | ns     |
| tpch_q14/vortex-file-compressed |     136056322 |     1.37913e+08 |          0.986535 | ns     |
| tpch_q15/vortex-file-compressed |     302303030 |     3.05556e+08 |          0.989355 | ns     |
| tpch_q16/vortex-file-compressed |     188250700 |     1.84416e+08 |          1.0208   | ns     |
| tpch_q17/vortex-file-compressed |     201852460 |     2.12865e+08 |          0.948266 | ns     |
| tpch_q18/vortex-file-compressed |     281819838 |     2.59974e+08 |          1.08403  | ns     |
| tpch_q19/vortex-file-compressed |     199187823 |     1.95573e+08 |          1.01848  | ns     |
| tpch_q20/vortex-file-compressed |     335804673 |     3.32338e+08 |          1.01043  | ns     |
| tpch_q21/vortex-file-compressed |     484938363 |     4.74994e+08 |          1.02094  | ns     |
| tpch_q22/vortex-file-compressed |     149268656 |     1.6136e+08  |          0.925066 | ns     |
</details>

<!-- thollander/actions-comment-pull-request "bench-pr-comment-tpch-s3" -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-02-19T15:12:01Z
body: # Benchmarks: Clickbench on NVME
<details>
<summary>Table of Results</summary>

| name                                  |   PR b97b9f2c |   base 9f331e15 |   ratio (PR/base) | unit   |
|:--------------------------------------|--------------:|----------------:|------------------:|:-------|
| clickbench_q00/parquet                |       2656236 |     2.65939e+06 |          0.998814 | ns     |
| clickbench_q01/parquet                |      35414010 |     3.41856e+07 |          1.03593  | ns     |
| clickbench_q02/parquet                |      70541109 |     7.17033e+07 |          0.983792 | ns     |
| clickbench_q03/parquet                |      59588335 |     5.50326e+07 |          1.08278  | ns     |
| clickbench_q04/parquet                |     339702596 |     3.55589e+08 |          0.955323 | ns     |
| clickbench_q05/parquet                |     345251703 |     3.23446e+08 |          1.06742  | ns     |
| clickbench_q06/parquet                |       2548505 |     1.94148e+06 |          1.31266  | ns     |
| clickbench_q07/parquet                |      38506566 |     3.21745e+07 |          1.1968   | ns     |
| clickbench_q08/parquet                |     422737899 |     3.93006e+08 |          1.07565  | ns     |
| clickbench_q09/parquet                |     622516931 |     5.89922e+08 |          1.05525  | ns     |
| clickbench_q10/parquet                |     129919377 |     1.24223e+08 |          1.04586  | ns     |
| clickbench_q11/parquet                |     153909179 |     1.49992e+08 |          1.02611  | ns     |
| clickbench_q12/parquet                |     335097882 |     3.33239e+08 |          1.00558  | ns     |
| clickbench_q13/parquet                |     518189026 |     5.07543e+08 |          1.02098  | ns     |
| clickbench_q14/parquet                |     348342579 |     3.31522e+08 |          1.05074  | ns     |
| clickbench_q15/parquet                |     367879479 |     3.72468e+08 |          0.987681 | ns     |
| clickbench_q16/parquet                |     816549219 |     7.81671e+08 |          1.04462  | ns     |
| clickbench_q17/parquet                |     711023750 |     6.93433e+08 |          1.02537  | ns     |
| clickbench_q18/parquet                |    1615806081 |     1.63466e+09 |          0.988467 | ns     |
| clickbench_q19/parquet                |      45276197 |     4.65668e+07 |          0.972284 | ns     |
| clickbench_q20/parquet                |     605644668 |     5.81967e+08 |          1.04069  | ns     |
| clickbench_q21/parquet                |     648416835 |     6.48385e+08 |          1.00005  | ns     |
| clickbench_q22/parquet                |    1049632621 |     1.01556e+09 |          1.03355  | ns     |
| clickbench_q23/parquet                |    4233286161 |     4.05897e+09 |          1.04294  | ns     |
| clickbench_q24/parquet                |     215288991 |     2.0253e+08  |          1.063    | ns     |
| clickbench_q25/parquet                |     189478037 |     1.82817e+08 |          1.03643  | ns     |
| clickbench_q26/parquet                |     242186191 |     2.33935e+08 |          1.03527  | ns     |
| clickbench_q27/parquet                |     827786956 |     7.72837e+08 |          1.0711   | ns     |
| clickbench_q28/parquet                |    4880736540 |     4.7768e+09  |          1.02176  | ns     |
| clickbench_q29/parquet                |     270618408 |     2.58673e+08 |          1.04618  | ns     |
| clickbench_q30/parquet                |     349663776 |     3.42947e+08 |          1.01958  | ns     |
| clickbench_q31/parquet                |     394186450 |     3.80248e+08 |          1.03666  | ns     |
| clickbench_q32/parquet                |    1865440161 |     1.78958e+09 |          1.04239  | ns     |
| clickbench_q33/parquet                |    1594913210 |     1.54834e+09 |          1.03008  | ns     |
| clickbench_q34/parquet                |    1570131226 |     1.51439e+09 |          1.03681  | ns     |
| clickbench_q35/parquet                |     540899451 |     5.25953e+08 |          1.02842  | ns     |
| clickbench_q36/parquet                |     157022581 |     1.53831e+08 |          1.02075  | ns     |
| clickbench_q37/parquet                |      74398807 |     7.12327e+07 |          1.04445  | ns     |
| clickbench_q38/parquet                |     102125442 |     9.91129e+07 |          1.03039  | ns     |
| clickbench_q39/parquet                |     301221396 |     2.81491e+08 |          1.07009  | ns     |
| clickbench_q40/parquet                |      48636384 |     4.73886e+07 |          1.02633  | ns     |
| clickbench_q41/parquet                |      45674189 |     4.49148e+07 |          1.01691  | ns     |
| clickbench_q42/parquet                |      56744418 |     5.56779e+07 |          1.01915  | ns     |
| clickbench_q00/vortex-file-compressed |       4616433 |     4.22226e+06 |          1.09336  | ns     |
| clickbench_q01/vortex-file-compressed |      19997101 |     1.64884e+07 |          1.2128   | ns     |
| clickbench_q02/vortex-file-compressed |      36129668 |     3.05251e+07 |          1.18361  | ns     |
| clickbench_q03/vortex-file-compressed |      42781839 |     4.06043e+07 |          1.05363  | ns     |
| clickbench_q04/vortex-file-compressed |     330976575 |     3.5486e+08  |          0.932697 | ns     |
| clickbench_q05/vortex-file-compressed |     347887609 |     3.37546e+08 |          1.03064  | ns     |
| clickbench_q06/vortex-file-compressed |       4698680 |     4.18671e+06 |          1.12228  | ns     |
| clickbench_q07/vortex-file-compressed |      23153006 |     1.81192e+07 |          1.27782  | ns     |
| clickbench_q08/vortex-file-compressed |     395170738 |     4.24105e+08 |          0.931775 | ns     |
| clickbench_q09/vortex-file-compressed |     520232874 |     5.02995e+08 |          1.03427  | ns     |
| clickbench_q10/vortex-file-compressed |      68326020 |     6.60657e+07 |          1.03421  | ns     |
| clickbench_q11/vortex-file-compressed |      78531033 |     7.41136e+07 |          1.0596   | ns     |
| clickbench_q12/vortex-file-compressed |     267400693 |     2.60301e+08 |          1.02727  | ns     |
| clickbench_q13/vortex-file-compressed |     384635762 |     3.66537e+08 |          1.04938  | ns     |
| clickbench_q14/vortex-file-compressed |     254321058 |     2.60563e+08 |          0.976044 | ns     |
| clickbench_q15/vortex-file-compressed |     402151383 |     4.17499e+08 |          0.963239 | ns     |
| clickbench_q16/vortex-file-compressed |     819637866 |     8.07829e+08 |          1.01462  | ns     |
| clickbench_q17/vortex-file-compressed |     798899931 |     8.00281e+08 |          0.998274 | ns     |
| clickbench_q18/vortex-file-compressed |    1378590932 |     1.33808e+09 |          1.03028  | ns     |
| clickbench_q19/vortex-file-compressed |      30698855 |     2.55615e+07 |          1.20098  | ns     |
| clickbench_q20/vortex-file-compressed |     476962746 |     4.32449e+08 |          1.10293  | ns     |
| clickbench_q21/vortex-file-compressed |     491150442 |     4.43262e+08 |          1.10804  | ns     |
| clickbench_q22/vortex-file-compressed |     630146769 |     5.98793e+08 |          1.05236  | ns     |
| clickbench_q23/vortex-file-compressed |    1317039298 |     1.28007e+09 |          1.02888  | ns     |
| clickbench_q24/vortex-file-compressed |      91281806 |     8.64505e+07 |          1.05588  | ns     |
| clickbench_q25/vortex-file-compressed |     105020790 |     1.02517e+08 |          1.02442  | ns     |
| clickbench_q26/vortex-file-compressed |     126554954 |     1.1816e+08  |          1.07105  | ns     |
| clickbench_q27/vortex-file-compressed |     757366534 |     6.99274e+08 |          1.08308  | ns     |
| clickbench_q28/vortex-file-compressed |    5599883720 |     5.31106e+09 |          1.05438  | ns     |
| clickbench_q29/vortex-file-compressed |     275041030 |     2.4964e+08  |          1.10175  | ns     |
| clickbench_q30/vortex-file-compressed |     240931313 |     2.32739e+08 |          1.0352   | ns     |
| clickbench_q31/vortex-file-compressed |     251416343 |     2.41182e+08 |          1.04244  | ns     |
| clickbench_q32/vortex-file-compressed |    1394931950 |     1.3242e+09  |          1.05342  | ns     |
| clickbench_q33/vortex-file-compressed |    1453567430 |     1.4039e+09  |          1.03538  | ns     |
| clickbench_q34/vortex-file-compressed |    1418277400 |     1.34507e+09 |          1.05443  | ns     |
| clickbench_q35/vortex-file-compressed |     618576381 |     6.13977e+08 |          1.00749  | ns     |
| clickbench_q36/vortex-file-compressed |      64478992 |     5.94661e+07 |          1.0843   | ns     |
| clickbench_q37/vortex-file-compressed |      40190051 |     3.60688e+07 |          1.11426  | ns     |
| clickbench_q38/vortex-file-compressed |      28494623 |     2.76737e+07 |          1.02966  | ns     |
| clickbench_q39/vortex-file-compressed |     120628746 |     1.11403e+08 |          1.08281  | ns     |
| clickbench_q40/vortex-file-compressed |      26192403 |     2.09268e+07 |          1.25162  | ns     |
| clickbench_q41/vortex-file-compressed |      25746705 |     2.03894e+07 |          1.26275  | ns     |
| clickbench_q42/vortex-file-compressed |      31746333 |     3.15281e+07 |          1.00692  | ns     |
</details>

<!-- thollander/actions-comment-pull-request "bench-pr-comment-clickbench-nvme" -->
author_association: CONTRIBUTOR
user: login: codspeed-hq[bot]
created_at: 2025-02-20T16:57:00Z
body: <!-- __CODSPEED_PERFORMANCE_REPORT_COMMENT__ -->
<!-- __CODSPEED_INSTRUMENTATION_PERFORMANCE_REPORT_COMMENT__ -->

## [CodSpeed Performance Report](https://codspeed.io/spiraldb/vortex/branches/adamg%2Fdatafusion-46-branch)

### Merging #2417 will **not alter performance**

<sub>Comparing <code>adamg/datafusion-46-branch</code> (aab23a3) with <code>develop</code> (9f331e1)</sub>



### Summary

`✅ 775` untouched benchmarks  




author_association: NONE
user: login: cloudflare-workers-and-pages[bot]
created_at: 2025-02-25T20:47:56Z
body: ## Deploying vortex-bench with &nbsp;<a href="https://pages.dev"><img alt="Cloudflare Pages" src="https://user-images.githubusercontent.com/23264/106598434-9e719e00-654f-11eb-9e59-6167043cfa01.png" width="16"></a> &nbsp;Cloudflare Pages

<table><tr><td><strong>Latest commit:</strong> </td><td>
<code>3e2c75a</code>
</td></tr>
<tr><td><strong>Status:</strong></td><td>&nbsp;✅&nbsp; Deploy successful!</td></tr>
<tr><td><strong>Preview URL:</strong></td><td>
<a href='https://a949d820.vortex-bench.pages.dev'>https://a949d820.vortex-bench.pages.dev</a>
</td></tr>
<tr><td><strong>Branch Preview URL:</strong></td><td>
<a href='https://adamg-datafusion-46-branch.vortex-bench.pages.dev'>https://adamg-datafusion-46-branch.vortex-bench.pages.dev</a>
</td></tr>
</table>

[View logs](https://dash.cloudflare.com/01e9655179bbec953276890b183039bc/pages/view/vortex-bench/a949d820-9f76-4f1d-bd4c-1c6bde6d2580)

author_association: NONE
user: login: robert3005
created_at: 2025-03-07T23:29:13Z
body: fwiw datafusion 46 is out
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-09T16:38:43Z
body: # Benchmarks: random_access
<details>
<summary>Table of Results</summary>

| name                                   |   PR 6ecb15ea |   base 9f331e15 |   ratio (PR/base) | unit   |
|:---------------------------------------|--------------:|----------------:|------------------:|:-------|
| random-access/vortex-tokio-local-disk  |       4536930 |     3.6962e+06  |           1.22746 | ns     |
| random-access/parquet-tokio-local-disk |     237671576 |     2.35674e+08 |           1.00848 | ns     |
</details>

<!-- thollander/actions-comment-pull-request "bench-pr-comment-random_access" -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-09T17:04:06Z
body: # Benchmarks: compress
<details>
<summary>Table of Results</summary>

| name                                                                                |      PR 6ecb15ea |    base 9f331e15 |   ratio (PR/base) | unit     |
|:------------------------------------------------------------------------------------|-----------------:|-----------------:|------------------:|:---------|
| compress time/taxi throughput                                                       |      0.293745    |      0.318358    |          0.922688 | bytes/ns |
| parquet_rs-zstd compress time/taxi throughput                                       |      0.336101    |      0.366336    |          0.917468 | bytes/ns |
| decompress time/taxi throughput                                                     |      2.05347     |      2.13232     |          0.963022 | bytes/ns |
| parquet_rs-zstd decompress time/taxi throughput                                     |      1.83627     |      1.98433     |          0.925385 | bytes/ns |
| compress time/AirlineSentiment throughput                                           |      0.00323045  |      0.00381827  |          0.84605  | bytes/ns |
| parquet_rs-zstd compress time/AirlineSentiment throughput                           |      0.0576746   |      0.0616052   |          0.936198 | bytes/ns |
| decompress time/AirlineSentiment throughput                                         |      0.0128051   |      0.0139299   |          0.919253 | bytes/ns |
| parquet_rs-zstd decompress time/AirlineSentiment throughput                         |      0.0922078   |      0.0944367   |          0.976398 | bytes/ns |
| compress time/Arade throughput                                                      |      0.211985    |      0.239195    |          0.886243 | bytes/ns |
| parquet_rs-zstd compress time/Arade throughput                                      |      0.479843    |      0.524092    |          0.91557  | bytes/ns |
| decompress time/Arade throughput                                                    |      1.96872     |      2.15756     |          0.912475 | bytes/ns |
| parquet_rs-zstd decompress time/Arade throughput                                    |      1.9332      |      2.07207     |          0.93298  | bytes/ns |
| compress time/Bimbo throughput                                                      |      0.551375    |      0.596863    |          0.923788 | bytes/ns |
| parquet_rs-zstd compress time/Bimbo throughput                                      |      0.425078    |      0.463411    |          0.91728  | bytes/ns |
| decompress time/Bimbo throughput                                                    |      2.43373     |      2.61018     |          0.932399 | bytes/ns |
| parquet_rs-zstd decompress time/Bimbo throughput                                    |      3.03312     |      3.26121     |          0.930062 | bytes/ns |
| compress time/CMSprovider throughput                                                |      0.0642161   |      0.0702219   |          0.914474 | bytes/ns |
| parquet_rs-zstd compress time/CMSprovider throughput                                |      0.421241    |      0.459591    |          0.916556 | bytes/ns |
| decompress time/CMSprovider throughput                                              |      4.57624     |      4.82146     |          0.94914  | bytes/ns |
| parquet_rs-zstd decompress time/CMSprovider throughput                              |      2.10423     |      2.28773     |          0.919791 | bytes/ns |
| compress time/Euro2016 throughput                                                   |      0.177567    |      0.192061    |          0.924533 | bytes/ns |
| parquet_rs-zstd compress time/Euro2016 throughput                                   |      0.345533    |      0.372171    |          0.928424 | bytes/ns |
| decompress time/Euro2016 throughput                                                 |      3.35727     |      3.7491      |          0.895486 | bytes/ns |
| parquet_rs-zstd decompress time/Euro2016 throughput                                 |      1.17589     |      1.27558     |          0.921843 | bytes/ns |
| compress time/Food throughput                                                       |      0.248863    |      0.273966    |          0.908371 | bytes/ns |
| parquet_rs-zstd compress time/Food throughput                                       |      0.39116     |      0.428981    |          0.911836 | bytes/ns |
| decompress time/Food throughput                                                     |      6.43487     |      7.20261     |          0.893408 | bytes/ns |
| parquet_rs-zstd decompress time/Food throughput                                     |      1.94258     |      2.11684     |          0.917678 | bytes/ns |
| compress time/HashTags throughput                                                   |      0.225188    |      0.244853    |          0.919685 | bytes/ns |
| parquet_rs-zstd compress time/HashTags throughput                                   |      0.947074    |      1.01183     |          0.936    | bytes/ns |
| decompress time/HashTags throughput                                                 |      7.53622     |      8.39421     |          0.897787 | bytes/ns |
| parquet_rs-zstd decompress time/HashTags throughput                                 |      3.28018     |      3.57385     |          0.917828 | bytes/ns |
| compress time/TPC-H l_comment chunked throughput                                    |      0.240286    |      0.258248    |          0.930444 | bytes/ns |
| parquet_rs-zstd compress time/TPC-H l_comment chunked throughput                    |      0.313111    |      0.339303    |          0.922807 | bytes/ns |
| decompress time/TPC-H l_comment chunked throughput                                  |      3.98011     |      4.09856     |          0.971101 | bytes/ns |
| parquet_rs-zstd decompress time/TPC-H l_comment chunked throughput                  |      1.37648     |      1.52439     |          0.902973 | bytes/ns |
| compress time/TPC-H l_comment canonical throughput                                  |      0.0318291   |      0.0344444   |          0.92407  | bytes/ns |
| parquet_rs-zstd compress time/TPC-H l_comment canonical throughput                  |      0.309153    |      0.337628    |          0.915661 | bytes/ns |
| decompress time/TPC-H l_comment canonical throughput                                |      4.39257     |      4.69208     |          0.936166 | bytes/ns |
| parquet_rs-zstd decompress time/TPC-H l_comment canonical throughput                |      1.37996     |      1.52104     |          0.907247 | bytes/ns |
| compress time/wide table cols=10 chunks=1 rows=1000 throughput                      |      0.173176    |      0.18121     |          0.955664 | bytes/ns |
| parquet_rs-zstd compress time/wide table cols=10 chunks=1 rows=1000 throughput      |      0.240631    |      0.250275    |          0.961465 | bytes/ns |
| decompress time/wide table cols=10 chunks=1 rows=1000 throughput                    |      0.847576    |      0.476448    |          1.77895  | bytes/ns |
| parquet_rs-zstd decompress time/wide table cols=10 chunks=1 rows=1000 throughput    |      0.522696    |      0.557055    |          0.93832  | bytes/ns |
| compress time/wide table cols=100 chunks=1 rows=1000 throughput                     |      0.170791    |      0.182348    |          0.936625 | bytes/ns |
| parquet_rs-zstd compress time/wide table cols=100 chunks=1 rows=1000 throughput     |      0.228476    |      0.24686     |          0.925526 | bytes/ns |
| decompress time/wide table cols=100 chunks=1 rows=1000 throughput                   |      1.23615     |      1.44198     |          0.857258 | bytes/ns |
| parquet_rs-zstd decompress time/wide table cols=100 chunks=1 rows=1000 throughput   |      0.516414    |      0.559773    |          0.922542 | bytes/ns |
| compress time/wide table cols=1000 chunks=1 rows=1000 throughput                    |      0.162846    |      0.179628    |          0.906575 | bytes/ns |
| parquet_rs-zstd compress time/wide table cols=1000 chunks=1 rows=1000 throughput    |      0.21315     |      0.242205    |          0.880042 | bytes/ns |
| decompress time/wide table cols=1000 chunks=1 rows=1000 throughput                  |      0.955398    |      1.01911     |          0.937479 | bytes/ns |
| parquet_rs-zstd decompress time/wide table cols=1000 chunks=1 rows=1000 throughput  |      0.489688    |      0.551662    |          0.887659 | bytes/ns |
| compress time/wide table cols=10 chunks=50 rows=1000 throughput                     |      0.0894774   |      0.0943711   |          0.948144 | bytes/ns |
| parquet_rs-zstd compress time/wide table cols=10 chunks=50 rows=1000 throughput     |      0.165779    |      0.172426    |          0.961451 | bytes/ns |
| decompress time/wide table cols=10 chunks=50 rows=1000 throughput                   |      0.838107    |      0.545271    |          1.53705  | bytes/ns |
| parquet_rs-zstd decompress time/wide table cols=10 chunks=50 rows=1000 throughput   |      0.513581    |      0.573166    |          0.896042 | bytes/ns |
| compress time/wide table cols=100 chunks=50 rows=1000 throughput                    |      0.0861333   |      0.0927617   |          0.928544 | bytes/ns |
| parquet_rs-zstd compress time/wide table cols=100 chunks=50 rows=1000 throughput    |      0.156215    |      0.166722    |          0.936981 | bytes/ns |
| decompress time/wide table cols=100 chunks=50 rows=1000 throughput                  |      1.16826     |      1.44746     |          0.807116 | bytes/ns |
| parquet_rs-zstd decompress time/wide table cols=100 chunks=50 rows=1000 throughput  |      0.5291      |      0.571058    |          0.926527 | bytes/ns |
| compress time/wide table cols=1000 chunks=50 rows=1000 throughput                   |      0.0699323   |      0.0852127   |          0.82068  | bytes/ns |
| parquet_rs-zstd compress time/wide table cols=1000 chunks=50 rows=1000 throughput   |      0.125104    |      0.139929    |          0.89405  | bytes/ns |
| decompress time/wide table cols=1000 chunks=50 rows=1000 throughput                 |      0.95582     |      1.04846     |          0.911646 | bytes/ns |
| parquet_rs-zstd decompress time/wide table cols=1000 chunks=50 rows=1000 throughput |      0.501524    |      0.562071    |          0.892278 | bytes/ns |
| vortex:raw size/taxi                                                                |      0.119399    |      0.119399    |          1        | ratio    |
| vortex size/taxi                                                                    |      5.9079e+07  |      5.9079e+07  |          1        | bytes    |
| vortex:parquet-zstd size/taxi                                                       |      1.05572     |      1.05572     |          1        | ratio    |
| vortex:raw size/AirlineSentiment                                                    |      1.35456     |      1.35456     |          1        | ratio    |
| vortex size/AirlineSentiment                                                        |   4424           |   4424           |          1        | bytes    |
| vortex:parquet-zstd size/AirlineSentiment                                           |      4.57497     |      4.57497     |          1        | ratio    |
| vortex:raw size/Arade                                                               |      0.255857    |      0.255857    |          1        | ratio    |
| vortex size/Arade                                                                   |      3.03621e+08 |      3.03621e+08 |          1        | bytes    |
| vortex:parquet-zstd size/Arade                                                      |      0.994197    |      0.994197    |          1        | ratio    |
| vortex:raw size/Bimbo                                                               |      0.116911    |      0.116911    |          1        | ratio    |
| vortex size/Bimbo                                                                   |      8.35811e+08 |      8.35811e+08 |          1        | bytes    |
| vortex:parquet-zstd size/Bimbo                                                      |      2.15331     |      2.15331     |          1        | ratio    |
| vortex:raw size/CMSprovider                                                         |      0.18503     |      0.18503     |          1        | ratio    |
| vortex size/CMSprovider                                                             |      1.16293e+09 |      1.16293e+09 |          1        | bytes    |
| vortex:parquet-zstd size/CMSprovider                                                |      1.51124     |      1.51124     |          1        | ratio    |
| vortex:raw size/Euro2016                                                            |      0.471335    |      0.471335    |          1        | ratio    |
| vortex size/Euro2016                                                                |      2.14468e+08 |      2.14468e+08 |          1        | bytes    |
| vortex:parquet-zstd size/Euro2016                                                   |      1.80392     |      1.80392     |          1        | ratio    |
| vortex:raw size/Food                                                                |      0.177323    |      0.177323    |          1        | ratio    |
| vortex size/Food                                                                    |      5.973e+07   |      5.973e+07   |          1        | bytes    |
| vortex:parquet-zstd size/Food                                                       |      1.64863     |      1.64863     |          1        | ratio    |
| vortex:raw size/HashTags                                                            |      0.142791    |      0.142791    |          1        | ratio    |
| vortex size/HashTags                                                                |      2.73577e+08 |      2.73577e+08 |          1        | bytes    |
| vortex:parquet-zstd size/HashTags                                                   |      2.04215     |      2.04215     |          1        | ratio    |
| vortex:raw size/TPC-H l_comment chunked                                             |      0.418167    |      0.418718    |          0.998682 | ratio    |
| vortex size/TPC-H l_comment chunked                                                 |      1.04206e+08 |      1.04343e+08 |          0.998682 | bytes    |
| vortex:parquet-zstd size/TPC-H l_comment chunked                                    |      1.83021     |      1.83262     |          0.998686 | ratio    |
| vortex:raw size/TPC-H l_comment canonical                                           |      0.425488    |      0.425482    |          1.00001  | ratio    |
| vortex size/TPC-H l_comment canonical                                               |      1.06028e+08 |      1.06026e+08 |          1.00001  | bytes    |
| vortex:parquet-zstd size/TPC-H l_comment canonical                                  |      1.86219     |      1.8621      |          1.00005  | ratio    |
| vortex:raw size/wide table cols=10 chunks=1 rows=1000                               |      0.626224    |      0.626224    |          1        | ratio    |
| vortex size/wide table cols=10 chunks=1 rows=1000                                   | 100256           | 100256           |          1        | bytes    |
| vortex:parquet-zstd size/wide table cols=10 chunks=1 rows=1000                      |      1.07244     |      1.07244     |          1        | ratio    |
| vortex:raw size/wide table cols=100 chunks=1 rows=1000                              |      0.623267    |      0.623267    |          1        | ratio    |
| vortex size/wide table cols=100 chunks=1 rows=1000                                  | 997736           | 997736           |          1        | bytes    |
| vortex:parquet-zstd size/wide table cols=100 chunks=1 rows=1000                     |      1.06732     |      1.06732     |          1        | ratio    |
| vortex:raw size/wide table cols=1000 chunks=1 rows=1000                             |      0.622971    |      0.622971    |          1        | ratio    |
| vortex size/wide table cols=1000 chunks=1 rows=1000                                 |      9.97254e+06 |      9.97254e+06 |          1        | bytes    |
| vortex:parquet-zstd size/wide table cols=1000 chunks=1 rows=1000                    |      1.06681     |      1.06681     |          1        | ratio    |
| vortex:raw size/wide table cols=10 chunks=50 rows=1000                              |      0.6098      |      0.6098      |          1        | ratio    |
| vortex size/wide table cols=10 chunks=50 rows=1000                                  | 100256           | 100256           |          1        | bytes    |
| vortex:parquet-zstd size/wide table cols=10 chunks=50 rows=1000                     |      1.07244     |      1.07244     |          1        | ratio    |
| vortex:raw size/wide table cols=100 chunks=50 rows=1000                             |      0.608224    |      0.608224    |          1        | ratio    |
| vortex size/wide table cols=100 chunks=50 rows=1000                                 | 997736           | 997736           |          1        | bytes    |
| vortex:parquet-zstd size/wide table cols=100 chunks=50 rows=1000                    |      1.06732     |      1.06732     |          1        | ratio    |
| vortex:raw size/wide table cols=1000 chunks=50 rows=1000                            |      0.608066    |      0.608066    |          1        | ratio    |
| vortex size/wide table cols=1000 chunks=50 rows=1000                                |      9.97254e+06 |      9.97254e+06 |          1        | bytes    |
| vortex:parquet-zstd size/wide table cols=1000 chunks=50 rows=1000                   |      1.06681     |      1.06681     |          1        | ratio    |
</details>

<!-- thollander/actions-comment-pull-request "bench-pr-comment-compress" -->
author_association: CONTRIBUTOR