Repository: vortex-data/vortex
Pull request #3363: File pruning when set
Author: login: joseph-isaacs
Labels: name: changelog/feature
description: A new feature
name: changelog/performance
description: A performance improvement
Created: 2025-05-21T11:10:15Z | Merged: 2025-05-21T14:44:40Z

Add whole file pruning to the vortex-file/ffi
Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>


Diff:
diff --git a/duckdb-vortex/src/include/vortex_common.hpp b/duckdb-vortex/src/include/vortex_common.hpp
index f5664521f3f..bc2e5185658 100644
--- a/duckdb-vortex/src/include/vortex_common.hpp
+++ b/duckdb-vortex/src/include/vortex_common.hpp
@@ -61,6 +61,10 @@ struct FileReader {
 		return duckdb::make_uniq<FileReader>(file);
 	}
 
+	vx_array_iterator *Scan(const vx_file_scan_options *options) {
+		return Try([&](auto err) { return vx_file_reader_scan(this->file, options, err); });
+	}
+
 	uint64_t FileRowCount() {
 		return Try([&](auto err) { return vx_file_row_count(file, err); });
 	}
diff --git a/duckdb-vortex/src/include/vortex_layout_reader.hpp b/duckdb-vortex/src/include/vortex_layout_reader.hpp
deleted file mode 100644
index 99ad14a8e2f..00000000000
--- a/duckdb-vortex/src/include/vortex_layout_reader.hpp
+++ /dev/null
@@ -1,29 +0,0 @@
-#pragma once
-
-#include "vortex_common.hpp"
-#include "vortex_expr.hpp"
-
-namespace vortex {
-
-class LayoutReader {
-public:
-	explicit LayoutReader(vx_layout_reader *reader) : reader(reader) {
-	}
-
-	~LayoutReader() {
-		vx_layout_reader_free(reader);
-	}
-
-	static std::shared_ptr<LayoutReader> CreateFromFile(vortex::FileReader *file) {
-		auto reader = Try([&](auto err) { return vx_layout_reader_create(file->file, err); });
-		return std::make_shared<LayoutReader>(reader);
-	}
-
-	vx_array_iterator *Scan(const vx_file_scan_options *options) {
-		return Try([&](auto err) { return vx_layout_reader_scan(this->reader, options, err); });
-	}
-
-	vx_layout_reader *reader;
-};
-
-} // namespace vortex
diff --git a/duckdb-vortex/src/vortex_scan.cpp b/duckdb-vortex/src/vortex_scan.cpp
index 6e760222f35..0771f19e6e5 100644
--- a/duckdb-vortex/src/vortex_scan.cpp
+++ b/duckdb-vortex/src/vortex_scan.cpp
@@ -18,8 +18,6 @@
 #include "concurrentqueue.h"
 
 #include "vortex.hpp"
-#include "vortex_extension.hpp"
-#include "vortex_layout_reader.hpp"
 #include "vortex_scan.hpp"
 #include "vortex_common.hpp"
 #include "vortex_expr.hpp"
@@ -42,7 +40,7 @@ struct BindData : public TableFunctionData {
 
 	// Used to read the schema during the bind phase and cached here to
 	// avoid having to open the same file again during the scan phase.
-	unique_ptr<FileReader> initial_file;
+	shared_ptr<FileReader> initial_file;
 
 	// Used to create an arena for protobuf exprs, need a ptr since the bind arg is const.
 	unique_ptr<google::protobuf::Arena> arena;
@@ -107,7 +105,7 @@ struct ScanGlobalState : public GlobalTableFunctionState {
 	// Multi producer, multi consumer lockfree queue.
 	duckdb_moodycamel::ConcurrentQueue<ScanPartition> scan_partitions {8192};
 
-	std::vector<std::shared_ptr<LayoutReader>> layout_readers;
+	std::vector<shared_ptr<FileReader>> file_readers;
 
 	// The column idx that must be returned by the scan.
 	vector<idx_t> column_ids;
@@ -252,9 +250,9 @@ static bool PinFileToThread(ScanGlobalState &global_state) {
 }
 
 static void CreateScanPartitions(ClientContext &context, const BindData &bind, ScanGlobalState &global_state,
-                                 ScanLocalState &local_state, uint64_t file_idx, unique_ptr<FileReader> &file_reader) {
+                                 ScanLocalState &local_state, uint64_t file_idx, FileReader &file_reader) {
 	const auto file_name = global_state.expanded_files[file_idx];
-	const auto row_count = Try([&](auto err) { return vx_file_row_count(file_reader->file, err); });
+	const auto row_count = Try([&](auto err) { return vx_file_row_count(file_reader.file, err); });
 
 	const auto thread_count = std::thread::hardware_concurrency();
 	const auto file_count = global_state.expanded_files.size();
@@ -292,8 +290,7 @@ static void CreateScanPartitions(ClientContext &context, const BindData &bind, S
 	D_ASSERT(global_state.files_partitioned <= global_state.expanded_files.size());
 }
 
-static unique_ptr<ArrayIterator> OpenArrayIter(ScanGlobalState &global_state,
-                                               std::shared_ptr<LayoutReader> &layout_reader,
+static unique_ptr<ArrayIterator> OpenArrayIter(ScanGlobalState &global_state, shared_ptr<FileReader> &file_reader,
                                                ScanPartition row_range_partition) {
 	const auto options = vx_file_scan_options {
 	    .projection = global_state.projected_column_names.data(),
@@ -305,7 +302,7 @@ static unique_ptr<ArrayIterator> OpenArrayIter(ScanGlobalState &global_state,
 	    .row_range_end = row_range_partition.end_row,
 	};
 
-	return make_uniq<ArrayIterator>(layout_reader->Scan(&options));
+	return make_uniq<ArrayIterator>(file_reader->Scan(&options));
 }
 
 // Assigns the next array from the array stream.
@@ -348,8 +345,8 @@ static bool GetNextArray(ClientContext &context, const BindData &bind_data, Scan
 
 		// Layout readers are safe to share across threads for reading. Further, they
 		// are created before pushing partitions of the corresponing files into a queue.
-		auto layout_reader = global_state.layout_readers[partition.file_idx];
-		local_state.array_iterator = OpenArrayIter(global_state, layout_reader, partition);
+		auto file_reader = global_state.file_readers[partition.file_idx];
+		local_state.array_iterator = OpenArrayIter(global_state, file_reader, partition);
 	}
 
 	local_state.currently_scanned_array = local_state.array_iterator->NextArray();
@@ -378,20 +375,25 @@ static void VortexScanFunction(ClientContext &context, TableFunctionInput &data,
 				return;
 			}
 
-			// Free layout readers as long as we pin files to threads.
+			// Free file readers when owned by the thread.
 			if (local_state.scan_partitions.empty() && local_state.thread_local_file_idx.has_value()) {
-				global_state.layout_readers[local_state.thread_local_file_idx.value()] = nullptr;
+				global_state.file_readers[local_state.thread_local_file_idx.value()] = nullptr;
 				local_state.thread_local_file_idx.reset();
 			}
 
 			// Create new scan partitions in case the queue is empty.
 			if (auto file_idx = global_state.next_file_idx.fetch_add(1);
 			    file_idx < global_state.expanded_files.size()) {
-				auto file_name = global_state.expanded_files[file_idx];
-				auto vortex_file =
-				    OpenFileAndVerify(FileSystem::GetFileSystem(context), *bind_data.session, file_name, bind_data);
-				global_state.layout_readers[file_idx] = LayoutReader::CreateFromFile(vortex_file.get());
-				CreateScanPartitions(context, bind_data, global_state, local_state, file_idx, vortex_file);
+				if (file_idx == 0) {
+					global_state.file_readers[0] = bind_data.initial_file;
+				} else {
+					auto file_name = global_state.expanded_files[file_idx];
+					global_state.file_readers[file_idx] =
+					    OpenFileAndVerify(FileSystem::GetFileSystem(context), *bind_data.session, file_name, bind_data);
+				}
+
+				CreateScanPartitions(context, bind_data, global_state, local_state, file_idx,
+				                     *global_state.file_readers[file_idx]);
 			}
 		}
 	}
@@ -491,7 +493,7 @@ void RegisterScanFunction(DatabaseInstance &instance) {
 		}
 
 		// Resizing the empty vector default constructs std::shared pointers at all indices with nullptr.
-		global_state->layout_readers.resize(global_state->expanded_files.size());
+		global_state->file_readers.resize(global_state->expanded_files.size());
 
 		bind.arena->Reset();
 		return std::move(global_state);
diff --git a/vortex-expr/src/pruning.rs b/vortex-expr/src/pruning.rs
index 04c87a8ed25..dff2b0620cc 100644
--- a/vortex-expr/src/pruning.rs
+++ b/vortex-expr/src/pruning.rs
@@ -397,7 +397,7 @@ pub enum FieldOrIdentity {
     Identity,
 }
 
-pub(crate) fn stat_field_name(field: &FieldName, stat: Stat) -> FieldName {
+pub fn stat_field_name(field: &FieldName, stat: Stat) -> FieldName {
     FieldName::from(stat_field_name_string(field, stat))
 }
 
diff --git a/vortex-ffi/cinclude/vortex.h b/vortex-ffi/cinclude/vortex.h
index 4d966e049c0..e9a67659a35 100644
--- a/vortex-ffi/cinclude/vortex.h
+++ b/vortex-ffi/cinclude/vortex.h
@@ -119,11 +119,6 @@ typedef struct vx_error vx_error;
  */
 typedef struct vx_file_reader vx_file_reader;
 
-/**
- * A Vortex layout reader.
- */
-typedef struct vx_layout_reader vx_layout_reader;
-
 /**
  * An object that stores registries and caches.
  * This should if possible be reused between queries in ann interactive session.
@@ -404,23 +399,15 @@ struct vx_dtype *vx_file_dtype(const struct vx_file_reader *file);
 /**
  * Build a new `vx_array_iterator` that returns a series of `vx_array`s from a scan over a `vx_layout_reader`.
  */
-struct vx_array_iterator *vx_layout_reader_scan(const struct vx_layout_reader *layout_reader,
-                                                const struct vx_file_scan_options *opts,
-                                                struct vx_error **error);
+struct vx_array_iterator *vx_file_reader_scan(const struct vx_file_reader *file_reader,
+                                              const struct vx_file_scan_options *opts,
+                                              struct vx_error **error);
 
 /**
  * Returns the row count for a given file reader.
  */
 uint64_t vx_file_row_count(struct vx_file_reader *file_reader, struct vx_error **error);
 
-/**
- * Creates a layout reader for a given file.
- */
-struct vx_layout_reader *vx_layout_reader_create(struct vx_file_reader *file_reader,
-                                                 struct vx_error **error);
-
-void vx_layout_reader_free(struct vx_layout_reader *layout_reader);
-
 /**
  * Free the file and all associated resources.
  *
diff --git a/vortex-ffi/src/array.rs b/vortex-ffi/src/array.rs
index bdff0a12416..f716fcc501f 100644
--- a/vortex-ffi/src/array.rs
+++ b/vortex-ffi/src/array.rs
@@ -29,15 +29,6 @@ pub struct vx_array_iterator {
     pub inner: Option<Box<dyn ArrayIterator>>,
 }
 
-/// Creates a new ArrayIterator wrapper for FFI use.
-pub fn vx_array_iterator<I>(iter: I) -> vortex::error::VortexResult<*mut vx_array_iterator>
-where
-    I: ArrayIterator + 'static,
-{
-    let inner = Some(Box::new(iter) as Box<dyn ArrayIterator>);
-    Ok(Box::into_raw(Box::new(vx_array_iterator { inner })))
-}
-
 /// Attempt to advance the `current` pointer of the iterator.
 ///
 /// A return value of `true` indicates that another element was pulled from the iterator, and a return
diff --git a/vortex-ffi/src/file.rs b/vortex-ffi/src/file.rs
index a10f97a3d03..ede2d1dd7b7 100644
--- a/vortex-ffi/src/file.rs
+++ b/vortex-ffi/src/file.rs
@@ -3,7 +3,7 @@
 use std::ffi::{CStr, c_char, c_int, c_uint, c_ulong};
 use std::str::FromStr;
 use std::sync::Arc;
-use std::{ptr, slice};
+use std::{iter, ptr, slice};
 
 use itertools::Itertools;
 use object_store::aws::{AmazonS3Builder, AmazonS3ConfigKey};
@@ -18,7 +18,7 @@ use vortex::error::{VortexError, VortexExpect, VortexResult, vortex_bail, vortex
 use vortex::expr::{ExprRef, Identity, deserialize_expr, select};
 use vortex::file::scan::SplitBy;
 use vortex::file::{VortexFile, VortexOpenOptions, VortexWriteOptions};
-use vortex::layout::LayoutReader;
+use vortex::iter::ArrayIteratorAdapter;
 use vortex::layout::scan::ScanBuilder;
 use vortex::proto::expr::Expr;
 
@@ -33,12 +33,6 @@ pub struct vx_file_reader {
     pub inner: VortexFile,
 }
 
-/// A Vortex layout reader.
-#[allow(non_camel_case_types)]
-pub struct vx_layout_reader {
-    pub inner: Arc<dyn LayoutReader>,
-}
-
 /// Options supplied for opening a file.
 #[repr(C)]
 pub struct vx_file_open_options {
@@ -242,20 +236,33 @@ pub unsafe extern "C-unwind" fn vx_file_dtype(file: *const vx_file_reader) -> *m
 
 /// Build a new `vx_array_iterator` that returns a series of `vx_array`s from a scan over a `vx_layout_reader`.
 #[unsafe(no_mangle)]
-pub unsafe extern "C-unwind" fn vx_layout_reader_scan(
-    layout_reader: *const vx_layout_reader,
+pub unsafe extern "C-unwind" fn vx_file_reader_scan(
+    file_reader: *const vx_file_reader,
     opts: *const vx_file_scan_options,
     error: *mut *mut vx_error,
 ) -> *mut vx_array_iterator {
     try_or(error, ptr::null_mut(), || {
-        let layout_reader = unsafe { layout_reader.as_ref().vortex_expect("null layout reader") };
-        let mut scan_builder = ScanBuilder::new(layout_reader.inner.clone());
+        let file_reader = unsafe { file_reader.as_ref().vortex_expect("null file reader") };
 
         let scan_options = unsafe { opts.as_ref() }.map_or_else(
             || Ok(ScanOptions::default()),
             |options| options.process_scan_options(),
         )?;
 
+        if let Some(expr) = &scan_options.filter_expr {
+            if file_reader.inner.can_prune(expr)? {
+                let dtype = file_reader.inner.dtype().clone();
+                let empty_iter = ArrayIteratorAdapter::new(dtype, iter::empty());
+
+                return Ok(Box::into_raw(Box::new(vx_array_iterator {
+                    inner: Some(Box::new(empty_iter)),
+                })));
+            }
+        };
+
+        let layout_reader = file_reader.inner.layout_reader()?;
+        let mut scan_builder = ScanBuilder::new(layout_reader.clone());
+
         // Apply options if provided.
         if let Some(field_names) = scan_options.field_names {
             // Field names are allowed to be `Some` and empty.
@@ -274,7 +281,9 @@ pub unsafe extern "C-unwind" fn vx_layout_reader_scan(
             scan_builder = scan_builder.with_split_by(split_by_value);
         }
 
-        vx_array_iterator(scan_builder.into_array_iter()?)
+        Ok(Box::into_raw(Box::new(vx_array_iterator {
+            inner: Some(Box::new(scan_builder.into_array_iter()?)),
+        })))
     })
 }
 
@@ -290,27 +299,6 @@ pub extern "C-unwind" fn vx_file_row_count(
     })
 }
 
-/// Creates a layout reader for a given file.
-#[unsafe(no_mangle)]
-pub extern "C-unwind" fn vx_layout_reader_create(
-    file_reader: *mut vx_file_reader,
-    error: *mut *mut vx_error,
-) -> *mut vx_layout_reader {
-    try_or(error, ptr::null_mut(), || {
-        let file_reader = unsafe { file_reader.as_ref().vortex_expect("null file_reader") };
-        let inner = file_reader.inner.layout_reader()?;
-
-        Ok(Box::into_raw(Box::new(vx_layout_reader { inner })))
-    })
-}
-
-#[unsafe(no_mangle)]
-pub extern "C-unwind" fn vx_layout_reader_free(layout_reader: *mut vx_layout_reader) {
-    if !layout_reader.is_null() {
-        drop(unsafe { Box::from_raw(layout_reader) });
-    }
-}
-
 /// Free the file and all associated resources.
 ///
 /// This function will not automatically free any :c:func:`vx_array_iterator` that were built from
diff --git a/vortex-file/src/file.rs b/vortex-file/src/file.rs
index f8758fa538b..325255b226b 100644
--- a/vortex-file/src/file.rs
+++ b/vortex-file/src/file.rs
@@ -8,12 +8,15 @@ use vortex_array::ArrayRef;
 use vortex_array::stats::StatsSet;
 use vortex_dtype::DType;
 use vortex_error::VortexResult;
+use vortex_expr::ExprRef;
+use vortex_expr::pruning::PruningPredicate;
 use vortex_layout::LayoutReader;
 use vortex_layout::scan::ScanBuilder;
 use vortex_layout::segments::SegmentSource;
 use vortex_metrics::VortexMetrics;
 
 use crate::footer::Footer;
+use crate::pruning::extract_relevant_stat_as_struct_row;
 
 /// Represents a Vortex file, providing access to its metadata and content.
 ///
@@ -80,6 +83,32 @@ impl VortexFile {
     pub fn scan(&self) -> VortexResult<ScanBuilder<ArrayRef>> {
         Ok(ScanBuilder::new(self.layout_reader()?).with_metrics(self.metrics.clone()))
     }
+
+    // Returns true if the expression will never match any rows in the file.
+    pub fn can_prune(&self, filter: &ExprRef) -> VortexResult<bool> {
+        let Some((file_stats, struct_dtype)) = self
+            .footer
+            .statistics()
+            .zip(self.footer.dtype().as_struct())
+        else {
+            return Ok(false);
+        };
+
+        let Some(predicate) = PruningPredicate::try_new(filter) else {
+            return Ok(false);
+        };
+
+        let Some(struct_row) =
+            extract_relevant_stat_as_struct_row(&predicate, file_stats, struct_dtype)?
+        else {
+            return Ok(false);
+        };
+
+        Ok(predicate
+            .evaluate(&struct_row)?
+            .and_then(|p| p.as_constant())
+            .is_some_and(|result| result.as_bool().value() == Some(true)))
+    }
 }
 
 /// A factory for creating segment sources that read data from a Vortex file.
diff --git a/vortex-file/src/lib.rs b/vortex-file/src/lib.rs
index b5d873f9fb5..ccce24f6398 100644
--- a/vortex-file/src/lib.rs
+++ b/vortex-file/src/lib.rs
@@ -96,6 +96,7 @@ mod footer;
 mod generic;
 mod memory;
 mod open;
+mod pruning;
 pub mod segments;
 mod strategy;
 #[cfg(test)]
diff --git a/vortex-file/src/pruning.rs b/vortex-file/src/pruning.rs
new file mode 100644
index 00000000000..97ecce6d2d3
--- /dev/null
+++ b/vortex-file/src/pruning.rs
@@ -0,0 +1,52 @@
+use std::sync::Arc;
+
+use vortex_array::ArrayRef;
+use vortex_array::arrays::{ConstantArray, StructArray};
+use vortex_array::stats::{Stat, StatsSet};
+use vortex_array::validity::Validity;
+use vortex_dtype::StructDType;
+use vortex_error::VortexResult;
+use vortex_expr::pruning::{FieldOrIdentity, PruningPredicate, stat_field_name};
+use vortex_scalar::Scalar;
+
+pub fn extract_relevant_stat_as_struct_row(
+    predicate: &PruningPredicate,
+    stats_set: &Arc<[StatsSet]>,
+    struct_dtype: &Arc<StructDType>,
+) -> VortexResult<Option<ArrayRef>> {
+    if predicate.required_stats().is_empty() {
+        return StructArray::try_new([].into(), vec![], 1, Validity::NonNullable)
+            .map(|s| Some(s.to_array()));
+    }
+    let mut columns = vec![];
+    for (field_name, stats) in predicate.required_stats() {
+        let FieldOrIdentity::Field(field) = field_name else {
+            return Ok(None);
+        };
+
+        let field_idx = struct_dtype.find(field)?;
+        let field_dtype = struct_dtype.field_by_index(field_idx)?;
+        let Some(cols) = stats_set[field_idx]
+            .iter()
+            .filter(|(stat, _)| stats.contains(stat))
+            .map(|(stat, value)| {
+                value.as_ref().as_exact().and_then(|value| {
+                    (stat == &Stat::Max || stat == &Stat::Min).then(|| {
+                        (
+                            stat_field_name(field, *stat),
+                            ConstantArray::new(Scalar::new(field_dtype.clone(), value.clone()), 1)
+                                .to_array(),
+                        )
+                    })
+                })
+            })
+            .collect::<Option<Vec<_>>>()
+        else {
+            return Ok(None);
+        };
+
+        columns.extend(cols)
+    }
+
+    StructArray::from_fields(&columns).map(|s| Some(s.to_array()))
+}
diff --git a/vortex-layout/src/scan/mod.rs b/vortex-layout/src/scan/mod.rs
index 2a5ba895094..47497e61188 100644
--- a/vortex-layout/src/scan/mod.rs
+++ b/vortex-layout/src/scan/mod.rs
@@ -13,6 +13,7 @@ use itertools::Itertools;
 pub use selection::*;
 pub use split_by::*;
 use vortex_array::iter::{ArrayIterator, ArrayIteratorAdapter};
+use vortex_array::stats::StatsSet;
 use vortex_array::stream::{ArrayStream, ArrayStreamAdapter};
 use vortex_array::{ArrayRef, ToCanonical};
 use vortex_buffer::Buffer;
@@ -48,6 +49,8 @@ pub struct ScanBuilder<A> {
     /// The executor used to spawn each split task.
     executor: Option<Arc<dyn TaskExecutor>>,
     metrics: VortexMetrics,
+    /// Should we try to prune the file (using stats) on open.
+    file_stats: Option<Arc<[StatsSet]>>,
 }
 
 impl<A: 'static + Send> ScanBuilder<A> {
@@ -118,6 +121,11 @@ impl<A: 'static + Send> ScanBuilder<A> {
         self
     }
 
+    pub fn with_prune_file_on_open(mut self, stats_set: Arc<[StatsSet]>) -> Self {
+        self.file_stats = Some(stats_set);
+        self
+    }
+
     /// Map each split of the scan. The function will be run on the spawned task.
     pub fn map<B: 'static>(
         self,
@@ -135,6 +143,7 @@ impl<A: 'static + Send> ScanBuilder<A> {
             map_fn: Arc::new(move |a| map_fn(old_map_fn(a)?)),
             executor: self.executor,
             metrics: self.metrics,
+            file_stats: self.file_stats,
         }
     }
 
@@ -289,6 +298,7 @@ impl ScanBuilder<ArrayRef> {
             map_fn: Arc::new(Ok),
             executor: None,
             metrics: Default::default(),
+            file_stats: None,
         }
     }
 


Reviews:
user: login: gatesn
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T12:19:38Z
user: login: joseph-isaacs
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T13:40:51Z
user: login: joseph-isaacs
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T13:49:02Z
user: login: 0ax1
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T14:25:28Z
user: login: joseph-isaacs
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T14:27:30Z
user: login: 0ax1
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T14:27:58Z
user: login: joseph-isaacs
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T14:34:51Z
user: login: 0ax1
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T14:35:25Z

Review comments:
diff_hunk: @@ -161,6 +174,26 @@ impl<A: 'static + Send> ScanBuilder<A> {
             .map(|f| simplify_typed(f, layout_reader.dtype()))
             .transpose()?;
 
+        if let Some(((file_stats, filter), struct_dtype)) = self
+            .file_stats
+            .zip(self.filter.as_ref())
+            .zip(layout_reader.dtype().as_struct())
+        {
+            if let Some(predicate) = PruningPredicate::try_new(filter) {
+                if let Some(struct_row) = extract_struct_row(&predicate, &file_stats, struct_dtype)?
+                {
+                    if let Some(result) = predicate
+                        .evaluate(&struct_row)?
path: vortex-layout/src/scan/mod.rs
user: login: gatesn
body: Can you fix the evaluate function to take a trait? Maybe it looks like:

```
StatsSource {
 fn fetch(&self, field_path: &FieldPath, stat: &Stat) -> Option<ArrayRef>;
}
```

That way you don't need to construct the full stats array if we only end up using one field.
created_at: 2025-05-21T12:18:45Z
author_association: CONTRIBUTOR
diff_hunk: @@ -48,6 +53,8 @@ pub struct ScanBuilder<A> {
     /// The executor used to spawn each split task.
     executor: Option<Arc<dyn TaskExecutor>>,
     metrics: VortexMetrics,
+    /// Should we try to prune the file (using stats) on open.
+    file_stats: Option<Arc<[StatsSet]>>,
path: vortex-layout/src/scan/mod.rs
user: login: gatesn
body: I think you could actually create a `FileStatsLayoutReader` and provide a `impl PruningEvaluator`. Then inside `VortexFile::layout_reader()` you can wrap up the root layout with the `FileStatsLayoutReader`, that way the scan never even knows.
created_at: 2025-05-21T12:19:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -48,6 +53,8 @@ pub struct ScanBuilder<A> {
     /// The executor used to spawn each split task.
     executor: Option<Arc<dyn TaskExecutor>>,
     metrics: VortexMetrics,
+    /// Should we try to prune the file (using stats) on open.
+    file_stats: Option<Arc<[StatsSet]>>,
path: vortex-layout/src/scan/mod.rs
user: login: joseph-isaacs
body: We agreed this does extra work, so going to add a method on the vortex file itself
created_at: 2025-05-21T13:40:51Z
author_association: CONTRIBUTOR
diff_hunk: @@ -161,6 +174,26 @@ impl<A: 'static + Send> ScanBuilder<A> {
             .map(|f| simplify_typed(f, layout_reader.dtype()))
             .transpose()?;
 
+        if let Some(((file_stats, filter), struct_dtype)) = self
+            .file_stats
+            .zip(self.filter.as_ref())
+            .zip(layout_reader.dtype().as_struct())
+        {
+            if let Some(predicate) = PruningPredicate::try_new(filter) {
+                if let Some(struct_row) = extract_struct_row(&predicate, &file_stats, struct_dtype)?
+                {
+                    if let Some(result) = predicate
+                        .evaluate(&struct_row)?
path: vortex-layout/src/scan/mod.rs
user: login: joseph-isaacs
body: We already do this, would you prefer this method to live in a impl/trait?
created_at: 2025-05-21T13:49:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -243,19 +237,32 @@ pub unsafe extern "C-unwind" fn vx_file_dtype(file: *const vx_file_reader) -> *m
 /// Build a new `vx_array_iterator` that returns a series of `vx_array`s from a scan over a `vx_layout_reader`.
 #[unsafe(no_mangle)]
 pub unsafe extern "C-unwind" fn vx_layout_reader_scan(
-    layout_reader: *const vx_layout_reader,
+    file_reader: *const vx_file_reader,
     opts: *const vx_file_scan_options,
     error: *mut *mut vx_error,
 ) -> *mut vx_array_iterator {
     try_or(error, ptr::null_mut(), || {
-        let layout_reader = unsafe { layout_reader.as_ref().vortex_expect("null layout reader") };
-        let mut scan_builder = ScanBuilder::new(layout_reader.inner.clone());
+        let file_reader = unsafe { file_reader.as_ref().vortex_expect("null file reader") };
 
         let scan_options = unsafe { opts.as_ref() }.map_or_else(
             || Ok(ScanOptions::default()),
             |options| options.process_scan_options(),
         )?;
 
+        if let Some(expr) = &scan_options.filter_expr {
path: vortex-ffi/src/file.rs
user: login: 0ax1
body: ```
       if let Some(expr) = &scan_options.filter_expr {
            if file_reader.inner.can_prune(expr)? {
                let dtype = file_reader.inner.dtype().clone();
                let empty_iter = ArrayIteratorAdapter::new(dtype, std::iter::empty());

                return Ok(Box::into_raw(Box::new(vx_array_iterator {
                    inner: Some(Box::new(empty_iter)),
                })));
            }
        };
```

?
created_at: 2025-05-21T14:13:19Z
author_association: CONTRIBUTOR
diff_hunk: @@ -292,8 +290,7 @@ static void CreateScanPartitions(ClientContext &context, const BindData &bind, S
 	D_ASSERT(global_state.files_partitioned <= global_state.expanded_files.size());
 }
 
-static unique_ptr<ArrayIterator> OpenArrayIter(ScanGlobalState &global_state,
-                                               std::shared_ptr<LayoutReader> &layout_reader,
+static unique_ptr<ArrayIterator> OpenArrayIter(ScanGlobalState &global_state, shared_ptr<FileReader> &layout_reader,
path: duckdb-vortex/src/vortex_scan.cpp
user: login: 0ax1
body: rename to `file_reader`?
created_at: 2025-05-21T14:14:24Z
author_association: CONTRIBUTOR
diff_hunk: @@ -380,18 +377,24 @@ static void VortexScanFunction(ClientContext &context, TableFunctionInput &data,
 
 			// Free layout readers as long as we pin files to threads.
 			if (local_state.scan_partitions.empty() && local_state.thread_local_file_idx.has_value()) {
-				global_state.layout_readers[local_state.thread_local_file_idx.value()] = nullptr;
+				global_state.file_readers[local_state.thread_local_file_idx.value()] = nullptr;
 				local_state.thread_local_file_idx.reset();
 			}
 
 			// Create new scan partitions in case the queue is empty.
 			if (auto file_idx = global_state.next_file_idx.fetch_add(1);
 			    file_idx < global_state.expanded_files.size()) {
-				auto file_name = global_state.expanded_files[file_idx];
-				auto vortex_file =
-				    OpenFileAndVerify(FileSystem::GetFileSystem(context), *bind_data.session, file_name, bind_data);
-				global_state.layout_readers[file_idx] = LayoutReader::CreateFromFile(vortex_file.get());
-				CreateScanPartitions(context, bind_data, global_state, local_state, file_idx, vortex_file);
+				if (file_idx == 0) {
+					global_state.file_readers[0] = bind_data.initial_file;
+				} else {
+					auto file_name = global_state.expanded_files[file_idx];
+					auto vortex_file =
+					    OpenFileAndVerify(FileSystem::GetFileSystem(context), *bind_data.session, file_name, bind_data);
+					global_state.file_readers[file_idx] = std::move(vortex_file);
path: duckdb-vortex/src/vortex_scan.cpp
user: login: 0ax1
body: This is correct but in general I think we should be *very* careful with using `move` unless there's a clear perf upside.
created_at: 2025-05-21T14:16:01Z
author_association: CONTRIBUTOR
diff_hunk: @@ -274,7 +281,9 @@ pub unsafe extern "C-unwind" fn vx_layout_reader_scan(
             scan_builder = scan_builder.with_split_by(split_by_value);
         }
 
-        vx_array_iterator(scan_builder.into_array_iter()?)
+        Ok(Box::into_raw(Box::new(vx_array_iterator {
+            inner: Some(Box::new(scan_builder.into_array_iter()?) as _),
path: vortex-ffi/src/file.rs
user: login: 0ax1
body: ```suggestion
            inner: Some(Box::new(scan_builder.into_array_iter()?)),
```
created_at: 2025-05-21T14:21:22Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,54 @@
+use std::sync::Arc;
+
+use vortex_array::ArrayRef;
+use vortex_array::arrays::{ConstantArray, StructArray};
+use vortex_array::stats::{Precision, Stat, StatsSet};
+use vortex_array::validity::Validity;
+use vortex_dtype::StructDType;
+use vortex_error::VortexResult;
+use vortex_expr::pruning::{FieldOrIdentity, PruningPredicate, stat_field_name};
+use vortex_scalar::Scalar;
+
+pub fn extract_relevant_stat_as_struct_row(
+    predicate: &PruningPredicate,
+    stats_set: &Arc<[StatsSet]>,
+    struct_dtype: &Arc<StructDType>,
+) -> VortexResult<Option<ArrayRef>> {
+    if predicate.required_stats().is_empty() {
+        return StructArray::try_new([].into(), vec![], 1, Validity::NonNullable)
+            .map(|s| Some(s.to_array()));
+    }
+    let mut columns = vec![];
+    for (field_name, stats) in predicate.required_stats() {
+        let FieldOrIdentity::Field(field) = field_name else {
+            return Ok(None);
+        };
+
+        let field_idx = struct_dtype.find(field)?;
+        let field_dtype = struct_dtype.field_by_index(field_idx)?;
+        let Some(cols) = stats_set[field_idx]
+            .iter()
+            .filter(|(stat, _)| stats.contains(stat))
+            .map(|(stat, value)| {
+                if let Precision::Exact(value) = value {
path: vortex-file/src/pruning.rs
user: login: 0ax1
body: This gets hard to read due to the formatting. Could consider extracting the closure, and move it outside of the loop.
created_at: 2025-05-21T14:25:10Z
author_association: CONTRIBUTOR
diff_hunk: @@ -274,7 +281,9 @@ pub unsafe extern "C-unwind" fn vx_layout_reader_scan(
             scan_builder = scan_builder.with_split_by(split_by_value);
         }
 
-        vx_array_iterator(scan_builder.into_array_iter()?)
+        Ok(Box::into_raw(Box::new(vx_array_iterator {
+            inner: Some(Box::new(scan_builder.into_array_iter()?) as _),
path: vortex-ffi/src/file.rs
user: login: joseph-isaacs
body: This is needed, I believe
created_at: 2025-05-21T14:27:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -274,7 +281,9 @@ pub unsafe extern "C-unwind" fn vx_layout_reader_scan(
             scan_builder = scan_builder.with_split_by(split_by_value);
         }
 
-        vx_array_iterator(scan_builder.into_array_iter()?)
+        Ok(Box::into_raw(Box::new(vx_array_iterator {
+            inner: Some(Box::new(scan_builder.into_array_iter()?) as _),
path: vortex-ffi/src/file.rs
user: login: 0ax1
body: Works on my machine tm.
created_at: 2025-05-21T14:27:58Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,54 @@
+use std::sync::Arc;
+
+use vortex_array::ArrayRef;
+use vortex_array::arrays::{ConstantArray, StructArray};
+use vortex_array::stats::{Precision, Stat, StatsSet};
+use vortex_array::validity::Validity;
+use vortex_dtype::StructDType;
+use vortex_error::VortexResult;
+use vortex_expr::pruning::{FieldOrIdentity, PruningPredicate, stat_field_name};
+use vortex_scalar::Scalar;
+
+pub fn extract_relevant_stat_as_struct_row(
+    predicate: &PruningPredicate,
+    stats_set: &Arc<[StatsSet]>,
+    struct_dtype: &Arc<StructDType>,
+) -> VortexResult<Option<ArrayRef>> {
+    if predicate.required_stats().is_empty() {
+        return StructArray::try_new([].into(), vec![], 1, Validity::NonNullable)
+            .map(|s| Some(s.to_array()));
+    }
+    let mut columns = vec![];
+    for (field_name, stats) in predicate.required_stats() {
+        let FieldOrIdentity::Field(field) = field_name else {
+            return Ok(None);
+        };
+
+        let field_idx = struct_dtype.find(field)?;
+        let field_dtype = struct_dtype.field_by_index(field_idx)?;
+        let Some(cols) = stats_set[field_idx]
+            .iter()
+            .filter(|(stat, _)| stats.contains(stat))
+            .map(|(stat, value)| {
+                if let Precision::Exact(value) = value {
path: vortex-file/src/pruning.rs
user: login: joseph-isaacs
body: I have refined
created_at: 2025-05-21T14:34:51Z
author_association: CONTRIBUTOR

PR comments:
user: login: github-actions[bot]
created_at: 2025-05-21T11:34:35Z
body: # Benchmarks: TPC-H on NVME
<details>
<summary>Table of Results</summary>

| name                                       |   PR a0259a18 |   base 11554a45 |   ratio (PR/base) | unit   |
|:-------------------------------------------|--------------:|----------------:|------------------:|:-------|
| tpch_q01/DataFusion:arrow                  |      65124517 |     6.45076e+07 |          1.00956  | ns     |
| tpch_q02/DataFusion:arrow                  |      16121183 |     1.65499e+07 |          0.974097 | ns     |
| tpch_q03/DataFusion:arrow                  |      23812352 |     2.38398e+07 |          0.99885  | ns     |
| tpch_q04/DataFusion:arrow                  |      18175486 |     1.90352e+07 |          0.954837 | ns     |
| tpch_q05/DataFusion:arrow                  |      31813194 |     3.2262e+07  |          0.986089 | ns     |
| tpch_q06/DataFusion:arrow                  |       6359870 |     6.09679e+06 |          1.04315  | ns     |
| tpch_q07/DataFusion:arrow                  |      52530086 |     5.36193e+07 |          0.979686 | ns     |
| tpch_q08/DataFusion:arrow                  |      18481650 |     1.91034e+07 |          0.967453 | ns     |
| tpch_q09/DataFusion:arrow                  |      33905457 |     3.45635e+07 |          0.98096  | ns     |
| tpch_q10/DataFusion:arrow                  |      38696651 |     3.91663e+07 |          0.98801  | ns     |
| tpch_q11/DataFusion:arrow                  |       7685587 |     7.84279e+06 |          0.979955 | ns     |
| tpch_q12/DataFusion:arrow                  |      18058135 |     1.82583e+07 |          0.989036 | ns     |
| tpch_q13/DataFusion:arrow                  |      14071561 |     1.47306e+07 |          0.955259 | ns     |
| tpch_q14/DataFusion:arrow                  |       8058840 |     8.34953e+06 |          0.965185 | ns     |
| tpch_q15/DataFusion:arrow                  |      17899891 |     1.86771e+07 |          0.958386 | ns     |
| tpch_q16/DataFusion:arrow                  |      14678339 |     1.58141e+07 |          0.928182 | ns     |
| tpch_q17/DataFusion:arrow                  |      48442547 |     5.28387e+07 |          0.916801 | ns     |
| tpch_q18/DataFusion:arrow                  |     104317450 |     1.12632e+08 |          0.926179 | ns     |
| tpch_q19/DataFusion:arrow                  |      17355815 |     1.7578e+07  |          0.987359 | ns     |
| tpch_q20/DataFusion:arrow                  |      23084889 |     2.40748e+07 |          0.958881 | ns     |
| tpch_q21/DataFusion:arrow                  |      82022840 |     8.45102e+07 |          0.970567 | ns     |
| tpch_q22/DataFusion:arrow                  |      10298757 |     1.03245e+07 |          0.997511 | ns     |
| tpch_q01/DataFusion:parquet                |     103177696 |     1.02958e+08 |          1.00213  | ns     |
| tpch_q02/DataFusion:parquet                |      50198922 |     5.07684e+07 |          0.988782 | ns     |
| tpch_q03/DataFusion:parquet                |      48838237 |     4.8814e+07  |          1.0005   | ns     |
| tpch_q04/DataFusion:parquet                |      30896264 |     3.15257e+07 |          0.980035 | ns     |
| tpch_q05/DataFusion:parquet                |      64288971 |     6.31736e+07 |          1.01766  | ns     |
| tpch_q06/DataFusion:parquet                |      29003956 |     2.80609e+07 |          1.03361  | ns     |
| tpch_q07/DataFusion:parquet                |      80196748 |     8.16718e+07 |          0.981939 | ns     |
| tpch_q08/DataFusion:parquet                |      62487762 |     6.33437e+07 |          0.986487 | ns     |
| tpch_q09/DataFusion:parquet                |      86515028 |     8.47538e+07 |          1.02078  | ns     |
| tpch_q10/DataFusion:parquet                |      93311796 |     9.47619e+07 |          0.984698 | ns     |
| tpch_q11/DataFusion:parquet                |      28912103 |     2.91266e+07 |          0.992637 | ns     |
| tpch_q12/DataFusion:parquet                |      40887814 |     4.05864e+07 |          1.00743  | ns     |
| tpch_q13/DataFusion:parquet                |      39878772 |     4.11827e+07 |          0.968338 | ns     |
| tpch_q14/DataFusion:parquet                |      32844801 |     3.26803e+07 |          1.00503  | ns     |
| tpch_q15/DataFusion:parquet                |      54378751 |     5.47226e+07 |          0.993717 | ns     |
| tpch_q16/DataFusion:parquet                |      34466028 |     3.4493e+07  |          0.999218 | ns     |
| tpch_q17/DataFusion:parquet                |      95593475 |     9.57426e+07 |          0.998443 | ns     |
| tpch_q18/DataFusion:parquet                |     116067890 |     1.15955e+08 |          1.00098  | ns     |
| tpch_q19/DataFusion:parquet                |      71403058 |     7.24413e+07 |          0.985668 | ns     |
| tpch_q20/DataFusion:parquet                |      56049144 |     5.53929e+07 |          1.01185  | ns     |
| tpch_q21/DataFusion:parquet                |     101934160 |     1.02583e+08 |          0.993676 | ns     |
| tpch_q22/DataFusion:parquet                |      50103233 |     5.01312e+07 |          0.999442 | ns     |
| tpch_q01/DataFusion:vortex-file-compressed |      58063244 |     5.99098e+07 |          0.969177 | ns     |
| tpch_q02/DataFusion:vortex-file-compressed |      38478690 |     3.87261e+07 |          0.993612 | ns     |
| tpch_q03/DataFusion:vortex-file-compressed |      29299024 |     2.89913e+07 |          1.01061  | ns     |
| tpch_q04/DataFusion:vortex-file-compressed |      14606775 |     1.49636e+07 |          0.976151 | ns     |
| tpch_q05/DataFusion:vortex-file-compressed |      46598467 |     4.70615e+07 |          0.990161 | ns     |
| tpch_q06/DataFusion:vortex-file-compressed |       2380986 |     2.2582e+06  |          1.05437  | ns     |
| tpch_q07/DataFusion:vortex-file-compressed |      62241979 |     6.42047e+07 |          0.969431 | ns     |
| tpch_q08/DataFusion:vortex-file-compressed |      51449799 |     5.35786e+07 |          0.960269 | ns     |
| tpch_q09/DataFusion:vortex-file-compressed |      74396313 |     7.49131e+07 |          0.993101 | ns     |
| tpch_q10/DataFusion:vortex-file-compressed |      42876143 |     4.43192e+07 |          0.96744  | ns     |
| tpch_q11/DataFusion:vortex-file-compressed |      25567868 |     2.57574e+07 |          0.992643 | ns     |
| tpch_q12/DataFusion:vortex-file-compressed |      15259310 |     1.49914e+07 |          1.01787  | ns     |
| tpch_q13/DataFusion:vortex-file-compressed |      17816217 |     1.79614e+07 |          0.991915 | ns     |
| tpch_q14/DataFusion:vortex-file-compressed |       9378199 |     9.48695e+06 |          0.988537 | ns     |
| tpch_q15/DataFusion:vortex-file-compressed |      16872907 |     1.61388e+07 |          1.04549  | ns     |
| tpch_q16/DataFusion:vortex-file-compressed |      23838187 |     2.40704e+07 |          0.990352 | ns     |
| tpch_q17/DataFusion:vortex-file-compressed |      77387689 |     7.6689e+07  |          1.00911  | ns     |
| tpch_q18/DataFusion:vortex-file-compressed |      94415216 |     9.4564e+07  |          0.998427 | ns     |
| tpch_q19/DataFusion:vortex-file-compressed |      10815984 |     1.10621e+07 |          0.977751 | ns     |
| tpch_q20/DataFusion:vortex-file-compressed |      26737389 |     2.55951e+07 |          1.04463  | ns     |
| tpch_q21/DataFusion:vortex-file-compressed |      86271633 |     8.60905e+07 |          1.0021   | ns     |
| tpch_q22/DataFusion:vortex-file-compressed |      12024670 |     1.19974e+07 |          1.00227  | ns     |
| tpch_q01/DuckDB:parquet                    |      55000000 |     5.5e+07     |          1        | ns     |
| tpch_q02/DuckDB:parquet                    |      33000000 |     3.4e+07     |          0.970588 | ns     |
| tpch_q03/DuckDB:parquet                    |      64000000 |     6.4e+07     |          1        | ns     |
| tpch_q04/DuckDB:parquet                    |      34000000 |     3.3e+07     |          1.0303   | ns     |
| tpch_q05/DuckDB:parquet                    |      53000000 |     5.2e+07     |          1.01923  | ns     |
| tpch_q06/DuckDB:parquet                    |      26000000 |     2.6e+07     |          1        | ns     |
| tpch_q07/DuckDB:parquet                    |      56000000 |     5.3e+07     |          1.0566   | ns     |
| tpch_q08/DuckDB:parquet                    |      61000000 |     6.3e+07     |          0.968254 | ns     |
| tpch_q09/DuckDB:parquet                    |      89000000 |     8.9e+07     |          1        | ns     |
| tpch_q10/DuckDB:parquet                    |     236000000 |     2.4e+08     |          0.983333 | ns     |
| tpch_q11/DuckDB:parquet                    |      17000000 |     1.7e+07     |          1        | ns     |
| tpch_q12/DuckDB:parquet                    |      40000000 |     4.2e+07     |          0.952381 | ns     |
| tpch_q13/DuckDB:parquet                    |      49000000 |     5.1e+07     |          0.960784 | ns     |
| tpch_q14/DuckDB:parquet                    |      44000000 |     4.4e+07     |          1        | ns     |
| tpch_q15/DuckDB:parquet                    |      56000000 |     5.4e+07     |          1.03704  | ns     |
| tpch_q16/DuckDB:parquet                    |      54000000 |     5.4e+07     |          1        | ns     |
| tpch_q17/DuckDB:parquet                    |      57000000 |     5.6e+07     |          1.01786  | ns     |
| tpch_q18/DuckDB:parquet                    |      66000000 |     6.6e+07     |          1        | ns     |
| tpch_q19/DuckDB:parquet                    |      65000000 |     6.6e+07     |          0.984848 | ns     |
| tpch_q20/DuckDB:parquet                    |      52000000 |     5.2e+07     |          1        | ns     |
| tpch_q21/DuckDB:parquet                    |     105000000 |     1.04e+08    |          1.00962  | ns     |
| tpch_q22/DuckDB:parquet                    |      37000000 |     3.6e+07     |          1.02778  | ns     |
| tpch_q01/DuckDB:vortex-file-compressed     |      61000000 |     4.1e+07     |          1.4878   | ns     |
| tpch_q02/DuckDB:vortex-file-compressed     |      37000000 |     3.9e+07     |          0.948718 | ns     |
| tpch_q03/DuckDB:vortex-file-compressed     |      64000000 |     5.7e+07     |          1.12281  | ns     |
| tpch_q04/DuckDB:vortex-file-compressed     |      50000000 |     4e+07       |          1.25     | ns     |
| tpch_q05/DuckDB:vortex-file-compressed     |      55000000 |     4.8e+07     |          1.14583  | ns     |
| tpch_q06/DuckDB:vortex-file-compressed     |       5000000 |     8e+06       |          0.625    | ns     |
| tpch_q07/DuckDB:vortex-file-compressed     |      61000000 |     4.5e+07     |          1.35556  | ns     |
| tpch_q08/DuckDB:vortex-file-compressed     |      68000000 |     6.2e+07     |          1.09677  | ns     |
| tpch_q09/DuckDB:vortex-file-compressed     |      99000000 |     8.9e+07     |          1.11236  | ns     |
| tpch_q10/DuckDB:vortex-file-compressed     |     210000000 |     2.02e+08    |          1.0396   | ns     |
| tpch_q11/DuckDB:vortex-file-compressed     |      24000000 |     2.1e+07     |          1.14286  | ns     |
| tpch_q12/DuckDB:vortex-file-compressed     |      49000000 |     3.1e+07     |          1.58065  | ns     |
| tpch_q13/DuckDB:vortex-file-compressed     |      59000000 |     5.2e+07     |          1.13462  | ns     |
| tpch_q14/DuckDB:vortex-file-compressed     |      34000000 |     2.4e+07     |          1.41667  | ns     |
| tpch_q15/DuckDB:vortex-file-compressed     |      59000000 |     4.1e+07     |          1.43902  | ns     |
| tpch_q16/DuckDB:vortex-file-compressed     |      54000000 |     5.4e+07     |          1        | ns     |
| tpch_q17/DuckDB:vortex-file-compressed     |      54000000 |     4.5e+07     |          1.2      | ns     |
| tpch_q18/DuckDB:vortex-file-compressed     |      72000000 |     6.3e+07     |          1.14286  | ns     |
| tpch_q19/DuckDB:vortex-file-compressed     |      56000000 |     3.6e+07     |          1.55556  | ns     |
| tpch_q20/DuckDB:vortex-file-compressed     |      48000000 |     3.9e+07     |          1.23077  | ns     |
| tpch_q21/DuckDB:vortex-file-compressed     |     138000000 |     1.15e+08    |          1.2      | ns     |
| tpch_q22/DuckDB:vortex-file-compressed     |      32000000 |     3.2e+07     |          1        | ns     |
| tpch_q01/DuckDB:duckdb                     |      25000000 |     2.5e+07     |          1        | ns     |
| tpch_q02/DuckDB:duckdb                     |      19000000 |     2e+07       |          0.95     | ns     |
| tpch_q03/DuckDB:duckdb                     |      43000000 |     4.3e+07     |          1        | ns     |
| tpch_q04/DuckDB:duckdb                     |      23000000 |     2.2e+07     |          1.04545  | ns     |
| tpch_q05/DuckDB:duckdb                     |      31000000 |     3e+07       |          1.03333  | ns     |
| tpch_q06/DuckDB:duckdb                     |      10000000 |     9e+06       |          1.11111  | ns     |
| tpch_q07/DuckDB:duckdb                     |      30000000 |     3.2e+07     |          0.9375   | ns     |
| tpch_q08/DuckDB:duckdb                     |      31000000 |     3e+07       |          1.03333  | ns     |
| tpch_q09/DuckDB:duckdb                     |      65000000 |     6.6e+07     |          0.984848 | ns     |
| tpch_q10/DuckDB:duckdb                     |     208000000 |     2.06e+08    |          1.00971  | ns     |
| tpch_q11/DuckDB:duckdb                     |      10000000 |     9e+06       |          1.11111  | ns     |
| tpch_q12/DuckDB:duckdb                     |      30000000 |     3e+07       |          1        | ns     |
| tpch_q13/DuckDB:duckdb                     |      40000000 |     4.1e+07     |          0.97561  | ns     |
| tpch_q14/DuckDB:duckdb                     |      29000000 |     3e+07       |          0.966667 | ns     |
| tpch_q15/DuckDB:duckdb                     |      38000000 |     3.4e+07     |          1.11765  | ns     |
| tpch_q16/DuckDB:duckdb                     |      45000000 |     4.5e+07     |          1        | ns     |
| tpch_q17/DuckDB:duckdb                     |      31000000 |     2.9e+07     |          1.06897  | ns     |
| tpch_q18/DuckDB:duckdb                     |      42000000 |     4.2e+07     |          1        | ns     |
| tpch_q19/DuckDB:duckdb                     |      42000000 |     4.2e+07     |          1        | ns     |
| tpch_q20/DuckDB:duckdb                     |      33000000 |     3.3e+07     |          1        | ns     |
| tpch_q21/DuckDB:duckdb                     |      59000000 |     6e+07       |          0.983333 | ns     |
| tpch_q22/DuckDB:duckdb                     |      28000000 |     2.8e+07     |          1        | ns     |
</details>

<!-- thollander/actions-comment-pull-request "bench-pr-comment-tpch-nvme" -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-21T11:40:44Z
body: # Benchmarks: TPC-H on S3
<details>
<summary>Table of Results</summary>

| name                                       |   PR a0259a18 |   base 11554a45 |   ratio (PR/base) | unit   |
|:-------------------------------------------|--------------:|----------------:|------------------:|:-------|
| tpch_q01/DataFusion:parquet                |     224244501 |     2.1939e+08  |          1.02213  | ns     |
| tpch_q02/DataFusion:parquet                |     771041416 |     7.85113e+08 |          0.982077 | ns     |
| tpch_q03/DataFusion:parquet                |     364726649 |     3.49851e+08 |          1.04252  | ns     |
| tpch_q04/DataFusion:parquet                |     233104482 |     2.27271e+08 |          1.02567  | ns     |
| tpch_q05/DataFusion:parquet                |     549857234 |     5.55735e+08 |          0.989423 | ns     |
| tpch_q06/DataFusion:parquet                |     153920959 |     1.53131e+08 |          1.00516  | ns     |
| tpch_q07/DataFusion:parquet                |     527126090 |     5.28059e+08 |          0.998233 | ns     |
| tpch_q08/DataFusion:parquet                |     753251361 |     7.90404e+08 |          0.952995 | ns     |
| tpch_q09/DataFusion:parquet                |     609879687 |     6.13623e+08 |          0.9939   | ns     |
| tpch_q10/DataFusion:parquet                |     542605957 |     5.30999e+08 |          1.02186  | ns     |
| tpch_q11/DataFusion:parquet                |     488172518 |     4.7927e+08  |          1.01857  | ns     |
| tpch_q12/DataFusion:parquet                |     297141751 |     2.89061e+08 |          1.02796  | ns     |
| tpch_q13/DataFusion:parquet                |     215314999 |     2.05339e+08 |          1.04858  | ns     |
| tpch_q14/DataFusion:parquet                |     293479615 |     2.93599e+08 |          0.999593 | ns     |
| tpch_q15/DataFusion:parquet                |     409178620 |     4.02779e+08 |          1.01589  | ns     |
| tpch_q16/DataFusion:parquet                |     380564759 |     3.95526e+08 |          0.962174 | ns     |
| tpch_q17/DataFusion:parquet                |     541131038 |     5.4789e+08  |          0.987664 | ns     |
| tpch_q18/DataFusion:parquet                |     469343894 |     4.66846e+08 |          1.00535  | ns     |
| tpch_q19/DataFusion:parquet                |     385383035 |     3.94202e+08 |          0.977628 | ns     |
| tpch_q20/DataFusion:parquet                |     528280011 |     5.55658e+08 |          0.950729 | ns     |
| tpch_q21/DataFusion:parquet                |     767710806 |     7.48456e+08 |          1.02573  | ns     |
| tpch_q22/DataFusion:parquet                |     288243195 |     2.77257e+08 |          1.03962  | ns     |
| tpch_q01/DataFusion:vortex-file-compressed |     157467501 |     1.58827e+08 |          0.991442 | ns     |
| tpch_q02/DataFusion:vortex-file-compressed |     163709730 |     1.54977e+08 |          1.05635  | ns     |
| tpch_q03/DataFusion:vortex-file-compressed |     241096368 |     2.29883e+08 |          1.04878  | ns     |
| tpch_q04/DataFusion:vortex-file-compressed |     167854192 |     1.37294e+08 |          1.22259  | ns     |
| tpch_q05/DataFusion:vortex-file-compressed |     246780382 |     2.37231e+08 |          1.04025  | ns     |
| tpch_q06/DataFusion:vortex-file-compressed |       9923532 |     8.10378e+06 |          1.22456  | ns     |
| tpch_q07/DataFusion:vortex-file-compressed |     239551893 |     2.47702e+08 |          0.967096 | ns     |
| tpch_q08/DataFusion:vortex-file-compressed |     284601193 |     2.97296e+08 |          0.9573   | ns     |
| tpch_q09/DataFusion:vortex-file-compressed |     308826255 |     3.00104e+08 |          1.02906  | ns     |
| tpch_q10/DataFusion:vortex-file-compressed |     274366801 |     2.66266e+08 |          1.03042  | ns     |
| tpch_q11/DataFusion:vortex-file-compressed |     128609528 |     1.27041e+08 |          1.01235  | ns     |
| tpch_q12/DataFusion:vortex-file-compressed |     200391596 |     2.01273e+08 |          0.995619 | ns     |
| tpch_q13/DataFusion:vortex-file-compressed |     131928073 |     1.2793e+08  |          1.03125  | ns     |
| tpch_q14/DataFusion:vortex-file-compressed |     134804647 |     1.42176e+08 |          0.94815  | ns     |
| tpch_q15/DataFusion:vortex-file-compressed |     214962476 |     2.1705e+08  |          0.990381 | ns     |
| tpch_q16/DataFusion:vortex-file-compressed |      90581889 |     9.50127e+07 |          0.953366 | ns     |
| tpch_q17/DataFusion:vortex-file-compressed |     264004596 |     2.63603e+08 |          1.00152  | ns     |
| tpch_q18/DataFusion:vortex-file-compressed |     292352800 |     2.81835e+08 |          1.03732  | ns     |
| tpch_q19/DataFusion:vortex-file-compressed |      43693181 |     3.93837e+07 |          1.10942  | ns     |
| tpch_q20/DataFusion:vortex-file-compressed |     231156995 |     2.11492e+08 |          1.09298  | ns     |
| tpch_q21/DataFusion:vortex-file-compressed |     374696351 |     3.75454e+08 |          0.997982 | ns     |
| tpch_q22/DataFusion:vortex-file-compressed |     104096230 |     1.05217e+08 |          0.989344 | ns     |
| tpch_q01/DuckDB:parquet                    |     499000000 |     4.68e+08    |          1.06624  | ns     |
| tpch_q02/DuckDB:parquet                    |    1131000000 |     1.101e+09   |          1.02725  | ns     |
| tpch_q03/DuckDB:parquet                    |     674000000 |     7.28e+08    |          0.925824 | ns     |
| tpch_q04/DuckDB:parquet                    |     443000000 |     4.42e+08    |          1.00226  | ns     |
| tpch_q05/DuckDB:parquet                    |     926000000 |     9.6e+08     |          0.964583 | ns     |
| tpch_q06/DuckDB:parquet                    |     310000000 |     3.13e+08    |          0.990415 | ns     |
| tpch_q07/DuckDB:parquet                    |     867000000 |     8.53e+08    |          1.01641  | ns     |
| tpch_q08/DuckDB:parquet                    |     982000000 |     1.087e+09   |          0.903404 | ns     |
| tpch_q09/DuckDB:parquet                    |    1035000000 |     9.61e+08    |          1.077    | ns     |
| tpch_q10/DuckDB:parquet                    |    1104000000 |     1.081e+09   |          1.02128  | ns     |
| tpch_q11/DuckDB:parquet                    |     618000000 |     6.47e+08    |          0.955178 | ns     |
| tpch_q12/DuckDB:parquet                    |     424000000 |     4.02e+08    |          1.05473  | ns     |
| tpch_q13/DuckDB:parquet                    |     333000000 |     3.02e+08    |          1.10265  | ns     |
| tpch_q14/DuckDB:parquet                    |     457000000 |     4.42e+08    |          1.03394  | ns     |
| tpch_q15/DuckDB:parquet                    |     781000000 |     7.91e+08    |          0.987358 | ns     |
| tpch_q16/DuckDB:parquet                    |     402000000 |     4.21e+08    |          0.954869 | ns     |
| tpch_q17/DuckDB:parquet                    |     695000000 |     6.74e+08    |          1.03116  | ns     |
| tpch_q18/DuckDB:parquet                    |     641000000 |     6.63e+08    |          0.966817 | ns     |
| tpch_q19/DuckDB:parquet                    |     602000000 |     5.91e+08    |          1.01861  | ns     |
| tpch_q20/DuckDB:parquet                    |     744000000 |     6.91e+08    |          1.0767   | ns     |
| tpch_q21/DuckDB:parquet                    |    1012000000 |     1.004e+09   |          1.00797  | ns     |
| tpch_q22/DuckDB:parquet                    |     414000000 |     3.99e+08    |          1.03759  | ns     |
| tpch_q01/DuckDB:vortex-file-compressed     |     374000000 |     3.14e+08    |          1.19108  | ns     |
| tpch_q02/DuckDB:vortex-file-compressed     |     453000000 |     5.31e+08    |          0.853107 | ns     |
| tpch_q03/DuckDB:vortex-file-compressed     |     485000000 |     6.81e+08    |          0.712188 | ns     |
| tpch_q04/DuckDB:vortex-file-compressed     |     363000000 |     4.38e+08    |          0.828767 | ns     |
| tpch_q05/DuckDB:vortex-file-compressed     |     553000000 |     7.78e+08    |          0.710797 | ns     |
| tpch_q06/DuckDB:vortex-file-compressed     |      48000000 |     8e+07       |          0.6      | ns     |
| tpch_q07/DuckDB:vortex-file-compressed     |     515000000 |     7.15e+08    |          0.72028  | ns     |
| tpch_q08/DuckDB:vortex-file-compressed     |     678000000 |     9.64e+08    |          0.70332  | ns     |
| tpch_q09/DuckDB:vortex-file-compressed     |     644000000 |     7.62e+08    |          0.845144 | ns     |
| tpch_q10/DuckDB:vortex-file-compressed     |     677000000 |     9.12e+08    |          0.742325 | ns     |
| tpch_q11/DuckDB:vortex-file-compressed     |     267000000 |     3.4e+08     |          0.785294 | ns     |
| tpch_q12/DuckDB:vortex-file-compressed     |     405000000 |     4.62e+08    |          0.876623 | ns     |
| tpch_q13/DuckDB:vortex-file-compressed     |     235000000 |     3.21e+08    |          0.732087 | ns     |
| tpch_q14/DuckDB:vortex-file-compressed     |     335000000 |     4.16e+08    |          0.805288 | ns     |
| tpch_q15/DuckDB:vortex-file-compressed     |     456000000 |     5.44e+08    |          0.838235 | ns     |
| tpch_q16/DuckDB:vortex-file-compressed     |     288000000 |     3.54e+08    |          0.813559 | ns     |
| tpch_q17/DuckDB:vortex-file-compressed     |     443000000 |     6.17e+08    |          0.71799  | ns     |
| tpch_q18/DuckDB:vortex-file-compressed     |     462000000 |     6.38e+08    |          0.724138 | ns     |
| tpch_q19/DuckDB:vortex-file-compressed     |     417000000 |     4.07e+08    |          1.02457  | ns     |
| tpch_q20/DuckDB:vortex-file-compressed     |     567000000 |     6.94e+08    |          0.817003 | ns     |
| tpch_q21/DuckDB:vortex-file-compressed     |     785000000 |     9.82e+08    |          0.799389 | ns     |
| tpch_q22/DuckDB:vortex-file-compressed     |     223000000 |     2.78e+08    |          0.802158 | ns     |
</details>

<!-- thollander/actions-comment-pull-request "bench-pr-comment-tpch-s3" -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-21T11:48:50Z
body: # Benchmarks: Clickbench on NVME
<details>
<summary>Table of Results</summary>

| name                                             |   PR a0259a18 |   base 11554a45 |   ratio (PR/base) | unit   |
|:-------------------------------------------------|--------------:|----------------:|------------------:|:-------|
| clickbench_q00/DataFusion:parquet                |       1338366 |     1.30337e+06 |          1.02685  | ns     |
| clickbench_q01/DataFusion:parquet                |      28544347 |     2.79637e+07 |          1.02076  | ns     |
| clickbench_q02/DataFusion:parquet                |      68997460 |     6.51087e+07 |          1.05973  | ns     |
| clickbench_q03/DataFusion:parquet                |      57569241 |     5.36278e+07 |          1.0735   | ns     |
| clickbench_q04/DataFusion:parquet                |     412861190 |     3.92543e+08 |          1.05176  | ns     |
| clickbench_q05/DataFusion:parquet                |     393237373 |     3.73935e+08 |          1.05162  | ns     |
| clickbench_q06/DataFusion:parquet                |       1365078 |     1.37235e+06 |          0.994704 | ns     |
| clickbench_q07/DataFusion:parquet                |      31977784 |     3.12115e+07 |          1.02455  | ns     |
| clickbench_q08/DataFusion:parquet                |     488915838 |     4.53822e+08 |          1.07733  | ns     |
| clickbench_q09/DataFusion:parquet                |     720869241 |     6.7361e+08  |          1.07016  | ns     |
| clickbench_q10/DataFusion:parquet                |     152569869 |     1.46925e+08 |          1.03842  | ns     |
| clickbench_q11/DataFusion:parquet                |     182022293 |     1.74143e+08 |          1.04525  | ns     |
| clickbench_q12/DataFusion:parquet                |     423935620 |     4.05997e+08 |          1.04418  | ns     |
| clickbench_q13/DataFusion:parquet                |     607104734 |     5.72347e+08 |          1.06073  | ns     |
| clickbench_q14/DataFusion:parquet                |     411247481 |     3.9073e+08  |          1.05251  | ns     |
| clickbench_q15/DataFusion:parquet                |     489857385 |     4.62252e+08 |          1.05972  | ns     |
| clickbench_q16/DataFusion:parquet                |     854980621 |     8.16039e+08 |          1.04772  | ns     |
| clickbench_q17/DataFusion:parquet                |     819009173 |     7.88329e+08 |          1.03892  | ns     |
| clickbench_q18/DataFusion:parquet                |    1558597646 |     1.49045e+09 |          1.04572  | ns     |
| clickbench_q19/DataFusion:parquet                |      45375390 |     4.13937e+07 |          1.09619  | ns     |
| clickbench_q20/DataFusion:parquet                |     664554183 |     6.46197e+08 |          1.02841  | ns     |
| clickbench_q21/DataFusion:parquet                |     758149492 |     7.44267e+08 |          1.01865  | ns     |
| clickbench_q22/DataFusion:parquet                |    1222086333 |     1.16853e+09 |          1.04583  | ns     |
| clickbench_q23/DataFusion:parquet                |    5151433197 |     4.90212e+09 |          1.05086  | ns     |
| clickbench_q24/DataFusion:parquet                |     254117649 |     2.49991e+08 |          1.01651  | ns     |
| clickbench_q25/DataFusion:parquet                |     216147475 |     2.07723e+08 |          1.04055  | ns     |
| clickbench_q26/DataFusion:parquet                |     288843210 |     2.75809e+08 |          1.04726  | ns     |
| clickbench_q27/DataFusion:parquet                |    1005492397 |     9.60749e+08 |          1.04657  | ns     |
| clickbench_q28/DataFusion:parquet                |    5888486868 |     5.82976e+09 |          1.01007  | ns     |
| clickbench_q29/DataFusion:parquet                |     249636122 |     2.35989e+08 |          1.05783  | ns     |
| clickbench_q30/DataFusion:parquet                |     409196935 |     3.88432e+08 |          1.05346  | ns     |
| clickbench_q31/DataFusion:parquet                |     454035917 |     4.28321e+08 |          1.06004  | ns     |
| clickbench_q32/DataFusion:parquet                |    1417640938 |     1.31801e+09 |          1.07559  | ns     |
| clickbench_q33/DataFusion:parquet                |    1757940453 |     1.68281e+09 |          1.04465  | ns     |
| clickbench_q34/DataFusion:parquet                |    1753863549 |     1.66769e+09 |          1.05167  | ns     |
| clickbench_q35/DataFusion:parquet                |     652823501 |     6.18528e+08 |          1.05545  | ns     |
| clickbench_q36/DataFusion:parquet                |     184308731 |     1.7268e+08  |          1.06734  | ns     |
| clickbench_q37/DataFusion:parquet                |      75266246 |     7.05761e+07 |          1.06646  | ns     |
| clickbench_q38/DataFusion:parquet                |     116084142 |     1.12205e+08 |          1.03457  | ns     |
| clickbench_q39/DataFusion:parquet                |     350362638 |     3.51366e+08 |          0.997145 | ns     |
| clickbench_q40/DataFusion:parquet                |      43897896 |     4.04467e+07 |          1.08533  | ns     |
| clickbench_q41/DataFusion:parquet                |      39560323 |     3.88252e+07 |          1.01893  | ns     |
| clickbench_q42/DataFusion:parquet                |      63298736 |     5.84148e+07 |          1.08361  | ns     |
| clickbench_q00/DataFusion:vortex-file-compressed |       1296801 |     1.3406e+06  |          0.967327 | ns     |
| clickbench_q01/DataFusion:vortex-file-compressed |       7903285 |     7.78924e+06 |          1.01464  | ns     |
| clickbench_q02/DataFusion:vortex-file-compressed |      27159188 |     2.72966e+07 |          0.994966 | ns     |
| clickbench_q03/DataFusion:vortex-file-compressed |      36178083 |     3.42716e+07 |          1.05563  | ns     |
| clickbench_q04/DataFusion:vortex-file-compressed |     353531991 |     3.36439e+08 |          1.05081  | ns     |
| clickbench_q05/DataFusion:vortex-file-compressed |     353018751 |     3.35384e+08 |          1.05258  | ns     |
| clickbench_q06/DataFusion:vortex-file-compressed |       1369114 |     1.35211e+06 |          1.01258  | ns     |
| clickbench_q07/DataFusion:vortex-file-compressed |      15085732 |     1.00245e+07 |          1.50489  | ns     |
| clickbench_q08/DataFusion:vortex-file-compressed |     404509243 |     3.82817e+08 |          1.05666  | ns     |
| clickbench_q09/DataFusion:vortex-file-compressed |     591858378 |     5.50991e+08 |          1.07417  | ns     |
| clickbench_q10/DataFusion:vortex-file-compressed |      74223004 |     7.52508e+07 |          0.986342 | ns     |
| clickbench_q11/DataFusion:vortex-file-compressed |      86013821 |     8.41324e+07 |          1.02236  | ns     |
| clickbench_q12/DataFusion:vortex-file-compressed |     313399865 |     2.84333e+08 |          1.10223  | ns     |
| clickbench_q13/DataFusion:vortex-file-compressed |     507833054 |     4.78387e+08 |          1.06155  | ns     |
| clickbench_q14/DataFusion:vortex-file-compressed |     284378442 |     2.69718e+08 |          1.05435  | ns     |
| clickbench_q15/DataFusion:vortex-file-compressed |     425899228 |     4.01798e+08 |          1.05998  | ns     |
| clickbench_q16/DataFusion:vortex-file-compressed |     854630520 |     8.28233e+08 |          1.03187  | ns     |
| clickbench_q17/DataFusion:vortex-file-compressed |     829594568 |     7.9114e+08  |          1.04861  | ns     |
| clickbench_q18/DataFusion:vortex-file-compressed |    1580563365 |     1.49699e+09 |          1.05583  | ns     |
| clickbench_q19/DataFusion:vortex-file-compressed |      14535696 |     1.42066e+07 |          1.02317  | ns     |
| clickbench_q20/DataFusion:vortex-file-compressed |     277383373 |     2.74292e+08 |          1.01127  | ns     |
| clickbench_q21/DataFusion:vortex-file-compressed |     339532010 |     3.3064e+08  |          1.02689  | ns     |
| clickbench_q22/DataFusion:vortex-file-compressed |     723161878 |     6.72312e+08 |          1.07563  | ns     |
| clickbench_q23/DataFusion:vortex-file-compressed |    2107398292 |     2.03702e+09 |          1.03455  | ns     |
| clickbench_q24/DataFusion:vortex-file-compressed |     131996471 |     1.304e+08   |          1.01225  | ns     |
| clickbench_q25/DataFusion:vortex-file-compressed |      97355298 |     1.01414e+08 |          0.959975 | ns     |
| clickbench_q26/DataFusion:vortex-file-compressed |     162250348 |     1.59026e+08 |          1.02027  | ns     |
| clickbench_q27/DataFusion:vortex-file-compressed |     718407140 |     6.98187e+08 |          1.02896  | ns     |
| clickbench_q28/DataFusion:vortex-file-compressed |    6830446199 |     6.71606e+09 |          1.01703  | ns     |
| clickbench_q29/DataFusion:vortex-file-compressed |     244908648 |     2.58824e+08 |          0.946235 | ns     |
| clickbench_q30/DataFusion:vortex-file-compressed |     259540997 |     2.4301e+08  |          1.06802  | ns     |
| clickbench_q31/DataFusion:vortex-file-compressed |     294759027 |     2.82725e+08 |          1.04257  | ns     |
| clickbench_q32/DataFusion:vortex-file-compressed |    1384237542 |     1.27876e+09 |          1.08248  | ns     |
| clickbench_q33/DataFusion:vortex-file-compressed |    1626045743 |     1.57537e+09 |          1.03217  | ns     |
| clickbench_q34/DataFusion:vortex-file-compressed |    1622136069 |     1.52076e+09 |          1.06666  | ns     |
| clickbench_q35/DataFusion:vortex-file-compressed |     575471717 |     5.48106e+08 |          1.04993  | ns     |
| clickbench_q36/DataFusion:vortex-file-compressed |     103155166 |     9.37577e+07 |          1.10023  | ns     |
| clickbench_q37/DataFusion:vortex-file-compressed |      44819663 |     3.72568e+07 |          1.20299  | ns     |
| clickbench_q38/DataFusion:vortex-file-compressed |      29956973 |     2.91144e+07 |          1.02894  | ns     |
| clickbench_q39/DataFusion:vortex-file-compressed |     208228794 |     1.86222e+08 |          1.11817  | ns     |
| clickbench_q40/DataFusion:vortex-file-compressed |      17786135 |     1.65575e+07 |          1.0742   | ns     |
| clickbench_q41/DataFusion:vortex-file-compressed |      15519461 |     1.42284e+07 |          1.09074  | ns     |
| clickbench_q42/DataFusion:vortex-file-compressed |      37483679 |     3.7254e+07  |          1.00616  | ns     |
| clickbench_q00/DuckDB:parquet                    |     236000000 |     2.22e+08    |          1.06306  | ns     |
| clickbench_q01/DuckDB:parquet                    |     147000000 |     1.43e+08    |          1.02797  | ns     |
| clickbench_q02/DuckDB:parquet                    |     170000000 |     1.62e+08    |          1.04938  | ns     |
| clickbench_q03/DuckDB:parquet                    |     171000000 |     1.61e+08    |          1.06211  | ns     |
| clickbench_q04/DuckDB:parquet                    |     282000000 |     2.71e+08    |          1.04059  | ns     |
| clickbench_q05/DuckDB:parquet                    |     309000000 |     2.94e+08    |          1.05102  | ns     |
| clickbench_q06/DuckDB:parquet                    |     159000000 |     1.52e+08    |          1.04605  | ns     |
| clickbench_q07/DuckDB:parquet                    |     152000000 |     1.48e+08    |          1.02703  | ns     |
| clickbench_q08/DuckDB:parquet                    |     317000000 |     3.03e+08    |          1.0462   | ns     |
| clickbench_q09/DuckDB:parquet                    |     384000000 |     3.77e+08    |          1.01857  | ns     |
| clickbench_q10/DuckDB:parquet                    |     204000000 |     1.96e+08    |          1.04082  | ns     |
| clickbench_q11/DuckDB:parquet                    |     218000000 |     2.09e+08    |          1.04306  | ns     |
| clickbench_q12/DuckDB:parquet                    |     315000000 |     3.09e+08    |          1.01942  | ns     |
| clickbench_q13/DuckDB:parquet                    |     492000000 |     4.72e+08    |          1.04237  | ns     |
| clickbench_q14/DuckDB:parquet                    |     347000000 |     3.38e+08    |          1.02663  | ns     |
| clickbench_q15/DuckDB:parquet                    |     301000000 |     2.9e+08     |          1.03793  | ns     |
| clickbench_q16/DuckDB:parquet                    |     606000000 |     5.91e+08    |          1.02538  | ns     |
| clickbench_q17/DuckDB:parquet                    |     527000000 |     5.08e+08    |          1.0374   | ns     |
| clickbench_q18/DuckDB:parquet                    |    1004000000 |     9.63e+08    |          1.04258  | ns     |
| clickbench_q19/DuckDB:parquet                    |     148000000 |     1.47e+08    |          1.0068   | ns     |
| clickbench_q20/DuckDB:parquet                    |     673000000 |     6.7e+08     |          1.00448  | ns     |
| clickbench_q21/DuckDB:parquet                    |     573000000 |     5.68e+08    |          1.0088   | ns     |
| clickbench_q22/DuckDB:parquet                    |     930000000 |     9.15e+08    |          1.01639  | ns     |
| clickbench_q23/DuckDB:parquet                    |    2169000000 |     2.123e+09   |          1.02167  | ns     |
| clickbench_q24/DuckDB:parquet                    |     137000000 |     1.34e+08    |          1.02239  | ns     |
| clickbench_q25/DuckDB:parquet                    |     206000000 |     1.99e+08    |          1.03518  | ns     |
| clickbench_q26/DuckDB:parquet                    |     136000000 |     1.33e+08    |          1.02256  | ns     |
| clickbench_q27/DuckDB:parquet                    |     657000000 |     6.46e+08    |          1.01703  | ns     |
| clickbench_q28/DuckDB:parquet                    |    6116000000 |     6.066e+09   |          1.00824  | ns     |
| clickbench_q29/DuckDB:parquet                    |     163000000 |     1.56e+08    |          1.04487  | ns     |
| clickbench_q30/DuckDB:parquet                    |     354000000 |     3.45e+08    |          1.02609  | ns     |
| clickbench_q31/DuckDB:parquet                    |     413000000 |     4e+08       |          1.0325   | ns     |
| clickbench_q32/DuckDB:parquet                    |    1110000000 |     1.067e+09   |          1.0403   | ns     |
| clickbench_q33/DuckDB:parquet                    |    1231000000 |     1.178e+09   |          1.04499  | ns     |
| clickbench_q34/DuckDB:parquet                    |    1278000000 |     1.239e+09   |          1.03148  | ns     |
| clickbench_q35/DuckDB:parquet                    |     338000000 |     3.47e+08    |          0.974063 | ns     |
| clickbench_q36/DuckDB:parquet                    |     143000000 |     1.38e+08    |          1.03623  | ns     |
| clickbench_q37/DuckDB:parquet                    |     136000000 |     1.33e+08    |          1.02256  | ns     |
| clickbench_q38/DuckDB:parquet                    |     149000000 |     1.43e+08    |          1.04196  | ns     |
| clickbench_q39/DuckDB:parquet                    |     170000000 |     1.64e+08    |          1.03659  | ns     |
| clickbench_q40/DuckDB:parquet                    |     142000000 |     1.36e+08    |          1.04412  | ns     |
| clickbench_q41/DuckDB:parquet                    |     149000000 |     1.45e+08    |          1.02759  | ns     |
| clickbench_q42/DuckDB:parquet                    |     136000000 |     1.29e+08    |          1.05426  | ns     |
| clickbench_q00/DuckDB:vortex-file-compressed     |      26000000 |     2.7e+07     |          0.962963 | ns     |
| clickbench_q01/DuckDB:vortex-file-compressed     |      36000000 |     3.3e+07     |          1.09091  | ns     |
| clickbench_q02/DuckDB:vortex-file-compressed     |     134000000 |     1.37e+08    |          0.978102 | ns     |
| clickbench_q03/DuckDB:vortex-file-compressed     |      91000000 |     8.8e+07     |          1.03409  | ns     |
| clickbench_q04/DuckDB:vortex-file-compressed     |     262000000 |     2.56e+08    |          1.02344  | ns     |
| clickbench_q05/DuckDB:vortex-file-compressed     |     305000000 |     2.89e+08    |          1.05536  | ns     |
| clickbench_q06/DuckDB:vortex-file-compressed     |      74000000 |     7.6e+07     |          0.973684 | ns     |
| clickbench_q07/DuckDB:vortex-file-compressed     |      43000000 |     4.4e+07     |          0.977273 | ns     |
| clickbench_q08/DuckDB:vortex-file-compressed     |     373000000 |     3.53e+08    |          1.05666  | ns     |
| clickbench_q09/DuckDB:vortex-file-compressed     |     520000000 |     5.02e+08    |          1.03586  | ns     |
| clickbench_q10/DuckDB:vortex-file-compressed     |     137000000 |     1.29e+08    |          1.06202  | ns     |
| clickbench_q11/DuckDB:vortex-file-compressed     |     150000000 |     1.45e+08    |          1.03448  | ns     |
| clickbench_q12/DuckDB:vortex-file-compressed     |     287000000 |     2.86e+08    |          1.0035   | ns     |
| clickbench_q13/DuckDB:vortex-file-compressed     |     488000000 |     4.68e+08    |          1.04274  | ns     |
| clickbench_q14/DuckDB:vortex-file-compressed     |     323000000 |     3.01e+08    |          1.07309  | ns     |
| clickbench_q15/DuckDB:vortex-file-compressed     |     282000000 |     2.81e+08    |          1.00356  | ns     |
| clickbench_q16/DuckDB:vortex-file-compressed     |     722000000 |     6.88e+08    |          1.04942  | ns     |
| clickbench_q17/DuckDB:vortex-file-compressed     |     645000000 |     6.21e+08    |          1.03865  | ns     |
| clickbench_q18/DuckDB:vortex-file-compressed     |    1346000000 |     1.27e+09    |          1.05984  | ns     |
| clickbench_q19/DuckDB:vortex-file-compressed     |     390000000 |     3.25e+08    |          1.2      | ns     |
| clickbench_q20/DuckDB:vortex-file-compressed     |     489000000 |     4.86e+08    |          1.00617  | ns     |
| clickbench_q21/DuckDB:vortex-file-compressed     |     456000000 |     4.61e+08    |          0.989154 | ns     |
| clickbench_q22/DuckDB:vortex-file-compressed     |     753000000 |     7.34e+08    |          1.02589  | ns     |
| clickbench_q23/DuckDB:vortex-file-compressed     |    1808000000 |     1.908e+09   |          0.947589 | ns     |
| clickbench_q24/DuckDB:vortex-file-compressed     |     193000000 |     1.9e+08     |          1.01579  | ns     |
| clickbench_q25/DuckDB:vortex-file-compressed     |     149000000 |     1.38e+08    |          1.07971  | ns     |
| clickbench_q26/DuckDB:vortex-file-compressed     |     194000000 |     1.79e+08    |          1.0838   | ns     |
| clickbench_q27/DuckDB:vortex-file-compressed     |     631000000 |     6.24e+08    |          1.01122  | ns     |
| clickbench_q28/DuckDB:vortex-file-compressed     |    6771000000 |     6.448e+09   |          1.05009  | ns     |
| clickbench_q29/DuckDB:vortex-file-compressed     |     102000000 |     1.03e+08    |          0.990291 | ns     |
| clickbench_q30/DuckDB:vortex-file-compressed     |     289000000 |     2.76e+08    |          1.0471   | ns     |
| clickbench_q31/DuckDB:vortex-file-compressed     |     391000000 |     3.73e+08    |          1.04826  | ns     |
| clickbench_q32/DuckDB:vortex-file-compressed     |    1285000000 |     1.241e+09   |          1.03546  | ns     |
| clickbench_q33/DuckDB:vortex-file-compressed     |    1250000000 |     1.227e+09   |          1.01874  | ns     |
| clickbench_q34/DuckDB:vortex-file-compressed     |    1404000000 |     1.351e+09   |          1.03923  | ns     |
| clickbench_q35/DuckDB:vortex-file-compressed     |     398000000 |     3.91e+08    |          1.0179   | ns     |
| clickbench_q36/DuckDB:vortex-file-compressed     |      47000000 |     7.3e+07     |          0.643836 | ns     |
| clickbench_q37/DuckDB:vortex-file-compressed     |      43000000 |     6.2e+07     |          0.693548 | ns     |
| clickbench_q38/DuckDB:vortex-file-compressed     |      44000000 |     7.3e+07     |          0.60274  | ns     |
| clickbench_q39/DuckDB:vortex-file-compressed     |      80000000 |     1.18e+08    |          0.677966 | ns     |
| clickbench_q40/DuckDB:vortex-file-compressed     |      35000000 |     4.4e+07     |          0.795455 | ns     |
| clickbench_q41/DuckDB:vortex-file-compressed     |      39000000 |     4.7e+07     |          0.829787 | ns     |
| clickbench_q42/DuckDB:vortex-file-compressed     |      37000000 |     4e+07       |          0.925    | ns     |
| clickbench_q00/DuckDB:duckdb                     |      16000000 |     1.6e+07     |          1        | ns     |
| clickbench_q01/DuckDB:duckdb                     |      34000000 |     3.4e+07     |          1        | ns     |
| clickbench_q02/DuckDB:duckdb                     |      67000000 |     6.5e+07     |          1.03077  | ns     |
| clickbench_q03/DuckDB:duckdb                     |      60000000 |     5.9e+07     |          1.01695  | ns     |
| clickbench_q04/DuckDB:duckdb                     |     209000000 |     2.02e+08    |          1.03465  | ns     |
| clickbench_q05/DuckDB:duckdb                     |     271000000 |     2.66e+08    |          1.0188   | ns     |
| clickbench_q06/DuckDB:duckdb                     |      36000000 |     3.7e+07     |          0.972973 | ns     |
| clickbench_q07/DuckDB:duckdb                     |      35000000 |     3.4e+07     |          1.02941  | ns     |
| clickbench_q08/DuckDB:duckdb                     |     251000000 |     2.43e+08    |          1.03292  | ns     |
| clickbench_q09/DuckDB:duckdb                     |     341000000 |     3.35e+08    |          1.01791  | ns     |
| clickbench_q10/DuckDB:duckdb                     |     113000000 |     1.1e+08     |          1.02727  | ns     |
| clickbench_q11/DuckDB:duckdb                     |     127000000 |     1.22e+08    |          1.04098  | ns     |
| clickbench_q12/DuckDB:duckdb                     |     254000000 |     2.45e+08    |          1.03673  | ns     |
| clickbench_q13/DuckDB:duckdb                     |     437000000 |     4.22e+08    |          1.03555  | ns     |
| clickbench_q14/DuckDB:duckdb                     |     278000000 |     2.73e+08    |          1.01832  | ns     |
| clickbench_q15/DuckDB:duckdb                     |     227000000 |     2.23e+08    |          1.01794  | ns     |
| clickbench_q16/DuckDB:duckdb                     |     568000000 |     5.46e+08    |          1.04029  | ns     |
| clickbench_q17/DuckDB:duckdb                     |     491000000 |     4.78e+08    |          1.0272   | ns     |
| clickbench_q18/DuckDB:duckdb                     |    1017000000 |     9.89e+08    |          1.02831  | ns     |
| clickbench_q19/DuckDB:duckdb                     |      32000000 |     3.1e+07     |          1.03226  | ns     |
| clickbench_q20/DuckDB:duckdb                     |     589000000 |     5.68e+08    |          1.03697  | ns     |
| clickbench_q21/DuckDB:duckdb                     |     480000000 |     4.57e+08    |          1.05033  | ns     |
| clickbench_q22/DuckDB:duckdb                     |     736000000 |     7.01e+08    |          1.04993  | ns     |
| clickbench_q23/DuckDB:duckdb                     |     299000000 |     2.86e+08    |          1.04545  | ns     |
| clickbench_q24/DuckDB:duckdb                     |      55000000 |     5.3e+07     |          1.03774  | ns     |
| clickbench_q25/DuckDB:duckdb                     |     101000000 |     9.8e+07     |          1.03061  | ns     |
| clickbench_q26/DuckDB:duckdb                     |      53000000 |     5.1e+07     |          1.03922  | ns     |
| clickbench_q27/DuckDB:duckdb                     |     539000000 |     5.26e+08    |          1.02471  | ns     |
| clickbench_q28/DuckDB:duckdb                     |    6002000000 |     5.951e+09   |          1.00857  | ns     |
| clickbench_q29/DuckDB:duckdb                     |      56000000 |     5.4e+07     |          1.03704  | ns     |
| clickbench_q30/DuckDB:duckdb                     |     278000000 |     2.67e+08    |          1.0412   | ns     |
| clickbench_q31/DuckDB:duckdb                     |     369000000 |     3.59e+08    |          1.02786  | ns     |
| clickbench_q32/DuckDB:duckdb                     |    1084000000 |     1.051e+09   |          1.0314   | ns     |
| clickbench_q33/DuckDB:duckdb                     |    1290000000 |     1.232e+09   |          1.04708  | ns     |
| clickbench_q34/DuckDB:duckdb                     |    1320000000 |     1.254e+09   |          1.05263  | ns     |
| clickbench_q35/DuckDB:duckdb                     |     268000000 |     2.57e+08    |          1.0428   | ns     |
| clickbench_q36/DuckDB:duckdb                     |      47000000 |     4.6e+07     |          1.02174  | ns     |
| clickbench_q37/DuckDB:duckdb                     |      31000000 |     3.4e+07     |          0.911765 | ns     |
| clickbench_q38/DuckDB:duckdb                     |      31000000 |     3.1e+07     |          1        | ns     |
| clickbench_q39/DuckDB:duckdb                     |      68000000 |     6.7e+07     |          1.01493  | ns     |
| clickbench_q40/DuckDB:duckdb                     |      30000000 |     3e+07       |          1        | ns     |
| clickbench_q41/DuckDB:duckdb                     |      31000000 |     3e+07       |          1.03333  | ns     |
| clickbench_q42/DuckDB:duckdb                     |      29000000 |     2.9e+07     |          1        | ns     |
</details>

<!-- thollander/actions-comment-pull-request "bench-pr-comment-clickbench-nvme" -->
author_association: CONTRIBUTOR