Repository: spiraldb/vortex
Pull request #2720: Use more stats in the btrblocks compressor
Author: login: AdamGS
Created: 2025-03-18T16:10:30Z | Merged: 2025-03-18T16:43:14Z

taken out of #2718 


Diff:
diff --git a/vortex-btrblocks/src/float/stats.rs b/vortex-btrblocks/src/float/stats.rs
index a15e5239764..1ef63fe750e 100644
--- a/vortex-btrblocks/src/float/stats.rs
+++ b/vortex-btrblocks/src/float/stats.rs
@@ -135,11 +135,12 @@ where
         };
     }
 
-    let validity = array.validity_mask().vortex_expect("logical_validity");
-    let null_count = validity.false_count();
-    let value_count = validity.true_count();
-    let mut min = T::max_value();
-    let mut max = T::min_value();
+    let null_count = array
+        .statistics()
+        .compute_null_count()
+        .vortex_expect("null count");
+    let value_count = array.len() - null_count;
+
     // Keep a HashMap of T, then convert the keys into PValue afterward since value is
     // so much more efficient to hash and search for.
     let mut distinct_values = if count_distinct_values {
@@ -148,6 +149,8 @@ where
         HashMap::with_hasher(FxBuildHasher)
     };
 
+    let validity = array.validity_mask().vortex_expect("logical_validity");
+
     let mut runs = 1;
     let head_idx = validity
         .first()
@@ -159,9 +162,6 @@ where
     match validity.boolean_buffer() {
         AllOr::All => {
             for value in first_valid_buff {
-                min = min.min(value);
-                max = max.max(value);
-
                 if count_distinct_values {
                     *distinct_values.entry(value.to_bits()).or_insert(0) += 1;
                 }
@@ -179,9 +179,6 @@ where
                 .zip_eq(v.slice(head_idx, array.len() - head_idx).iter())
             {
                 if valid {
-                    min = min.min(value);
-                    max = max.max(value);
-
                     if count_distinct_values {
                         *distinct_values.entry(value.to_bits()).or_insert(0) += 1;
                     }
diff --git a/vortex-btrblocks/src/integer/stats.rs b/vortex-btrblocks/src/integer/stats.rs
index 409ea1f37d0..e54a5a0716b 100644
--- a/vortex-btrblocks/src/integer/stats.rs
+++ b/vortex-btrblocks/src/integer/stats.rs
@@ -5,12 +5,13 @@ use num_traits::PrimInt;
 use rustc_hash::FxBuildHasher;
 use vortex_array::aliases::hash_map::HashMap;
 use vortex_array::arrays::PrimitiveArray;
+use vortex_array::stats::Stat;
 use vortex_array::variants::PrimitiveArrayTrait;
 use vortex_array::{Array, ToCanonical};
 use vortex_dtype::{NativePType, match_each_integer_ptype};
-use vortex_error::{VortexExpect, VortexUnwrap};
+use vortex_error::{VortexError, VortexExpect, VortexUnwrap};
 use vortex_mask::AllOr;
-use vortex_scalar::PValue;
+use vortex_scalar::{PValue, ScalarValue};
 
 use crate::sample::sample;
 use crate::{CompressorStats, GenerateStatsOptions};
@@ -150,11 +151,9 @@ impl CompressorStats for IntegerStats {
     }
 }
 
-fn typed_int_stats<T: NativePType + Hash + PrimInt>(
-    array: &PrimitiveArray,
-    count_distinct_values: bool,
-) -> IntegerStats
+fn typed_int_stats<T>(array: &PrimitiveArray, count_distinct_values: bool) -> IntegerStats
 where
+    T: NativePType + Hash + PrimInt + for<'a> TryFrom<&'a ScalarValue, Error = VortexError>,
     TypedStats<T>: Into<ErasedStats>,
 {
     // Special case: empty array
@@ -204,8 +203,6 @@ where
     let head = buffer[head_idx];
 
     let mut loop_state = LoopState {
-        min: head,
-        max: head,
         distinct_values: if count_distinct_values {
             HashMap::with_capacity_and_hasher(array.len() / 2, FxBuildHasher)
         } else {
@@ -281,9 +278,19 @@ where
         u32::MAX
     };
 
+    let min = array
+        .statistics()
+        .compute_as::<T>(Stat::Min)
+        .vortex_expect("min should be computed");
+
+    let max = array
+        .statistics()
+        .compute_as::<T>(Stat::Max)
+        .vortex_expect("max should be computed");
+
     let typed = TypedStats {
-        min: loop_state.min,
-        max: loop_state.max,
+        min,
+        max,
         distinct_values: loop_state.distinct_values,
         top_value,
         top_count,
@@ -307,8 +314,6 @@ where
 }
 
 struct LoopState<T> {
-    min: T,
-    max: T,
     prev: T,
     runs: u32,
     distinct_values: HashMap<T, u32, FxBuildHasher>,
@@ -321,9 +326,6 @@ fn inner_loop_nonnull<T: PrimInt + Hash>(
     state: &mut LoopState<T>,
 ) {
     for &value in values {
-        state.min = state.min.min(value);
-        state.max = state.max.max(value);
-
         if count_distinct_values {
             *state.distinct_values.entry(value).or_insert(0) += 1;
         }
@@ -344,9 +346,6 @@ fn inner_loop_nullable<T: PrimInt + Hash>(
 ) {
     for (idx, &value) in values.iter().enumerate() {
         if is_valid.value(idx) {
-            state.min = state.min.min(value);
-            state.max = state.max.max(value);
-
             if count_distinct_values {
                 *state.distinct_values.entry(value).or_insert(0) += 1;
             }
@@ -368,9 +367,6 @@ fn inner_loop_naive<T: PrimInt + Hash>(
 ) {
     for (idx, &value) in values.iter().enumerate() {
         if is_valid.value(idx) {
-            state.min = state.min.min(value);
-            state.max = state.max.max(value);
-
             if count_distinct_values {
                 *state.distinct_values.entry(value).or_insert(0) += 1;
             }
diff --git a/vortex-btrblocks/src/string.rs b/vortex-btrblocks/src/string.rs
index 3c90c23e78c..d41b655c270 100644
--- a/vortex-btrblocks/src/string.rs
+++ b/vortex-btrblocks/src/string.rs
@@ -29,15 +29,15 @@ fn estimate_distinct_count(strings: &VarBinViewArray) -> u32 {
     // Iterate the views. Two strings which are equal must have the same first 8-bytes.
     // NOTE: there are cases where this performs pessimally, e.g. when we have strings that all
     // share a 4-byte prefix and have the same length.
-    let mut disinct = HashSet::with_capacity(views.len() / 2);
+    let mut distinct = HashSet::with_capacity(views.len() / 2);
     views.iter().for_each(|&view| {
         // SAFETY: we're doing bitwise manipulations. Did you expect that to be safe??
         let view_u128: u128 = unsafe { std::mem::transmute(view) };
         let len_and_prefix = view_u128 as u64;
-        disinct.insert(len_and_prefix);
+        distinct.insert(len_and_prefix);
     });
 
-    disinct
+    distinct
         .len()
         .try_into()
         .vortex_expect("distinct count must fit in u32")
@@ -48,9 +48,9 @@ impl CompressorStats for StringStats {
 
     fn generate_opts(input: &Self::ArrayType, opts: GenerateStatsOptions) -> Self {
         let null_count = input
-            .validity()
-            .null_count(input.len())
-            .vortex_expect("null_count");
+            .statistics()
+            .compute_null_count()
+            .vortex_expect("null count");
         let value_count = input.len() - null_count;
         let estimated_distinct = if opts.count_distinct_values {
             estimate_distinct_count(input)


Reviews:
user: login: AdamGS
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T16:11:54Z
user: login: a10y
body: curious how the benches shake out
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T16:15:25Z
user: login: a10y
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T16:17:07Z

Review comments:
diff_hunk: @@ -135,11 +135,12 @@ where
         };
     }
 
-    let validity = array.validity_mask().vortex_expect("logical_validity");
-    let null_count = validity.false_count();
-    let value_count = validity.true_count();
-    let mut min = T::max_value();
-    let mut max = T::min_value();
path: vortex-btrblocks/src/float/stats.rs
user: login: AdamGS
body: doesn't seem like we actually do anything with min/max here, we can always add them later if that makes sense.
created_at: 2025-03-18T16:11:54Z
author_association: CONTRIBUTOR
diff_hunk: @@ -135,11 +135,12 @@ where
         };
     }
 
-    let validity = array.validity_mask().vortex_expect("logical_validity");
-    let null_count = validity.false_count();
-    let value_count = validity.true_count();
-    let mut min = T::max_value();
-    let mut max = T::min_value();
path: vortex-btrblocks/src/float/stats.rs
user: login: a10y
body: oh lol, i guess clippy did't notice these b/c they are read/written even though they aren't returned at the end
created_at: 2025-03-18T16:17:07Z
author_association: CONTRIBUTOR

PR comments:
user: login: github-actions[bot]
created_at: 2025-03-18T16:14:52Z
body: # Benchmarks: random_access
<details>
<summary>Table of Results</summary>

| name                                   |   PR 1307a6c4 |   base 52ffd0f5 |   ratio (PR/base) | unit   |
|:---------------------------------------|--------------:|----------------:|------------------:|:-------|
| random-access/vortex-tokio-local-disk  |       2322280 |     2.32731e+06 |           0.99784 | ns     |
| random-access/parquet-tokio-local-disk |     241355175 |     2.39812e+08 |           1.00643 | ns     |
</details>

<!-- thollander/actions-comment-pull-request "bench-pr-comment-random_access" -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-18T16:17:01Z
body: # Benchmarks: TPC-H on NVME
<details>
<summary>Table of Results</summary>

| name                            |   PR 1307a6c4 |   base 52ffd0f5 |   ratio (PR/base) | unit   |
|:--------------------------------|--------------:|----------------:|------------------:|:-------|
| tpch_q01/arrow                  |      48093051 |     4.46814e+07 |          1.07635  | ns     |
| tpch_q02/arrow                  |      49534838 |     4.84245e+07 |          1.02293  | ns     |
| tpch_q03/arrow                  |      29901999 |     3.11299e+07 |          0.960554 | ns     |
| tpch_q04/arrow                  |      21938733 |     2.30994e+07 |          0.949754 | ns     |
| tpch_q05/arrow                  |      46619613 |     4.52773e+07 |          1.02965  | ns     |
| tpch_q06/arrow                  |       9927307 |     1.02166e+07 |          0.971685 | ns     |
| tpch_q07/arrow                  |      75350131 |     7.28126e+07 |          1.03485  | ns     |
| tpch_q08/arrow                  |      57854573 |     5.54558e+07 |          1.04326  | ns     |
| tpch_q09/arrow                  |      70865870 |     7.02875e+07 |          1.00823  | ns     |
| tpch_q10/arrow                  |      46433341 |     4.48012e+07 |          1.03643  | ns     |
| tpch_q11/arrow                  |      24440986 |     2.28448e+07 |          1.06987  | ns     |
| tpch_q12/arrow                  |      31304007 |     2.74316e+07 |          1.14117  | ns     |
| tpch_q13/arrow                  |      16938851 |     1.56398e+07 |          1.08306  | ns     |
| tpch_q14/arrow                  |      15117004 |     1.43323e+07 |          1.05475  | ns     |
| tpch_q15/arrow                  |      28770491 |     2.58534e+07 |          1.11283  | ns     |
| tpch_q16/arrow                  |      22583364 |     2.11587e+07 |          1.06733  | ns     |
| tpch_q17/arrow                  |      60227542 |     6.13106e+07 |          0.982334 | ns     |
| tpch_q18/arrow                  |      95920679 |     9.55493e+07 |          1.00389  | ns     |
| tpch_q19/arrow                  |      26573794 |     2.54527e+07 |          1.04404  | ns     |
| tpch_q20/arrow                  |      32770375 |     3.36356e+07 |          0.974277 | ns     |
| tpch_q21/arrow                  |     113893600 |     1.12475e+08 |          1.01261  | ns     |
| tpch_q22/arrow                  |      14737051 |     1.49697e+07 |          0.98446  | ns     |
| tpch_q01/parquet                |     122169971 |     1.13764e+08 |          1.07389  | ns     |
| tpch_q02/parquet                |     117730227 |     1.09847e+08 |          1.07177  | ns     |
| tpch_q03/parquet                |     105625005 |     1.05365e+08 |          1.00247  | ns     |
| tpch_q04/parquet                |      62172495 |     6.02828e+07 |          1.03135  | ns     |
| tpch_q05/parquet                |     116639824 |     1.15348e+08 |          1.0112   | ns     |
| tpch_q06/parquet                |      25910181 |     2.50284e+07 |          1.03523  | ns     |
| tpch_q07/parquet                |     134678129 |     1.32836e+08 |          1.01387  | ns     |
| tpch_q08/parquet                |     162015530 |     1.61098e+08 |          1.0057   | ns     |
| tpch_q09/parquet                |     211930603 |     2.05456e+08 |          1.03152  | ns     |
| tpch_q10/parquet                |     130232454 |     1.27759e+08 |          1.01936  | ns     |
| tpch_q11/parquet                |      54962096 |     5.1357e+07  |          1.0702   | ns     |
| tpch_q12/parquet                |      92521467 |     9.37016e+07 |          0.987406 | ns     |
| tpch_q13/parquet                |     150427824 |     1.48133e+08 |          1.01549  | ns     |
| tpch_q14/parquet                |      46616025 |     4.53869e+07 |          1.02708  | ns     |
| tpch_q15/parquet                |      64315830 |     7.22771e+07 |          0.88985  | ns     |
| tpch_q16/parquet                |      52191444 |     5.07217e+07 |          1.02898  | ns     |
| tpch_q17/parquet                |     128484587 |     1.25451e+08 |          1.02418  | ns     |
| tpch_q18/parquet                |     189691138 |     1.87789e+08 |          1.01013  | ns     |
| tpch_q19/parquet                |      77568952 |     7.42834e+07 |          1.04423  | ns     |
| tpch_q20/parquet                |      96258731 |     9.49261e+07 |          1.01404  | ns     |
| tpch_q21/parquet                |     184662437 |     1.79022e+08 |          1.03151  | ns     |
| tpch_q22/parquet                |      50958003 |     5.01829e+07 |          1.01544  | ns     |
| tpch_q01/vortex-file-compressed |      35845000 |     3.61495e+07 |          0.991576 | ns     |
| tpch_q02/vortex-file-compressed |      57289989 |     5.62555e+07 |          1.01839  | ns     |
| tpch_q03/vortex-file-compressed |      28705943 |     2.9505e+07  |          0.97292  | ns     |
| tpch_q04/vortex-file-compressed |      18345169 |     1.86353e+07 |          0.984429 | ns     |
| tpch_q05/vortex-file-compressed |      44396215 |     4.47925e+07 |          0.991153 | ns     |
| tpch_q06/vortex-file-compressed |       9575152 |     1.04221e+07 |          0.918732 | ns     |
| tpch_q07/vortex-file-compressed |      69188702 |     6.80985e+07 |          1.01601  | ns     |
| tpch_q08/vortex-file-compressed |      51275491 |     5.30406e+07 |          0.966721 | ns     |
| tpch_q09/vortex-file-compressed |      68538573 |     6.50258e+07 |          1.05402  | ns     |
| tpch_q10/vortex-file-compressed |      51207143 |     4.92382e+07 |          1.03999  | ns     |
| tpch_q11/vortex-file-compressed |      25603583 |     2.5436e+07  |          1.00659  | ns     |
| tpch_q12/vortex-file-compressed |      24569262 |     2.3089e+07  |          1.06411  | ns     |
| tpch_q13/vortex-file-compressed |      24102928 |     2.41468e+07 |          0.998185 | ns     |
| tpch_q14/vortex-file-compressed |      12772000 |     1.29248e+07 |          0.988177 | ns     |
| tpch_q15/vortex-file-compressed |      28613239 |     2.82769e+07 |          1.01189  | ns     |
| tpch_q16/vortex-file-compressed |      27061060 |     2.67439e+07 |          1.01186  | ns     |
| tpch_q17/vortex-file-compressed |      55945931 |     5.48604e+07 |          1.01979  | ns     |
| tpch_q18/vortex-file-compressed |      87797959 |     8.6229e+07  |          1.0182   | ns     |
| tpch_q19/vortex-file-compressed |      31777624 |     2.91089e+07 |          1.09168  | ns     |
| tpch_q20/vortex-file-compressed |      35494111 |     3.63149e+07 |          0.977399 | ns     |
| tpch_q21/vortex-file-compressed |      88324996 |     8.50194e+07 |          1.03888  | ns     |
| tpch_q22/vortex-file-compressed |      27461331 |     2.71091e+07 |          1.01299  | ns     |
</details>

<!-- thollander/actions-comment-pull-request "bench-pr-comment-tpch-nvme" -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-18T16:17:23Z
body: # Benchmarks: TPC-H on S3
<details>
<summary>Table of Results</summary>

| name                            |   PR 1307a6c4 |   base 52ffd0f5 |   ratio (PR/base) | unit   |
|:--------------------------------|--------------:|----------------:|------------------:|:-------|
| tpch_q01/parquet                |     273773421 |     2.60689e+08 |          1.05019  | ns     |
| tpch_q02/parquet                |     663524248 |     6.82055e+08 |          0.972831 | ns     |
| tpch_q03/parquet                |     416427276 |     4.26928e+08 |          0.975404 | ns     |
| tpch_q04/parquet                |     259988048 |     2.23133e+08 |          1.16517  | ns     |
| tpch_q05/parquet                |     584320217 |     5.46456e+08 |          1.06929  | ns     |
| tpch_q06/parquet                |     177904486 |     1.82879e+08 |          0.972801 | ns     |
| tpch_q07/parquet                |     618310785 |     6.31831e+08 |          0.978602 | ns     |
| tpch_q08/parquet                |     779887745 |     7.94967e+08 |          0.981032 | ns     |
| tpch_q09/parquet                |     709456373 |     6.93489e+08 |          1.02302  | ns     |
| tpch_q10/parquet                |     550541321 |     5.30582e+08 |          1.03762  | ns     |
| tpch_q11/parquet                |     286771740 |     2.75932e+08 |          1.03929  | ns     |
| tpch_q12/parquet                |     273724639 |     2.72691e+08 |          1.00379  | ns     |
| tpch_q13/parquet                |     395781956 |     3.92676e+08 |          1.00791  | ns     |
| tpch_q14/parquet                |     254955341 |     2.4511e+08  |          1.04017  | ns     |
| tpch_q15/parquet                |     465620092 |     4.73196e+08 |          0.983991 | ns     |
| tpch_q16/parquet                |     266593602 |     2.56881e+08 |          1.03781  | ns     |
| tpch_q17/parquet                |     379313174 |     4.0558e+08  |          0.935235 | ns     |
| tpch_q18/parquet                |     543057339 |     5.35003e+08 |          1.01506  | ns     |
| tpch_q19/parquet                |     282580175 |     2.91058e+08 |          0.970871 | ns     |
| tpch_q20/parquet                |     503300616 |     5.14578e+08 |          0.978084 | ns     |
| tpch_q21/parquet                |     633522242 |     6.39666e+08 |          0.990396 | ns     |
| tpch_q22/parquet                |     269929905 |     2.71647e+08 |          0.99368  | ns     |
| tpch_q01/vortex-file-compressed |     128757148 |     1.29922e+08 |          0.991037 | ns     |
| tpch_q02/vortex-file-compressed |     323680856 |     3.26943e+08 |          0.990021 | ns     |
| tpch_q03/vortex-file-compressed |     202185075 |     2.03435e+08 |          0.993858 | ns     |
| tpch_q04/vortex-file-compressed |     118873314 |     1.23286e+08 |          0.964208 | ns     |
| tpch_q05/vortex-file-compressed |     272065809 |     2.69602e+08 |          1.00914  | ns     |
| tpch_q06/vortex-file-compressed |     101018194 |     1.02045e+08 |          0.989933 | ns     |
| tpch_q07/vortex-file-compressed |     303762359 |     3.13608e+08 |          0.968604 | ns     |
| tpch_q08/vortex-file-compressed |     363166824 |     3.67525e+08 |          0.988143 | ns     |
| tpch_q09/vortex-file-compressed |     395144746 |     3.86274e+08 |          1.02296  | ns     |
| tpch_q10/vortex-file-compressed |     336726300 |     3.35496e+08 |          1.00367  | ns     |
| tpch_q11/vortex-file-compressed |     135497835 |     1.31871e+08 |          1.0275   | ns     |
| tpch_q12/vortex-file-compressed |     162685090 |     1.59565e+08 |          1.01955  | ns     |
| tpch_q13/vortex-file-compressed |     210651479 |     2.0574e+08  |          1.02387  | ns     |
| tpch_q14/vortex-file-compressed |     127985800 |     1.16412e+08 |          1.09942  | ns     |
| tpch_q15/vortex-file-compressed |     278226317 |     2.78408e+08 |          0.999347 | ns     |
| tpch_q16/vortex-file-compressed |     112061534 |     1.16494e+08 |          0.961951 | ns     |
| tpch_q17/vortex-file-compressed |     163890471 |     1.73988e+08 |          0.941964 | ns     |
| tpch_q18/vortex-file-compressed |     282992070 |     2.86099e+08 |          0.989139 | ns     |
| tpch_q19/vortex-file-compressed |     154121099 |     1.61313e+08 |          0.955418 | ns     |
| tpch_q20/vortex-file-compressed |     225389191 |     2.39078e+08 |          0.942742 | ns     |
| tpch_q21/vortex-file-compressed |     338921870 |     3.401e+08   |          0.996537 | ns     |
| tpch_q22/vortex-file-compressed |     121895863 |     1.19569e+08 |          1.01946  | ns     |
</details>

<!-- thollander/actions-comment-pull-request "bench-pr-comment-tpch-s3" -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-18T16:20:05Z
body: # Benchmarks: Clickbench on NVME
<details>
<summary>Table of Results</summary>

| name                                  |   PR 1307a6c4 |   base 52ffd0f5 |   ratio (PR/base) | unit   |
|:--------------------------------------|--------------:|----------------:|------------------:|:-------|
| clickbench_q00/parquet                |       2333719 |     2.43575e+06 |          0.958111 | ns     |
| clickbench_q01/parquet                |      32898211 |     3.17666e+07 |          1.03562  | ns     |
| clickbench_q02/parquet                |      62525486 |     6.35815e+07 |          0.983391 | ns     |
| clickbench_q03/parquet                |      54800331 |     5.30454e+07 |          1.03308  | ns     |
| clickbench_q04/parquet                |     322967805 |     3.11979e+08 |          1.03522  | ns     |
| clickbench_q05/parquet                |     316589241 |     3.02659e+08 |          1.04602  | ns     |
| clickbench_q06/parquet                |       2195201 |     2.32281e+06 |          0.945062 | ns     |
| clickbench_q07/parquet                |      35095933 |     3.22624e+07 |          1.08783  | ns     |
| clickbench_q08/parquet                |     389692045 |     3.81858e+08 |          1.02052  | ns     |
| clickbench_q09/parquet                |     586727837 |     5.53271e+08 |          1.06047  | ns     |
| clickbench_q10/parquet                |     120084876 |     1.15459e+08 |          1.04006  | ns     |
| clickbench_q11/parquet                |     140447553 |     1.39294e+08 |          1.00828  | ns     |
| clickbench_q12/parquet                |     323185837 |     3.10141e+08 |          1.04206  | ns     |
| clickbench_q13/parquet                |     490839408 |     4.8002e+08  |          1.02254  | ns     |
| clickbench_q14/parquet                |     329974519 |     3.13413e+08 |          1.05284  | ns     |
| clickbench_q15/parquet                |     363729631 |     3.60111e+08 |          1.01005  | ns     |
| clickbench_q16/parquet                |     776171178 |     7.48906e+08 |          1.03641  | ns     |
| clickbench_q17/parquet                |     683196485 |     6.6603e+08  |          1.02577  | ns     |
| clickbench_q18/parquet                |    1560107080 |     1.53028e+09 |          1.01949  | ns     |
| clickbench_q19/parquet                |      41746804 |     4.16868e+07 |          1.00144  | ns     |
| clickbench_q20/parquet                |     584697647 |     5.68209e+08 |          1.02902  | ns     |
| clickbench_q21/parquet                |     644984918 |     6.10779e+08 |          1.056    | ns     |
| clickbench_q22/parquet                |     978325098 |     9.44681e+08 |          1.03561  | ns     |
| clickbench_q23/parquet                |    4027991524 |     3.80084e+09 |          1.05976  | ns     |
| clickbench_q24/parquet                |     196433312 |     1.90658e+08 |          1.03029  | ns     |
| clickbench_q25/parquet                |     171823525 |     1.68863e+08 |          1.01753  | ns     |
| clickbench_q26/parquet                |     226594407 |     2.18406e+08 |          1.03749  | ns     |
| clickbench_q27/parquet                |     759883797 |     7.46929e+08 |          1.01734  | ns     |
| clickbench_q28/parquet                |    4453651836 |     4.31961e+09 |          1.03103  | ns     |
| clickbench_q29/parquet                |     248101669 |     2.38335e+08 |          1.04098  | ns     |
| clickbench_q30/parquet                |     332435773 |     3.19303e+08 |          1.04113  | ns     |
| clickbench_q31/parquet                |     384222119 |     3.62454e+08 |          1.06006  | ns     |
| clickbench_q32/parquet                |    1781312596 |     1.70047e+09 |          1.04754  | ns     |
| clickbench_q33/parquet                |    1568620149 |     1.49082e+09 |          1.05219  | ns     |
| clickbench_q34/parquet                |    1518173065 |     1.46736e+09 |          1.03463  | ns     |
| clickbench_q35/parquet                |     510638238 |     5.1352e+08  |          0.994388 | ns     |
| clickbench_q36/parquet                |     153132592 |     1.52567e+08 |          1.00371  | ns     |
| clickbench_q37/parquet                |      68612234 |     6.67187e+07 |          1.02838  | ns     |
| clickbench_q38/parquet                |      94969514 |     9.56535e+07 |          0.99285  | ns     |
| clickbench_q39/parquet                |     286015200 |     2.82045e+08 |          1.01408  | ns     |
| clickbench_q40/parquet                |      47023438 |     4.5531e+07  |          1.03278  | ns     |
| clickbench_q41/parquet                |      42166866 |     4.13035e+07 |          1.0209   | ns     |
| clickbench_q42/parquet                |      53580092 |     5.25193e+07 |          1.0202   | ns     |
| clickbench_q00/vortex-file-compressed |       4538754 |     4.75252e+06 |          0.95502  | ns     |
| clickbench_q01/vortex-file-compressed |      23067337 |     2.19759e+07 |          1.04967  | ns     |
| clickbench_q02/vortex-file-compressed |      37061247 |     3.4342e+07  |          1.07918  | ns     |
| clickbench_q03/vortex-file-compressed |      43965393 |     4.43854e+07 |          0.990537 | ns     |
| clickbench_q04/vortex-file-compressed |     304446505 |     3.46063e+08 |          0.879743 | ns     |
| clickbench_q05/vortex-file-compressed |     356074111 |     3.39787e+08 |          1.04793  | ns     |
| clickbench_q06/vortex-file-compressed |       4359200 |     4.71819e+06 |          0.923914 | ns     |
| clickbench_q07/vortex-file-compressed |      24061059 |     2.6428e+07  |          0.910437 | ns     |
| clickbench_q08/vortex-file-compressed |     387210233 |     3.94551e+08 |          0.981395 | ns     |
| clickbench_q09/vortex-file-compressed |     490230255 |     5.00034e+08 |          0.980394 | ns     |
| clickbench_q10/vortex-file-compressed |      70619867 |     6.6119e+07  |          1.06807  | ns     |
| clickbench_q11/vortex-file-compressed |      72525632 |     7.47194e+07 |          0.97064  | ns     |
| clickbench_q12/vortex-file-compressed |     280940008 |     2.56212e+08 |          1.09651  | ns     |
| clickbench_q13/vortex-file-compressed |     382609227 |     3.66294e+08 |          1.04454  | ns     |
| clickbench_q14/vortex-file-compressed |     300123169 |     2.53307e+08 |          1.18482  | ns     |
| clickbench_q15/vortex-file-compressed |     379556006 |     4.00358e+08 |          0.948041 | ns     |
| clickbench_q16/vortex-file-compressed |     786762975 |     7.70845e+08 |          1.02065  | ns     |
| clickbench_q17/vortex-file-compressed |     773189706 |     7.36247e+08 |          1.05018  | ns     |
| clickbench_q18/vortex-file-compressed |    1330008649 |     1.276e+09   |          1.04232  | ns     |
| clickbench_q19/vortex-file-compressed |      33031745 |     2.94367e+07 |          1.12213  | ns     |
| clickbench_q20/vortex-file-compressed |     246638601 |     2.42924e+08 |          1.01529  | ns     |
| clickbench_q21/vortex-file-compressed |     285516200 |     2.84195e+08 |          1.00465  | ns     |
| clickbench_q22/vortex-file-compressed |     477049899 |     4.71065e+08 |          1.0127   | ns     |
| clickbench_q23/vortex-file-compressed |     868011600 |     8.49909e+08 |          1.0213   | ns     |
| clickbench_q24/vortex-file-compressed |      91631418 |     8.76794e+07 |          1.04507  | ns     |
| clickbench_q25/vortex-file-compressed |     100387278 |     9.66809e+07 |          1.03834  | ns     |
| clickbench_q26/vortex-file-compressed |     128015705 |     1.17371e+08 |          1.0907   | ns     |
| clickbench_q27/vortex-file-compressed |     540530919 |     5.46966e+08 |          0.988235 | ns     |
| clickbench_q28/vortex-file-compressed |    5130807397 |     5.15553e+09 |          0.995205 | ns     |
| clickbench_q29/vortex-file-compressed |     255577340 |     2.54841e+08 |          1.00289  | ns     |
| clickbench_q30/vortex-file-compressed |     238660654 |     2.25965e+08 |          1.05618  | ns     |
| clickbench_q31/vortex-file-compressed |     247496526 |     2.38722e+08 |          1.03676  | ns     |
| clickbench_q32/vortex-file-compressed |    1355377889 |     1.31005e+09 |          1.0346   | ns     |
| clickbench_q33/vortex-file-compressed |    1265989732 |     1.24063e+09 |          1.02044  | ns     |
| clickbench_q34/vortex-file-compressed |    1261090271 |     1.23722e+09 |          1.01929  | ns     |
| clickbench_q35/vortex-file-compressed |     612002493 |     6.07438e+08 |          1.00751  | ns     |
| clickbench_q36/vortex-file-compressed |      97123448 |     9.79235e+07 |          0.99183  | ns     |
| clickbench_q37/vortex-file-compressed |      64289440 |     6.56227e+07 |          0.979682 | ns     |
| clickbench_q38/vortex-file-compressed |      67585923 |     6.31181e+07 |          1.07079  | ns     |
| clickbench_q39/vortex-file-compressed |     194334072 |     1.91633e+08 |          1.01409  | ns     |
| clickbench_q40/vortex-file-compressed |      46801388 |     4.39911e+07 |          1.06388  | ns     |
| clickbench_q41/vortex-file-compressed |      40068508 |     3.92548e+07 |          1.02073  | ns     |
| clickbench_q42/vortex-file-compressed |      34152787 |     3.40075e+07 |          1.00427  | ns     |
</details>

<!-- thollander/actions-comment-pull-request "bench-pr-comment-clickbench-nvme" -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-18T16:40:18Z
body: # Benchmarks: compress
<details>
<summary>Table of Results</summary>

| name                                                                                |      PR 1307a6c4 |    base 52ffd0f5 |   ratio (PR/base) | unit     |
|:------------------------------------------------------------------------------------|-----------------:|-----------------:|------------------:|:---------|
| compress time/taxi throughput                                                       |      0.28977     |      0.303679    |          0.9542   | bytes/ns |
| parquet_rs-zstd compress time/taxi throughput                                       |      0.34372     |      0.356934    |          0.96298  | bytes/ns |
| decompress time/taxi throughput                                                     |      2.10027     |      2.06597     |          1.01661  | bytes/ns |
| parquet_rs-zstd decompress time/taxi throughput                                     |      1.86702     |      1.92664     |          0.969057 | bytes/ns |
| compress time/AirlineSentiment throughput                                           |      0.00194361  |      0.00205034  |          0.947945 | bytes/ns |
| parquet_rs-zstd compress time/AirlineSentiment throughput                           |      0.058665    |      0.0593268   |          0.988845 | bytes/ns |
| decompress time/AirlineSentiment throughput                                         |      0.0120849   |      0.0144809   |          0.834538 | bytes/ns |
| parquet_rs-zstd decompress time/AirlineSentiment throughput                         |      0.0897573   |      0.0965844   |          0.929315 | bytes/ns |
| compress time/Arade throughput                                                      |      0.204613    |      0.216173    |          0.946522 | bytes/ns |
| parquet_rs-zstd compress time/Arade throughput                                      |      0.482239    |      0.510212    |          0.945174 | bytes/ns |
| decompress time/Arade throughput                                                    |      1.9681      |      1.98681     |          0.990584 | bytes/ns |
| parquet_rs-zstd decompress time/Arade throughput                                    |      1.94597     |      2.02081     |          0.962967 | bytes/ns |
| compress time/Bimbo throughput                                                      |      0.562881    |      0.56115     |          1.00308  | bytes/ns |
| parquet_rs-zstd compress time/Bimbo throughput                                      |      0.436747    |      0.446754    |          0.9776   | bytes/ns |
| decompress time/Bimbo throughput                                                    |      2.45703     |      2.54026     |          0.967234 | bytes/ns |
| parquet_rs-zstd decompress time/Bimbo throughput                                    |      3.09201     |      3.1358      |          0.986036 | bytes/ns |
| compress time/CMSprovider throughput                                                |      0.0636105   |      0.0649762   |          0.978982 | bytes/ns |
| parquet_rs-zstd compress time/CMSprovider throughput                                |      0.430166    |      0.440557    |          0.976414 | bytes/ns |
| decompress time/CMSprovider throughput                                              |      4.45016     |      4.42595     |          1.00547  | bytes/ns |
| parquet_rs-zstd decompress time/CMSprovider throughput                              |      2.11501     |      2.20427     |          0.959506 | bytes/ns |
| compress time/Euro2016 throughput                                                   |      0.179368    |      0.178472    |          1.00502  | bytes/ns |
| parquet_rs-zstd compress time/Euro2016 throughput                                   |      0.34807     |      0.354411    |          0.982107 | bytes/ns |
| decompress time/Euro2016 throughput                                                 |      2.77877     |      2.83009     |          0.981865 | bytes/ns |
| parquet_rs-zstd decompress time/Euro2016 throughput                                 |      1.18322     |      1.19567     |          0.989585 | bytes/ns |
| compress time/Food throughput                                                       |      0.247509    |      0.243918    |          1.01472  | bytes/ns |
| parquet_rs-zstd compress time/Food throughput                                       |      0.397274    |      0.402859    |          0.986137 | bytes/ns |
| decompress time/Food throughput                                                     |      6.26338     |      6.15577     |          1.01748  | bytes/ns |
| parquet_rs-zstd decompress time/Food throughput                                     |      1.97185     |      1.99576     |          0.98802  | bytes/ns |
| compress time/HashTags throughput                                                   |      0.222135    |      0.229509    |          0.967871 | bytes/ns |
| parquet_rs-zstd compress time/HashTags throughput                                   |      0.953027    |      0.977641    |          0.974824 | bytes/ns |
| decompress time/HashTags throughput                                                 |      6.6129      |      6.65832     |          0.993179 | bytes/ns |
| parquet_rs-zstd decompress time/HashTags throughput                                 |      3.14195     |      2.99268     |          1.04988  | bytes/ns |
| compress time/TPC-H l_comment chunked throughput                                    |      0.241142    |      0.242614    |          0.993933 | bytes/ns |
| parquet_rs-zstd compress time/TPC-H l_comment chunked throughput                    |      0.314596    |      0.319934    |          0.983316 | bytes/ns |
| decompress time/TPC-H l_comment chunked throughput                                  |      3.0955      |      3.17911     |          0.973701 | bytes/ns |
| parquet_rs-zstd decompress time/TPC-H l_comment chunked throughput                  |      1.42048     |      1.45457     |          0.976567 | bytes/ns |
| compress time/TPC-H l_comment canonical throughput                                  |      0.0322071   |      0.0330167   |          0.975479 | bytes/ns |
| parquet_rs-zstd compress time/TPC-H l_comment canonical throughput                  |      0.312486    |      0.321468    |          0.972059 | bytes/ns |
| decompress time/TPC-H l_comment canonical throughput                                |      3.10826     |      3.12409     |          0.994931 | bytes/ns |
| parquet_rs-zstd decompress time/TPC-H l_comment canonical throughput                |      1.41117     |      1.43433     |          0.983856 | bytes/ns |
| compress time/wide table cols=10 chunks=1 rows=1000 throughput                      |      0.151932    |      0.180703    |          0.840781 | bytes/ns |
| parquet_rs-zstd compress time/wide table cols=10 chunks=1 rows=1000 throughput      |      0.239116    |      0.250933    |          0.952908 | bytes/ns |
| decompress time/wide table cols=10 chunks=1 rows=1000 throughput                    |      0.738001    |      0.842198    |          0.876279 | bytes/ns |
| parquet_rs-zstd decompress time/wide table cols=10 chunks=1 rows=1000 throughput    |      0.527041    |      0.548646    |          0.960621 | bytes/ns |
| compress time/wide table cols=100 chunks=1 rows=1000 throughput                     |      0.147032    |      0.167914    |          0.875637 | bytes/ns |
| parquet_rs-zstd compress time/wide table cols=100 chunks=1 rows=1000 throughput     |      0.235355    |      0.234227    |          1.00482  | bytes/ns |
| decompress time/wide table cols=100 chunks=1 rows=1000 throughput                   |      1.1553      |      1.25173     |          0.922959 | bytes/ns |
| parquet_rs-zstd decompress time/wide table cols=100 chunks=1 rows=1000 throughput   |      0.519745    |      0.526801    |          0.986605 | bytes/ns |
| compress time/wide table cols=1000 chunks=1 rows=1000 throughput                    |      0.134542    |      0.151179    |          0.889952 | bytes/ns |
| parquet_rs-zstd compress time/wide table cols=1000 chunks=1 rows=1000 throughput    |      0.203829    |      0.21245     |          0.959418 | bytes/ns |
| decompress time/wide table cols=1000 chunks=1 rows=1000 throughput                  |      0.852982    |      0.83487     |          1.02169  | bytes/ns |
| parquet_rs-zstd decompress time/wide table cols=1000 chunks=1 rows=1000 throughput  |      0.489606    |      0.455062    |          1.07591  | bytes/ns |
| compress time/wide table cols=10 chunks=50 rows=1000 throughput                     |      0.0831325   |      0.0912903   |          0.910639 | bytes/ns |
| parquet_rs-zstd compress time/wide table cols=10 chunks=50 rows=1000 throughput     |      0.162566    |      0.171043    |          0.950443 | bytes/ns |
| decompress time/wide table cols=10 chunks=50 rows=1000 throughput                   |      0.816885    |      0.489394    |          1.66918  | bytes/ns |
| parquet_rs-zstd decompress time/wide table cols=10 chunks=50 rows=1000 throughput   |      0.546769    |      0.560763    |          0.975044 | bytes/ns |
| compress time/wide table cols=100 chunks=50 rows=1000 throughput                    |      0.0779438   |      0.0739018   |          1.05469  | bytes/ns |
| parquet_rs-zstd compress time/wide table cols=100 chunks=50 rows=1000 throughput    |      0.153728    |      0.146812    |          1.04711  | bytes/ns |
| decompress time/wide table cols=100 chunks=50 rows=1000 throughput                  |      1.21954     |      1.18685     |          1.02754  | bytes/ns |
| parquet_rs-zstd decompress time/wide table cols=100 chunks=50 rows=1000 throughput  |      0.542332    |      0.530223    |          1.02284  | bytes/ns |
| compress time/wide table cols=1000 chunks=50 rows=1000 throughput                   |      0.0622059   |      0.0643916   |          0.966057 | bytes/ns |
| parquet_rs-zstd compress time/wide table cols=1000 chunks=50 rows=1000 throughput   |      0.114661    |      0.105657    |          1.08522  | bytes/ns |
| decompress time/wide table cols=1000 chunks=50 rows=1000 throughput                 |      0.847686    |      0.846376    |          1.00155  | bytes/ns |
| parquet_rs-zstd decompress time/wide table cols=1000 chunks=50 rows=1000 throughput |      0.460834    |      0.436767    |          1.0551   | bytes/ns |
| vortex:raw size/taxi                                                                |      0.119085    |      0.119081    |          1.00003  | ratio    |
| vortex size/taxi                                                                    |      5.89239e+07 |      5.89219e+07 |          1.00003  | bytes    |
| vortex:parquet-zstd size/taxi                                                       |      1.05295     |      1.05291     |          1.00003  | ratio    |
| vortex:raw size/AirlineSentiment                                                    |      1.37906     |      1.34966     |          1.02178  | ratio    |
| vortex size/AirlineSentiment                                                        |   4504           |   4408           |          1.02178  | bytes    |
| vortex:parquet-zstd size/AirlineSentiment                                           |      4.6577      |      4.55843     |          1.02178  | ratio    |
| vortex:raw size/Arade                                                               |      0.255619    |      0.255616    |          1.00001  | ratio    |
| vortex size/Arade                                                                   |      3.03339e+08 |      3.03335e+08 |          1.00001  | bytes    |
| vortex:parquet-zstd size/Arade                                                      |      0.993272    |      0.993262    |          1.00001  | ratio    |
| vortex:raw size/Bimbo                                                               |      0.11651     |      0.116509    |          1.00001  | ratio    |
| vortex size/Bimbo                                                                   |      8.32948e+08 |      8.32939e+08 |          1.00001  | bytes    |
| vortex:parquet-zstd size/Bimbo                                                      |      2.14594     |      2.14592     |          1.00001  | ratio    |
| vortex:raw size/CMSprovider                                                         |      0.184562    |      0.184507    |          1.00029  | ratio    |
| vortex size/CMSprovider                                                             |      1.15999e+09 |      1.15965e+09 |          1.00029  | bytes    |
| vortex:parquet-zstd size/CMSprovider                                                |      1.50741     |      1.50697     |          1.00029  | ratio    |
| vortex:raw size/Euro2016                                                            |      0.399038    |      0.399036    |          1        | ratio    |
| vortex size/Euro2016                                                                |      1.81571e+08 |      1.81571e+08 |          1        | bytes    |
| vortex:parquet-zstd size/Euro2016                                                   |      1.52722     |      1.52722     |          1        | ratio    |
| vortex:raw size/Food                                                                |      0.177019    |      0.177001    |          1.0001   | ratio    |
| vortex size/Food                                                                    |      5.96275e+07 |      5.96217e+07 |          1.0001   | bytes    |
| vortex:parquet-zstd size/Food                                                       |      1.6458      |      1.64564     |          1.0001   | ratio    |
| vortex:raw size/HashTags                                                            |      0.139651    |      0.139622    |          1.0002   | ratio    |
| vortex size/HashTags                                                                |      2.67559e+08 |      2.67505e+08 |          1.0002   | bytes    |
| vortex:parquet-zstd size/HashTags                                                   |      1.99723     |      1.99682     |          1.0002   | ratio    |
| vortex:raw size/TPC-H l_comment chunked                                             |      0.41795     |      0.417522    |          1.00103  | ratio    |
| vortex size/TPC-H l_comment chunked                                                 |      1.04152e+08 |      1.04045e+08 |          1.00103  | bytes    |
| vortex:parquet-zstd size/TPC-H l_comment chunked                                    |      1.82932     |      1.8273      |          1.00111  | ratio    |
| vortex:raw size/TPC-H l_comment canonical                                           |      0.425034    |      0.425098    |          0.999848 | ratio    |
| vortex size/TPC-H l_comment canonical                                               |      1.05915e+08 |      1.05931e+08 |          0.999848 | bytes    |
| vortex:parquet-zstd size/TPC-H l_comment canonical                                  |      1.86013     |      1.86055     |          0.999774 | ratio    |
| vortex:raw size/wide table cols=10 chunks=1 rows=1000                               |      0.629222    |      0.625225    |          1.00639  | ratio    |
| vortex size/wide table cols=10 chunks=1 rows=1000                                   | 100736           | 100096           |          1.00639  | bytes    |
| vortex:parquet-zstd size/wide table cols=10 chunks=1 rows=1000                      |      1.07757     |      1.07073     |          1.00639  | ratio    |
| vortex:raw size/wide table cols=100 chunks=1 rows=1000                              |      0.626266    |      0.622268    |          1.00642  | ratio    |
| vortex size/wide table cols=100 chunks=1 rows=1000                                  |      1.00254e+06 | 996136           |          1.00642  | bytes    |
| vortex:parquet-zstd size/wide table cols=100 chunks=1 rows=1000                     |      1.07246     |      1.06561     |          1.00642  | ratio    |
| vortex:raw size/wide table cols=1000 chunks=1 rows=1000                             |      0.62597     |      0.621972    |          1.00643  | ratio    |
| vortex size/wide table cols=1000 chunks=1 rows=1000                                 |      1.00205e+07 |      9.95654e+06 |          1.00643  | bytes    |
| vortex:parquet-zstd size/wide table cols=1000 chunks=1 rows=1000                    |      1.07194     |      1.0651      |          1.00643  | ratio    |
| vortex:raw size/wide table cols=10 chunks=50 rows=1000                              |      0.61272     |      0.608827    |          1.00639  | ratio    |
| vortex size/wide table cols=10 chunks=50 rows=1000                                  | 100736           | 100096           |          1.00639  | bytes    |
| vortex:parquet-zstd size/wide table cols=10 chunks=50 rows=1000                     |      1.07757     |      1.07073     |          1.00639  | ratio    |
| vortex:raw size/wide table cols=100 chunks=50 rows=1000                             |      0.61115     |      0.607249    |          1.00642  | ratio    |
| vortex size/wide table cols=100 chunks=50 rows=1000                                 |      1.00254e+06 | 996136           |          1.00642  | bytes    |
| vortex:parquet-zstd size/wide table cols=100 chunks=50 rows=1000                    |      1.07246     |      1.06561     |          1.00642  | ratio    |
| vortex:raw size/wide table cols=1000 chunks=50 rows=1000                            |      0.610993    |      0.607091    |          1.00643  | ratio    |
| vortex size/wide table cols=1000 chunks=50 rows=1000                                |      1.00205e+07 |      9.95654e+06 |          1.00643  | bytes    |
| vortex:parquet-zstd size/wide table cols=1000 chunks=50 rows=1000                   |      1.07194     |      1.0651      |          1.00643  | ratio    |
</details>

<!-- thollander/actions-comment-pull-request "bench-pr-comment-compress" -->
author_association: CONTRIBUTOR
user: login: AdamGS
created_at: 2025-03-18T16:43:09Z
body: other than one variant that regressed (`decompress time/wide table cols=10 chunks=50 rows=1000 throughput`), seems like we get slightly better in many benchmarks, so I'm moving forward with the PR.
author_association: CONTRIBUTOR