Repository: dathere/qsv (language: rust)

Issue:
Refactor stats caching and eliminate binary format
When the stats cache was first implemented, we created a binary format of the stats in addition to the CSV cache, with the `--stats-binout` option.

This was an attempt to directly load the stats into memory, directly into the Stats data structure and eliminate parsing.

However, it seems the decompression/direct-load approach actually takes a long time. Even longer than parsing the CSV file!

For "automagical" commands that uses Stats caches, we should just parse the human-readable CSV version of the cache, and eliminate the binary format.
Refactor stats caching and eliminate binary format
When the stats cache was first implemented, we created a binary format of the stats in addition to the CSV cache, with the `--stats-binout` option.

This was an attempt to directly load the stats into memory, directly into the Stats data structure and eliminate parsing.

However, it seems the decompression/direct-load approach actually takes a long time. Even longer than parsing the CSV file!

For "automagical" commands that uses Stats caches, we should just parse the human-readable CSV version of the cache, and eliminate the binary format.


Discussion / hints:

Fix patch:
diff --git a/src/cmd/frequency.rs b/src/cmd/frequency.rs
index abc013535..6518685a4 100644
--- a/src/cmd/frequency.rs
+++ b/src/cmd/frequency.rs
@@ -596,8 +596,7 @@ impl Args {
             "_schema" => StatsMode::Schema, // only meant for internal use by schema command
             _ => return fail_incorrectusage_clierror!("Invalid stats mode"),
         };
-        let (csv_fields, csv_stats, stats_col_index_map) =
-            get_stats_records(&schema_args, stats_mode)?;
+        let (csv_fields, csv_stats) = get_stats_records(&schema_args, stats_mode)?;
 
         if stats_mode == StatsMode::None || stats_mode == StatsMode::Schema || csv_fields.is_empty()
         {
@@ -614,30 +613,24 @@ impl Args {
             csv_fields.len() == csv_stats.len(),
             "Mismatch between the number of fields and stats records"
         );
-        let col_cardinality_vec: Vec<(String, usize)> = csv_stats
+        let col_cardinality_vec: Vec<(String, u64)> = csv_stats
             .iter()
             .enumerate()
-            .map(|(i, _record)| {
+            .map(|(i, stats_record)| {
                 // get the column name and stats record
                 // safety: we know that csv_fields and csv_stats have the same length
                 let col_name = csv_fields.get(i).unwrap();
-                let stats_record = csv_stats.get(i).unwrap().clone().to_record(4, false);
-
-                let col_cardinality = match stats_record.get(stats_col_index_map["cardinality"]) {
-                    Some(s) => s.parse::<usize>().unwrap_or(0_usize),
-                    None => 0_usize,
-                };
                 (
                     simdutf8::basic::from_utf8(col_name)
                         .unwrap_or(NON_UTF8_ERR)
                         .to_string(),
-                    col_cardinality,
+                    stats_record.cardinality,
                 )
             })
             .collect();
 
         // now, get the unique headers, where cardinality == rowcount
-        let row_count = util::count_rows(&self.rconfig())? as usize;
+        let row_count = util::count_rows(&self.rconfig())?;
         FREQ_ROW_COUNT.set(row_count as u64).unwrap();
 
         let mut all_unique_headers_vec: Vec<usize> = Vec::with_capacity(5);
@@ -659,7 +652,9 @@ impl Args {
         let headers = rdr.byte_headers()?;
         let all_unique_headers_vec = self.get_unique_headers(headers)?;
 
-        UNIQUE_COLUMNS.set(all_unique_headers_vec).unwrap();
+        UNIQUE_COLUMNS
+            .set(all_unique_headers_vec)
+            .map_err(|_| "Cannot set UNIQUE_COLUMNS")?;
 
         let sel = self.rconfig().selection(headers)?;
         Ok((sel.select(headers).map(<[u8]>::to_vec).collect(), sel))
diff --git a/src/cmd/schema.rs b/src/cmd/schema.rs
index acad31ddc..8445cff37 100644
--- a/src/cmd/schema.rs
+++ b/src/cmd/schema.rs
@@ -88,7 +88,7 @@ use rayon::slice::ParallelSliceMut;
 use serde_json::{json, value::Number, Map, Value};
 use stats::Frequencies;
 
-use crate::{cmd::stats::Stats, config::Config, util, util::StatsMode, CliResult};
+use crate::{cmd::stats::StatsData, config::Config, util, util::StatsMode, CliResult};
 
 const STDIN_CSV: &str = "stdin.csv";
 
@@ -208,12 +208,11 @@ pub fn infer_schema_from_stats(
     input_filename: &str,
 ) -> CliResult<Map<String, Value>> {
     // invoke cmd::stats
-    let (csv_fields, csv_stats, stats_col_index_map) =
-        util::get_stats_records(args, StatsMode::Schema)?;
+    let (csv_fields, csv_stats) = util::get_stats_records(args, StatsMode::Schema)?;
 
     // amortize memory allocation
-    let mut low_cardinality_column_indices: Vec<usize> =
-        Vec::with_capacity(args.flag_enum_threshold);
+    let mut low_cardinality_column_indices: Vec<u64> =
+        Vec::with_capacity(args.flag_enum_threshold as usize);
 
     // build column selector arg to invoke cmd::frequency with
     let column_select_arg: String = build_low_cardinality_column_selector_arg(
@@ -221,7 +220,6 @@ pub fn infer_schema_from_stats(
         args.flag_enum_threshold,
         &csv_fields,
         &csv_stats,
-        &stats_col_index_map,
     );
 
     // invoke cmd::frequency to get unique values for each field
@@ -233,7 +231,7 @@ pub fn infer_schema_from_stats(
     // amortize memory allocations
     let mut field_map: Map<String, Value> = Map::with_capacity(10);
     let mut type_list: Vec<Value> = Vec::with_capacity(4);
-    let mut enum_list: Vec<Value> = Vec::with_capacity(args.flag_enum_threshold);
+    let mut enum_list: Vec<Value> = Vec::with_capacity(args.flag_enum_threshold as usize);
     let mut header_byte_slice;
     let mut header_string;
     let mut stats_record;
@@ -241,6 +239,7 @@ pub fn infer_schema_from_stats(
     let mut col_null_count;
 
     // generate definition for each CSV column/field and add to properties_map
+    #[allow(clippy::needless_range_loop)]
     for i in 0..csv_fields.len() {
         header_byte_slice = csv_fields.get(i).unwrap();
 
@@ -248,20 +247,17 @@ pub fn infer_schema_from_stats(
         header_string = convert_to_string(header_byte_slice)?;
 
         // grab stats record for current column
-        stats_record = csv_stats.get(i).unwrap().clone().to_record(4, false);
+        stats_record = csv_stats[i].clone();
 
         if log::log_enabled!(log::Level::Debug) {
             debug!("stats[{header_string}]: {stats_record:?}");
         }
 
         // get Type from stats record
-        col_type = stats_record.get(stats_col_index_map["type"]).unwrap();
+        col_type = stats_record.r#type.clone();
+
         // get NullCount
-        col_null_count = if let Some(s) = stats_record.get(stats_col_index_map["nullcount"]) {
-            s.parse::<usize>().unwrap_or(0_usize)
-        } else {
-            0_usize
-        };
+        col_null_count = stats_record.nullcount;
 
         // debug!(
         //     "{header_string}: type={col_type}, optional={}",
@@ -277,27 +273,25 @@ pub fn infer_schema_from_stats(
         type_list.clear();
         enum_list.clear();
 
-        match col_type {
+        match col_type.as_str() {
             "String" => {
                 type_list.push(Value::String("string".to_string()));
 
                 // minLength constraint
-                if let Some(min_length_str) = stats_record.get(stats_col_index_map["min_length"]) {
-                    let min_length = min_length_str.parse::<u32>().unwrap();
+                if let Some(min_length) = stats_record.min_length {
                     field_map.insert(
                         "minLength".to_string(),
                         Value::Number(Number::from(min_length)),
                     );
-                };
+                }
 
                 // maxLength constraint
-                if let Some(max_length_str) = stats_record.get(stats_col_index_map["max_length"]) {
-                    let max_length = max_length_str.parse::<u32>().unwrap();
+                if let Some(max_length) = stats_record.max_length {
                     field_map.insert(
                         "maxLength".to_string(),
                         Value::Number(Number::from(max_length)),
                     );
-                };
+                }
 
                 // enum constraint
                 if let Some(values) = unique_values_map.get(&header_string) {
@@ -309,15 +303,23 @@ pub fn infer_schema_from_stats(
             "Integer" => {
                 type_list.push(Value::String("integer".to_string()));
 
-                if let Some(min_str) = stats_record.get(stats_col_index_map["min"]) {
-                    let min = atoi_simd::parse::<i64>(min_str.as_bytes()).unwrap();
-                    field_map.insert("minimum".to_string(), Value::Number(Number::from(min)));
-                };
+                if let Some(min) = stats_record.min {
+                    field_map.insert(
+                        "minimum".to_string(),
+                        Value::Number(Number::from(
+                            atoi_simd::parse::<i64>(min.as_bytes()).unwrap(),
+                        )),
+                    );
+                }
 
-                if let Some(max_str) = stats_record.get(stats_col_index_map["max"]) {
-                    let max = atoi_simd::parse::<i64>(max_str.as_bytes()).unwrap();
-                    field_map.insert("maximum".to_string(), Value::Number(Number::from(max)));
-                };
+                if let Some(max) = stats_record.max {
+                    field_map.insert(
+                        "maximum".to_string(),
+                        Value::Number(Number::from(
+                            atoi_simd::parse::<i64>(max.as_bytes()).unwrap(),
+                        )),
+                    );
+                }
 
                 // enum constraint
                 if let Some(values) = unique_values_map.get(&header_string) {
@@ -330,21 +332,19 @@ pub fn infer_schema_from_stats(
             "Float" => {
                 type_list.push(Value::String("number".to_string()));
 
-                if let Some(min_str) = stats_record.get(stats_col_index_map["min"]) {
-                    let min = min_str.parse::<f64>().unwrap();
+                if let Some(min) = stats_record.min {
                     field_map.insert(
                         "minimum".to_string(),
-                        Value::Number(Number::from_f64(min).unwrap()),
+                        Value::Number(Number::from_f64(min.parse::<f64>().unwrap()).unwrap()),
                     );
-                };
+                }
 
-                if let Some(max_str) = stats_record.get(stats_col_index_map["max"]) {
-                    let max = max_str.parse::<f64>().unwrap();
+                if let Some(max) = stats_record.max {
                     field_map.insert(
                         "maximum".to_string(),
-                        Value::Number(Number::from_f64(max).unwrap()),
+                        Value::Number(Number::from_f64(max.parse::<f64>().unwrap()).unwrap()),
                     );
-                };
+                }
             },
             "NULL" => {
                 type_list.push(Value::String("null".to_string()));
@@ -403,29 +403,22 @@ pub fn infer_schema_from_stats(
 
 /// get column selector argument string for low cardinality columns
 fn build_low_cardinality_column_selector_arg(
-    low_cardinality_column_indices: &mut Vec<usize>,
-    enum_cardinality_threshold: usize,
+    low_cardinality_column_indices: &mut Vec<u64>,
+    enum_cardinality_threshold: u64,
     csv_fields: &ByteRecord,
-    csv_stats: &[Stats],
-    stats_col_index_map: &AHashMap<String, usize>,
+    csv_stats: &[StatsData],
 ) -> String {
     low_cardinality_column_indices.clear();
 
     // identify low cardinality columns
+    #[allow(clippy::needless_range_loop)]
     for i in 0..csv_fields.len() {
-        // grab stats record for current column
-        let stats_record = csv_stats.get(i).unwrap().clone().to_record(4, false);
-
         // get Cardinality
-        let col_cardinality = match stats_record.get(stats_col_index_map["cardinality"]) {
-            Some(s) => s.parse::<usize>().unwrap_or(0_usize),
-            None => 0_usize,
-        };
-        // debug!("column_{i}: cardinality={col_cardinality}");
+        let col_cardinality = csv_stats[i].cardinality;
 
         if col_cardinality > 0 && col_cardinality <= enum_cardinality_threshold {
             // column selector uses 1-based index
-            low_cardinality_column_indices.push(i + 1);
+            low_cardinality_column_indices.push((i + 1) as u64);
         };
     }
 
@@ -450,7 +443,7 @@ fn get_unique_values(
         arg_input:            args.arg_input.clone(),
         flag_select:          crate::select::SelectColumns::parse(column_select_arg).unwrap(),
         flag_limit:           args.flag_enum_threshold as isize,
-        flag_unq_limit:       args.flag_enum_threshold,
+        flag_unq_limit:       args.flag_enum_threshold as usize,
         flag_lmt_threshold:   0,
         flag_pct_dec_places:  -5,
         flag_other_sorted:    false,
@@ -545,8 +538,6 @@ fn generate_string_patterns(
     args: &util::SchemaArgs,
     properties_map: &Map<String, Value>,
 ) -> CliResult<AHashMap<String, String>> {
-    // standard boiler-plate for reading CSV
-
     let rconfig = Config::new(&args.arg_input)
         .delimiter(args.flag_delimiter)
         .no_headers(args.flag_no_headers)
diff --git a/src/cmd/stats.rs b/src/cmd/stats.rs
index 9f1dae542..c503552cc 100644
--- a/src/cmd/stats.rs
+++ b/src/cmd/stats.rs
@@ -51,9 +51,8 @@ chunks and each chunk is processed in parallel. The number of chunks is determin
 number of logical CPUs detected. You can override this by setting the --jobs option.
 
 As stats is a central command in qsv, and can be expensive to compute, `stats` caches results
-in <FILESTEM>.stats.csv & if the --stats-binout option is used, <FILESTEM>.stats.csv.bin.sz 
-(e.g., qsv stats nyc311.csv will create nyc311.stats.csv & nyc311.stats.csv.bin.sz).
-The .bin.sz file is the snappy-compressed binary format of the computed stats.
+in <FILESTEM>.stats.csv & if the --stats-json option is used, <FILESTEM>.stats.csv.data.json 
+(e.g., qsv stats nyc311.csv will create nyc311.stats.csv & nyc311.stats.csv.data.json).
 The arguments used to generate the cached stats are saved in <FILESTEM>.stats.csv.json.
 
 If stats have already been computed for the input file with similar arguments and the file
@@ -183,14 +182,13 @@ stats options:
                               Note that a file handle is opened for each job.
                               When not set, the number of jobs is set to the
                               number of CPUs detected.
-    --stats-binout            Write the stats in binary format. This is used internally by other
-                              qsv commands (currently `frequency`, `schema` & `tojsonl`) to load
-                              cached stats into memory faster. If set, the snappy compressed
-                              binary encoded stats will be written to <FILESTEM>.stats.csv.bin.sz.
-                              You can preemptively create the binary encoded stats file by using
+    --stats-json              Also write the stats in json format. 
+                              If set, the stats will be written to <FILESTEM>.stats.csv.data.json.
+                              Note that this option used internally by other qsv commands
+                              (currently `frequency`, `schema` & `tojsonl`) to load cached stats. 
+                              You can preemptively create the stats-json file by using
                               this option BEFORE running the `frequency`, `schema` & `tojsonl`
-                              commands and they will automatically load the binary encoded
-                              stats file if it exists.
+                              commands and they will automatically use it.
  -c, --cache-threshold <arg>  When greater than 1, the threshold in milliseconds before caching
                               stats results. If a stats run takes longer than this threshold,
                               the stats results will be cached.
@@ -245,7 +243,7 @@ use std::{
     sync::OnceLock,
 };
 
-use gzp::{par::compress::ParCompressBuilder, snap::Snap};
+// use gzp::{par::compress::ParCompressBuilder, snap::Snap};
 use itertools::Itertools;
 use qsv_dateparser::parse_with_preference;
 use serde::{Deserialize, Serialize};
@@ -256,9 +254,10 @@ use threadpool::ThreadPool;
 
 use self::FieldType::{TDate, TDateTime, TFloat, TInteger, TNull, TString};
 use crate::{
-    config::{Config, Delimiter, DEFAULT_WTR_BUFFER_CAPACITY},
+    config::{Config, Delimiter}, //, DEFAULT_WTR_BUFFER_CAPACITY},
     select::{SelectColumns, Selection},
-    util, CliResult,
+    util,
+    CliResult,
 };
 
 #[allow(clippy::unsafe_derive_deserialize)]
@@ -281,7 +280,7 @@ pub struct Args {
     pub flag_prefer_dmy:      bool,
     pub flag_force:           bool,
     pub flag_jobs:            Option<usize>,
-    pub flag_stats_binout:    bool,
+    pub flag_stats_json:      bool,
     pub flag_cache_threshold: isize,
     pub flag_output:          Option<String>,
     pub flag_no_headers:      bool,
@@ -320,6 +319,92 @@ struct StatsArgs {
     qsv_version:          String,
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
+pub struct StatsData {
+    pub field:                String,
+    // type is a reserved keyword in Rust
+    // so we escape it as r#type
+    // we need to do this for serde to work
+    pub r#type:               String,
+    pub is_ascii:             bool,
+    pub sum:                  Option<f64>,
+    pub min:                  Option<String>,
+    pub max:                  Option<String>,
+    pub range:                Option<f64>,
+    pub min_length:           Option<usize>,
+    pub max_length:           Option<usize>,
+    pub mean:                 Option<f64>,
+    pub sem:                  Option<f64>,
+    pub stddev:               Option<f64>,
+    pub variance:             Option<f64>,
+    pub cv:                   Option<f64>,
+    pub nullcount:            u64,
+    pub max_precision:        Option<u32>,
+    pub sparsity:             Option<f64>,
+    pub mad:                  Option<f64>,
+    pub lower_outer_fence:    Option<f64>,
+    pub lower_inner_fence:    Option<f64>,
+    pub q1:                   Option<f64>,
+    pub q2_median:            Option<f64>,
+    pub q3:                   Option<f64>,
+    pub iqr:                  Option<f64>,
+    pub upper_inner_fence:    Option<f64>,
+    pub upper_outer_fence:    Option<f64>,
+    pub skewness:             Option<f64>,
+    pub cardinality:          u64,
+    pub mode:                 Option<String>,
+    pub mode_count:           Option<u64>,
+    pub mode_occurrences:     Option<u64>,
+    pub antimode:             Option<String>,
+    pub antimode_count:       Option<u64>,
+    pub antimode_occurrences: Option<u64>,
+}
+
+#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub enum JsonTypes {
+    Int,
+    Float,
+    Bool,
+    String,
+}
+
+pub static STATSDATA_TYPES_ARRAY: [JsonTypes; 34] = [
+    JsonTypes::String, //field
+    JsonTypes::String, //type
+    JsonTypes::Bool,   //is_ascii
+    JsonTypes::Float,  //sum
+    JsonTypes::String, //min
+    JsonTypes::String, //max
+    JsonTypes::Float,  //range
+    JsonTypes::Int,    //min_length
+    JsonTypes::Int,    //max_length
+    JsonTypes::Float,  //mean
+    JsonTypes::Float,  //sem
+    JsonTypes::Float,  //stddev
+    JsonTypes::Float,  //variance
+    JsonTypes::Float,  //cv
+    JsonTypes::Int,    //nullcount
+    JsonTypes::Int,    //max_precision
+    JsonTypes::Float,  //sparsity
+    JsonTypes::Float,  //mad
+    JsonTypes::Float,  //lower_outer_fence
+    JsonTypes::Float,  //lower_inner_fence
+    JsonTypes::Float,  //q1
+    JsonTypes::Float,  //q2_median
+    JsonTypes::Float,  //q3
+    JsonTypes::Float,  //iqr
+    JsonTypes::Float,  //upper_inner_fence
+    JsonTypes::Float,  //upper_outer_fence
+    JsonTypes::Float,  //skewness
+    JsonTypes::Int,    //cardinality
+    JsonTypes::String, //mode
+    JsonTypes::Int,    //mode_count
+    JsonTypes::Int,    //mode_occurrences
+    JsonTypes::String, //antimode
+    JsonTypes::Int,    //antimode_count
+    JsonTypes::Int,    //antimode_occurrences
+];
+
 static INFER_DATE_FLAGS: OnceLock<Vec<bool>> = OnceLock::new();
 static RECORD_COUNT: OnceLock<u64> = OnceLock::new();
 
@@ -436,14 +521,11 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
         }
     }
 
-    // create stats_for_encoding to store the stats in binary format
-    let mut stats_for_encoding: Vec<Stats> = Vec::new();
-
     let mut compute_stats = true;
-    let mut create_cache = args.flag_cache_threshold > 0 || args.flag_stats_binout;
+    let mut create_cache = args.flag_cache_threshold > 0 || args.flag_stats_json;
     let mut autoindex_set = false;
 
-    let write_stats_binout = args.flag_stats_binout;
+    let write_stats_json = args.flag_stats_json;
 
     if let Some(path) = fconfig.path.clone() {
         let path_file_stem = path.file_stem().unwrap().to_str().unwrap();
@@ -572,14 +654,10 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
                 },
             }?;
 
-            // clone a copy of stats so we can binary encode it to disk later
-            if write_stats_binout {
-                stats_for_encoding.clone_from(&stats);
-            }
-
             let stats_sr_vec = args.stats_to_records(stats);
 
-            wtr.write_record(&args.stat_headers())?;
+            let stats_headers_sr = args.stat_headers();
+            wtr.write_record(&stats_headers_sr)?;
             let fields = headers.iter().zip(stats_sr_vec);
             for (i, (header, stat)) in fields.enumerate() {
                 let header = if args.flag_no_headers {
@@ -692,28 +770,10 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
                 serde_json::to_string_pretty(&current_stats_args).unwrap(),
             )?;
 
-            // only create the binary encoded files if we computed the stats
-            // and the user specified --stats-binout
-            if write_stats_binout {
-                // binary encode the stats to "<FILESTEM>.stats.csv.bin.sz"
-                let mut stats_bin_pathbuf = path;
-                stats_bin_pathbuf.set_extension("stats.csv.bin.sz");
-                // we do the binary encoding inside a block so that the encoded_file
-                // automatically gets dropped/flushed before we copy it to the output file
-                {
-                    let encoded_file = ParCompressBuilder::<Snap>::new()
-                        .num_threads(util::max_jobs())?
-                        .buffer_size(DEFAULT_WTR_BUFFER_CAPACITY * 2)?
-                        .pin_threads(Some(0))
-                        .from_writer(fs::File::create(stats_bin_pathbuf.clone())?);
-
-                    if let Err(e) = bincode::serialize_into(encoded_file, &stats_for_encoding) {
-                        return fail_clierror!(
-                            "Failed to write binary encoded stats {}: {e:?}",
-                            stats_bin_pathbuf.display()
-                        );
-                    }
-                }
+            // save the stats data to "<FILESTEM>.stats.csv.data.json"
+            if write_stats_json {
+                stats_pathbuf.set_extension("data.json");
+                util::csv_to_jsonl(&currstats_filename, &STATSDATA_TYPES_ARRAY, stats_pathbuf)?;
             }
         }
     }
diff --git a/src/util.rs b/src/util.rs
index 2a96d6ac8..8bfa283e0 100644
--- a/src/util.rs
+++ b/src/util.rs
@@ -1,24 +1,21 @@
 #[cfg(any(feature = "feature_capable", feature = "lite"))]
 use std::borrow::Cow;
-#[cfg(any(feature = "feature_capable", feature = "lite"))]
-use std::io::BufRead;
 use std::{
     cmp::min,
     env, fs,
     fs::File,
-    io::{BufReader, BufWriter, Read, Write},
+    io::{BufRead, BufReader, BufWriter, Read, Write},
     path::{Path, PathBuf},
     str,
     sync::OnceLock,
     time::SystemTime,
 };
 
-use ahash::AHashMap;
 use csv::ByteRecord;
 use docopt::Docopt;
 #[cfg(any(feature = "feature_capable", feature = "lite"))]
 use indicatif::{HumanCount, ProgressBar, ProgressDrawTarget, ProgressStyle};
-use log::{info, log_enabled, warn};
+use log::{info, log_enabled};
 use reqwest::Client;
 use serde::de::DeserializeOwned;
 #[cfg(any(feature = "feature_capable", feature = "lite"))]
@@ -28,6 +25,7 @@ use sysinfo::System;
 #[cfg(feature = "polars")]
 use crate::cmd::count::polars_count_input;
 use crate::{
+    cmd::stats::{JsonTypes, StatsData, STATSDATA_TYPES_ARRAY},
     config,
     config::{Config, Delimiter, DEFAULT_RDR_BUFFER_CAPACITY, DEFAULT_WTR_BUFFER_CAPACITY},
     select::SelectColumns,
@@ -60,7 +58,7 @@ pub enum StatsMode {
 #[allow(dead_code)]
 #[derive(serde::Deserialize, Clone)]
 pub struct SchemaArgs {
-    pub flag_enum_threshold:  usize,
+    pub flag_enum_threshold:  u64,
     pub flag_ignore_case:     bool,
     pub flag_strict_dates:    bool,
     pub flag_pattern_columns: SelectColumns,
@@ -1917,16 +1915,12 @@ pub fn trim_bs_whitespace(bytes: &[u8]) -> &[u8] {
     &bytes[start..end]
 }
 
-// get stats records from stats.bin file, or if its invalid, by running the stats command
+/// get stats records from stats.csv.data.json file, or if its invalid, by running the stats command
 /// returns tuple (`csv_fields`, `csv_stats`, `stats_col_index_map`)
 pub fn get_stats_records(
     args: &SchemaArgs,
     mode: StatsMode,
-) -> CliResult<(
-    ByteRecord,
-    Vec<crate::cmd::stats::Stats>,
-    AHashMap<String, usize>,
-)> {
+) -> CliResult<(ByteRecord, Vec<StatsData>)> {
     let stats_args = crate::cmd::stats::Args {
         arg_input:            args.arg_input.clone(),
         flag_select:          crate::select::SelectColumns::parse("").unwrap(),
@@ -1945,7 +1939,7 @@ pub fn get_stats_records(
         flag_prefer_dmy:      args.flag_prefer_dmy,
         flag_force:           args.flag_force,
         flag_jobs:            Some(njobs(args.flag_jobs)),
-        flag_stats_binout:    true,
+        flag_stats_json:      true,
         flag_cache_threshold: 1, // force the creation of stats cache files
         flag_output:          None,
         flag_no_headers:      args.flag_no_headers,
@@ -1953,72 +1947,78 @@ pub fn get_stats_records(
         flag_memcheck:        args.flag_memcheck,
     };
 
+    if mode == StatsMode::None
+        || args.arg_input.is_none()
+        || args.arg_input.as_ref() == Some(&"-".to_string())
+    {
+        // if stdin or StatsMode::None,
+        // we're just doing frequency old school w/o cardinality
+        return Ok((ByteRecord::new(), Vec::new()));
+    };
+
     let canonical_input_path = Path::new(&args.arg_input.clone().unwrap()).canonicalize()?;
-    let stats_binary_encoded_path = canonical_input_path.with_extension("stats.csv.bin.sz");
+    let statsdata_path = canonical_input_path.with_extension("stats.csv.data.json");
 
-    let stats_bin_current = if stats_binary_encoded_path.exists() {
-        let stats_bin_metadata = std::fs::metadata(&stats_binary_encoded_path)?;
+    let stats_data_current = if statsdata_path.exists() {
+        let statsdata_metadata = std::fs::metadata(&statsdata_path)?;
 
         let input_metadata = std::fs::metadata(args.arg_input.clone().unwrap())?;
 
-        if stats_bin_metadata.modified()? > input_metadata.modified()? {
-            info!("Valid stats.csv.bin.sz file found!");
+        if statsdata_metadata.modified()? > input_metadata.modified()? {
+            info!("Valid stats.csv.data.json file found!");
             true
         } else {
-            info!("stats.csv.bin.sz file is older than input file. Regenerating stats.bin file.");
+            info!(
+                "stats.csv.data.json file is older than input file. Regenerating stats json file."
+            );
             false
         }
     } else {
-        info!("stats.csv.bin.sz file does not exist: {stats_binary_encoded_path:?}");
+        info!("stats.csv.data.json file does not exist: {statsdata_path:?}");
         false
     };
 
-    if mode == StatsMode::None || (mode == StatsMode::Frequency && !stats_bin_current) {
-        // if the stats.bin file is not present, we're just doing frequency old school
-        // without cardinality
-        return Ok((ByteRecord::new(), Vec::new(), AHashMap::new()));
+    if mode == StatsMode::Frequency && !stats_data_current {
+        // if the stats.data file is not current,
+        // we're also doing frequency old school w/o cardinality
+        return Ok((ByteRecord::new(), Vec::new()));
     }
 
-    let mut stats_bin_loaded = false;
+    let mut stats_data_loaded = false;
 
-    // if stats.bin file exists and is current, use it
-    let mut csv_stats: Vec<crate::cmd::stats::Stats> = Vec::new();
+    let mut csv_stats: Vec<StatsData> = Vec::new();
 
-    if stats_bin_current && !args.flag_force {
-        let bin_file = BufReader::with_capacity(
-            DEFAULT_RDR_BUFFER_CAPACITY * 4,
-            File::open(stats_binary_encoded_path)?,
-        );
-        let mut buf_binsz_decoder = snap::read::FrameDecoder::new(bin_file);
-        match bincode::deserialize_from(&mut buf_binsz_decoder) {
-            Ok(stats) => {
-                csv_stats = stats;
-                stats_bin_loaded = true;
-            },
-            Err(e) => {
-                wwarn!(
-                    "Error reading stats.csv.bin.sz file: {e:?}. Regenerating stats.bin.sz file."
-                );
-            },
+    // if stats_data file exists and is current, use it
+    if stats_data_current && !args.flag_force {
+        stats_data_loaded = true;
+
+        let statsdata_file = std::fs::File::open(&statsdata_path)?;
+        let statsdata_reader = std::io::BufReader::new(statsdata_file);
+        let statsdata_lines = statsdata_reader.lines();
+
+        for line in statsdata_lines {
+            let line = line?;
+            let stats_record: StatsData = serde_json::from_str(&line)?;
+            csv_stats.push(stats_record);
         }
     }
 
-    if !stats_bin_loaded {
-        // otherwise, run stats command to generate stats.csv.bin.sz file
+    if !stats_data_loaded {
+        // otherwise, run stats command to generate stats.csv.data.json file
         let tempfile = tempfile::Builder::new()
             .suffix(".stats.csv")
             .tempfile()
             .unwrap();
         let tempfile_path = tempfile.path().to_str().unwrap().to_string();
 
-        let statsbin_path = canonical_input_path.with_extension("stats.csv.bin.sz");
+        let statsdatajson_path = canonical_input_path.with_extension("stats.csv.data.json");
 
         let mut stats_args_str = if mode == StatsMode::Schema {
             // mode is GetStatsMode::Schema
-            // we're generating schema, so we need all the stats
+            // we're generating schema, so we cardinality and to infer-dates
             format!(
                 "stats {input} --infer-dates --dates-whitelist {dates_whitelist} --round 4 \
-                 --cardinality --output {output} --stats-binout --force",
+                 --cardinality --output {output} --stats-json --force",
                 input = {
                     if let Some(arg_input) = stats_args.arg_input.clone() {
                         arg_input
@@ -2033,7 +2033,7 @@ pub fn get_stats_records(
             // mode is GetStatsMode::Frequency or GetStatsMode::FrequencyForceStats
             // we're doing frequency, so we just need cardinality
             format!(
-                "stats {input} --cardinality --stats-binout --output {output}",
+                "stats {input} --cardinality --stats-json --output {output}",
                 input = {
                     if let Some(arg_input) = stats_args.arg_input.clone() {
                         arg_input
@@ -2071,19 +2071,24 @@ pub fn get_stats_records(
         stats_cmd.args(stats_args_vec);
         let _stats_output = stats_cmd.output()?;
 
-        let bin_file =
-            BufReader::with_capacity(DEFAULT_RDR_BUFFER_CAPACITY * 2, File::open(statsbin_path)?);
-        let mut buf_binsz_decoder = snap::read::FrameDecoder::new(bin_file);
+        // create a statsdatajon from the output of the stats command
+        csv_to_jsonl(
+            &tempfile_path,
+            &STATSDATA_TYPES_ARRAY,
+            statsdatajson_path.clone(),
+        )?;
 
-        match bincode::deserialize_from(&mut buf_binsz_decoder) {
-            Ok(stats) => {
-                csv_stats = stats;
-            },
-            Err(e) => {
-                return fail_clierror!(
-                    "Error reading stats.csv.bin.sz file: {e:?}. Schema generation aborted."
-                );
-            },
+        let statsdatajson_rdr = BufReader::with_capacity(
+            DEFAULT_RDR_BUFFER_CAPACITY * 2,
+            File::open(statsdatajson_path)?,
+        );
+
+        let mut statsrecord: StatsData;
+        let mut curr_line: String;
+        for line in statsdatajson_rdr.lines() {
+            curr_line = line?;
+            statsrecord = serde_json::from_str(&curr_line)?;
+            csv_stats.push(statsrecord);
         }
     };
 
@@ -2092,29 +2097,74 @@ pub fn get_stats_records(
     let csv_fields = rdr.byte_headers()?.clone();
     drop(rdr);
 
-    let stats_columns = if stats_bin_loaded {
-        // if stats.bin file is loaded, we need to get the headers from the stats.csv file
-        let stats_bin_csv_path = canonical_input_path.with_extension("stats.csv");
-        let mut stats_csv_reader = csv::Reader::from_path(stats_bin_csv_path)?;
-        let stats_csv_headers = stats_csv_reader.headers()?.clone();
-        drop(stats_csv_reader);
-        stats_csv_headers
-    } else {
-        // otherwise, we generate the headers from the stats_args struct
-        // as we used the stats_args struct to generate the stats.csv file
-        stats_args.stat_headers()
-    };
+    Ok((csv_fields, csv_stats))
+}
+
+/// simple helper to convert a CSV file to a JSONL file
+/// no type inferencing is done unlike tojsonl, so all fields are strings
+pub fn csv_to_jsonl(
+    input_csv: &str,
+    csv_types: &[JsonTypes],
+    output_jsonl: PathBuf,
+) -> CliResult<()> {
+    let file = File::open(input_csv)?;
+    let mut rdr = csv::ReaderBuilder::new()
+        .has_headers(true) // requires headers for keys
+        .from_reader(file);
+
+    // Get the headers and create a vector of of keys
+    let headers = rdr.headers()?;
+    let key_vec: Vec<String> = headers
+        .iter()
+        .map(std::string::ToString::to_string)
+        .collect();
+
+    let output = File::create(output_jsonl)?;
+    let mut writer = BufWriter::new(output);
 
-    let mut stats_col_index_map = AHashMap::new();
+    // amortize allocations
+    let mut json_object = serde_json::Map::with_capacity(key_vec.len());
+    let mut record = csv::StringRecord::new();
+    let mut json_line: String;
 
-    for (i, col) in stats_columns.iter().enumerate() {
-        if col != "field" {
-            // need offset by 1 due to extra "field" column in headers that's not in stats records
-            stats_col_index_map.insert(col.to_owned(), i - 1);
+    // Iterate over each record in the CSV
+    while rdr.read_record(&mut record)? {
+        json_object.clear();
+
+        // safety: we know the record length is the same as the key_vec length
+        for (i, val) in record.iter().enumerate() {
+            let key = unsafe { key_vec.get_unchecked(i) };
+            let data_type = if key == "cardinality" {
+                &JsonTypes::Int
+            } else {
+                csv_types.get(i).unwrap_or(&JsonTypes::String)
+            };
+            let value = if val.is_empty() && data_type != &JsonTypes::Bool {
+                continue;
+            } else {
+                match *data_type {
+                    JsonTypes::String => serde_json::Value::String(val.to_owned()),
+                    JsonTypes::Int => {
+                        let num = val.parse::<u64>().unwrap_or_default();
+                        serde_json::Value::Number(serde_json::Number::from(num))
+                    },
+                    JsonTypes::Float => serde_json::Value::Number(
+                        serde_json::Number::from_f64(val.parse::<f64>().unwrap_or_default())
+                            .unwrap_or_else(|| serde_json::Number::from(0)),
+                    ),
+                    JsonTypes::Bool => {
+                        serde_json::Value::Bool(val.parse::<bool>().unwrap_or(false))
+                    },
+                }
+            };
+            json_object.insert(key.to_string(), value);
         }
+
+        json_line = serde_json::to_string(&json_object)?;
+        writeln!(writer, "{json_line}")?;
     }
 
-    Ok((csv_fields, csv_stats, stats_col_index_map))
+    Ok(writer.flush()?)
 }
 
 // comment out for now as this is still WIP


Test patch:
diff --git a/tests/test_frequency.rs b/tests/test_frequency.rs
index 576924144..cbf7825e2 100644
--- a/tests/test_frequency.rs
+++ b/tests/test_frequency.rs
@@ -482,7 +482,7 @@ fn frequency_all_unique_with_stats_cache() {
     stats_cmd
         .arg(testdata.clone())
         .arg("--cardinality")
-        .arg("--stats-binout");
+        .arg("--stats-json");
 
     wrk.assert_success(&mut stats_cmd);
 
@@ -508,7 +508,7 @@ fn frequency_all_unique_with_stats_cache_alt_all_unique_text() {
     stats_cmd
         .arg(testdata.clone())
         .arg("--cardinality")
-        .arg("--stats-binout");
+        .arg("--stats-json");
 
     wrk.assert_success(&mut stats_cmd);
 
@@ -555,7 +555,7 @@ fn frequency_all_unique_stats_mode_none() {
     stats_cmd
         .arg(testdata.clone())
         .arg("--cardinality")
-        .arg("--stats-binout");
+        .arg("--stats-json");
 
     wrk.assert_success(&mut stats_cmd);