Repository: dathere/qsv (language: rust)

Issue:
Smarter frequency compilation
If a stats cache is present, `frequency` will check if the cardinality for a column has already been computed.

If cardinality is equal to the number of rows, then it will short-circuit compiling the frequency table for a column as it knows all values are unique.

Also, `frequency` can take the column's cardinality into account when compiling the hashmap and short-circuit compiling the "Other" values if it already can infer that it has completed the frequency table for a column.

This should make frequency much faster and greatly reduce its memory requirements as it won't need to unnecessarily maintain a hashmap for a column's values if it has the needed frequency values already.


Discussion / hints:
This was a bit more complicated than I originally thought.

For the initial implementation, limiting the cardinality check to columns with unique identifiers - which has the highest payoff anyways in terms of memory footprint and performance.

Short-circuiting frequency calculations for non-unique columns based on cardinality unnecessarily complicates the algorithm for nominal payoff.

Fix patch:
diff --git a/src/clitypes.rs b/src/clitypes.rs
index 82287a767..6612debab 100644
--- a/src/clitypes.rs
+++ b/src/clitypes.rs
@@ -35,7 +35,6 @@ macro_rules! werr {
     });
 }
 
-#[cfg(any(feature = "feature_capable", feature = "lite"))]
 /// write to stderr and log::warn
 macro_rules! wwarn {
     ($($arg:tt)*) => ({
diff --git a/src/cmd/frequency.rs b/src/cmd/frequency.rs
index b10570ad1..74674bf72 100644
--- a/src/cmd/frequency.rs
+++ b/src/cmd/frequency.rs
@@ -24,6 +24,17 @@ items and not to columns with a small number of unique items.
 Since this computes an exact frequency table, memory proportional to the
 cardinality of each column is required.
 
+This is particularly problematic for columns with all unique values (e.g. record IDs),
+as the memory usage is equal to the number of rows in the data, which can cause
+Out-of-Memory (OOM) errors for larger-than-memory datasets.
+
+To overcome this, the frequency command will automatically use the stats cache if it exists
+to get column cardinalities. This short-circuits frequency compilation for columns with
+all unique values, eliminating memory usage as well, allowing you to compute frequencies
+for larger-than-memory datasets.
+
+This behavior can be overridden with the --stats-mode option.
+
 For examples, see https://github.com/jqnatividad/qsv/blob/master/tests/test_frequency.rs.
 
 Usage:
@@ -71,6 +82,17 @@ frequency options:
                             The default is to trim leading and trailing whitespaces.
     --no-nulls              Don't include NULLs in the frequency table.
     -i, --ignore-case       Ignore case when computing frequencies.
+    --stats-mode <arg>      The stats mode to use when computing frequencies with cardinalities.
+                            Having column cardinalities short-circuits frequency compilation and
+                            eliminates memory usage for columns with all unique values.
+                            There are three modes:
+                              auto: use stats cache if it already exists to get column cardinalities.
+                                    For columns with all unique values, "ALL_UNIQUE" will be used.
+                              force: force stats calculation to get cardinalities.
+                              none: don't use cardinality information.
+                                    For columns with all unique values, the first N sorted unique
+                                    values (based on the --limit and --unq-limit options) will be used.
+                            [default: auto]
     -j, --jobs <arg>        The number of jobs to run in parallel.
                             This works much faster when the given CSV data has
                             an index already created. Note that a file handle
@@ -91,7 +113,7 @@ Common options:
                            CSV into memory using CONSERVATIVE heuristics.
 "#;
 
-use std::{fs, io};
+use std::{fs, io, sync::OnceLock};
 
 use indicatif::HumanCount;
 use rust_decimal::prelude::*;
@@ -104,7 +126,7 @@ use crate::{
     index::Indexed,
     select::{SelectColumns, Selection},
     util,
-    util::ByteString,
+    util::{get_stats_records, ByteString, StatsMode},
     CliResult,
 };
 
@@ -123,6 +145,7 @@ pub struct Args {
     pub flag_no_trim:        bool,
     pub flag_no_nulls:       bool,
     pub flag_ignore_case:    bool,
+    pub flag_stats_mode:     String,
     pub flag_jobs:           Option<usize>,
     pub flag_output:         Option<String>,
     pub flag_no_headers:     bool,
@@ -132,6 +155,9 @@ pub struct Args {
 
 const NULL_VAL: &[u8] = b"(NULL)";
 
+static UNIQUE_COLUMNS: OnceLock<Vec<usize>> = OnceLock::new();
+static FREQ_ROW_COUNT: OnceLock<u64> = OnceLock::new();
+
 pub fn run(argv: &[&str]) -> CliResult<()> {
     let args: Args = util::get_args(USAGE, argv)?;
     let rconfig = args.rconfig();
@@ -153,13 +179,20 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
     let mut pct_decimal: Decimal;
     let mut final_pct_decimal: Decimal;
     let mut pct_string: String;
-    let mut pct_scale;
-    let mut current_scale;
+    let mut pct_scale: u32;
+    let mut current_scale: u32;
     let abs_dec_places = args.flag_pct_dec_places.unsigned_abs() as u32;
-    let mut row;
+    let mut row: Vec<&[u8]>;
+    let mut all_unique_header: bool;
+
+    // safety: we know that UNIQUE_COLUMNS has been previously set when compiling frequencies
+    // by sel_headers fn
+    let all_unique_headers = UNIQUE_COLUMNS.get().unwrap();
 
     wtr.write_record(vec!["field", "value", "count", "percentage"])?;
     let head_ftables = headers.iter().zip(tables);
+    let row_count = *FREQ_ROW_COUNT.get().unwrap_or(&0);
+
     for (i, (header, ftab)) in head_ftables.enumerate() {
         header_vec = if rconfig.no_headers {
             (i + 1).to_string().into_bytes()
@@ -167,16 +200,24 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
             header.to_vec()
         };
 
-        let mut sorted_counts: Vec<(Vec<u8>, u64, f64)> = args.counts(&ftab);
+        let mut sorted_counts: Vec<(Vec<u8>, u64, f64)>;
+        all_unique_header = all_unique_headers.contains(&i);
 
-        // if not --other_sorted and the first value is "Other (", rotate it to the end
-        if !args.flag_other_sorted
-            && sorted_counts.first().is_some_and(|(value, _, _)| {
-                value.starts_with(format!("{} (", args.flag_other_text).as_bytes())
-            })
-        {
-            sorted_counts.rotate_left(1);
-        }
+        if all_unique_header {
+            // if the column has all unique values, we don't need to sort the counts
+            sorted_counts = vec![(b"ALL_UNIQUE".to_vec(), row_count, 100.0_f64)];
+        } else {
+            sorted_counts = args.counts(&ftab);
+
+            // if not --other_sorted and the first value is "Other (", rotate it to the end
+            if !args.flag_other_sorted
+                && sorted_counts.first().is_some_and(|(value, _, _)| {
+                    value.starts_with(format!("{} (", args.flag_other_text).as_bytes())
+                })
+            {
+                sorted_counts.rotate_left(1);
+            }
+        };
 
         for (value, count, percentage) in sorted_counts {
             pct_decimal = Decimal::from_f64(percentage).unwrap_or_default();
@@ -370,6 +411,8 @@ impl Args {
         let mut field_buffer: Vec<u8> = Vec::with_capacity(nsel_len);
         let mut row_buffer: csv::ByteRecord = csv::ByteRecord::with_capacity(200, nsel_len);
 
+        let all_unique_headers = UNIQUE_COLUMNS.get().unwrap();
+
         // assign flags to local variables for faster access
         let flag_no_nulls = self.flag_no_nulls;
         let flag_ignore_case = self.flag_ignore_case;
@@ -385,6 +428,11 @@ impl Args {
                     // safety: we know the row is not empty
                     row_buffer.clone_from(&row.unwrap());
                     for (i, field) in nsel.select(row_buffer.into_iter()).enumerate() {
+                        if all_unique_headers.contains(&i) {
+                            // if the column has all unique values,
+                            // we don't need to compute frequencies
+                            continue;
+                        }
                         field_buffer = {
                             if let Ok(s) = simdutf8::basic::from_utf8(field) {
                                 util::to_lowercase_into(s, &mut buf);
@@ -414,6 +462,9 @@ impl Args {
                     // safety: we know the row is not empty
                     row_buffer.clone_from(&row.unwrap());
                     for (i, field) in nsel.select(row_buffer.into_iter()).enumerate() {
+                        if all_unique_headers.contains(&i) {
+                            continue;
+                        }
                         field_buffer = {
                             if let Ok(s) = simdutf8::basic::from_utf8(field) {
                                 util::to_lowercase_into(s.trim(), &mut buf);
@@ -447,6 +498,9 @@ impl Args {
                 if flag_no_trim {
                     // case-sensitive, don't trim whitespace
                     for (i, field) in nsel.select(row_buffer.into_iter()).enumerate() {
+                        if all_unique_headers.contains(&i) {
+                            continue;
+                        }
                         // no need to convert to string and back to bytes for a "case-sensitive"
                         // comparison we can just use the field directly
                         field_buffer = field.to_vec();
@@ -465,6 +519,9 @@ impl Args {
                 } else {
                     // case-sensitive, trim whitespace
                     for (i, field) in nsel.select(row_buffer.into_iter()).enumerate() {
+                        if all_unique_headers.contains(&i) {
+                            continue;
+                        }
                         field_buffer = {
                             if let Ok(s) = simdutf8::basic::from_utf8(field) {
                                 s.trim().as_bytes().to_vec()
@@ -490,11 +547,93 @@ impl Args {
         freq_tables
     }
 
+    /// return the names of headers/columns that are unique identifiers
+    /// (i.e. where cardinality == rowcount)
+    fn get_unique_headers(&self, headers: &Headers) -> CliResult<Vec<usize>> {
+        // get the stats records for the entire CSV
+        let schema_args = util::SchemaArgs {
+            flag_enum_threshold:  0,
+            flag_ignore_case:     self.flag_ignore_case,
+            flag_strict_dates:    false,
+            // we still get all the stats columns so we can use the stats cache
+            flag_pattern_columns: crate::select::SelectColumns::parse("").unwrap(),
+            flag_dates_whitelist: String::new(),
+            flag_prefer_dmy:      false,
+            flag_force:           false,
+            flag_stdout:          false,
+            flag_jobs:            Some(util::njobs(self.flag_jobs)),
+            flag_no_headers:      self.flag_no_headers,
+            flag_delimiter:       self.flag_delimiter,
+            arg_input:            self.arg_input.clone(),
+            flag_memcheck:        false,
+        };
+        let stats_mode = match self.flag_stats_mode.as_str() {
+            "auto" => StatsMode::Frequency,
+            "force" => StatsMode::FrequencyForceStats,
+            "none" => StatsMode::None,
+            "_schema" => StatsMode::Schema, // only meant for schema to use
+            _ => return fail_incorrectusage_clierror!("Invalid stats mode"),
+        };
+        let (csv_fields, csv_stats, stats_col_index_map) =
+            get_stats_records(&schema_args, stats_mode)?;
+
+        if stats_mode == StatsMode::None || stats_mode == StatsMode::Schema || csv_fields.is_empty()
+        {
+            // the stats cache does not exist, just return an empty vector
+            // we're not going to be able to get the cardinalities, so
+            // this signals that we just compute frequencies for all columns
+            return Ok(Vec::new());
+        }
+
+        if csv_fields.len() != csv_stats.len() {
+            // this should never happen
+            return fail_clierror!("Mismatch between the number of fields and stats records");
+        }
+        let col_cardinality_vec: Vec<(String, usize)> = csv_stats
+            .iter()
+            .enumerate()
+            .map(|(i, _record)| {
+                // get the column name and stats record
+                // safety: we know that csv_fields and csv_stats have the same length
+                let col_name = csv_fields.get(i).unwrap();
+                let stats_record = csv_stats.get(i).unwrap().clone().to_record(4, false);
+
+                let col_cardinality = match stats_record.get(stats_col_index_map["cardinality"]) {
+                    Some(s) => s.parse::<usize>().unwrap_or(0_usize),
+                    None => 0_usize,
+                };
+                (
+                    std::str::from_utf8(col_name).unwrap().to_string(),
+                    col_cardinality,
+                )
+            })
+            .collect();
+
+        // now, get the unique headers, where cardinality == rowcount
+        let row_count = util::count_rows(&self.rconfig())? as usize;
+        FREQ_ROW_COUNT.set(row_count as u64).unwrap();
+
+        let mut all_unique_headers_vec: Vec<usize> = Vec::with_capacity(5);
+        for (i, _header) in headers.iter().enumerate() {
+            let cardinality = col_cardinality_vec[i].1;
+
+            if cardinality == row_count {
+                all_unique_headers_vec.push(i);
+            }
+        }
+
+        Ok(all_unique_headers_vec)
+    }
+
     fn sel_headers<R: io::Read>(
         &self,
         rdr: &mut csv::Reader<R>,
     ) -> CliResult<(csv::ByteRecord, Selection)> {
         let headers = rdr.byte_headers()?;
+        let all_unique_headers_vec = self.get_unique_headers(headers)?;
+
+        UNIQUE_COLUMNS.set(all_unique_headers_vec).unwrap();
+
         let sel = self.rconfig().selection(headers)?;
         Ok((sel.select(headers).map(<[u8]>::to_vec).collect(), sel))
     }
diff --git a/src/cmd/schema.rs b/src/cmd/schema.rs
index f933b941f..fa8ac998b 100644
--- a/src/cmd/schema.rs
+++ b/src/cmd/schema.rs
@@ -77,11 +77,7 @@ Common options:
                                CSV into memory using CONSERVATIVE heuristics.
 "#;
 
-use std::{
-    fs::File,
-    io::{BufReader, Write},
-    path::Path,
-};
+use std::{fs::File, io::Write, path::Path};
 
 use ahash::{AHashMap, AHashSet};
 use csv::ByteRecord;
@@ -89,38 +85,15 @@ use grex::RegExpBuilder;
 use itertools::Itertools;
 use log::{debug, error, info, warn};
 use rayon::slice::ParallelSliceMut;
-use serde::Deserialize;
 use serde_json::{json, value::Number, Map, Value};
 use stats::Frequencies;
 
-use crate::{
-    cmd::stats::Stats,
-    config::{Config, Delimiter, DEFAULT_RDR_BUFFER_CAPACITY},
-    select::SelectColumns,
-    util, CliResult,
-};
-
-#[derive(Deserialize, Clone)]
-pub struct Args {
-    pub flag_enum_threshold:  usize,
-    pub flag_ignore_case:     bool,
-    pub flag_strict_dates:    bool,
-    pub flag_pattern_columns: SelectColumns,
-    pub flag_dates_whitelist: String,
-    pub flag_prefer_dmy:      bool,
-    pub flag_force:           bool,
-    pub flag_stdout:          bool,
-    pub flag_jobs:            Option<usize>,
-    pub flag_no_headers:      bool,
-    pub flag_delimiter:       Option<Delimiter>,
-    pub arg_input:            Option<String>,
-    pub flag_memcheck:        bool,
-}
+use crate::{cmd::stats::Stats, config::Config, util, util::StatsMode, CliResult};
 
 const STDIN_CSV: &str = "stdin.csv";
 
 pub fn run(argv: &[&str]) -> CliResult<()> {
-    let mut args: Args = util::get_args(USAGE, argv)?;
+    let mut args: util::SchemaArgs = util::get_args(USAGE, argv)?;
 
     // if using stdin, we create a stdin.csv file as stdin is not seekable and we need to
     // open the file multiple times to compile stats/unique values, etc.
@@ -230,9 +203,13 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
 ///  * maxLength
 ///  * min
 ///  * max
-pub fn infer_schema_from_stats(args: &Args, input_filename: &str) -> CliResult<Map<String, Value>> {
+pub fn infer_schema_from_stats(
+    args: &util::SchemaArgs,
+    input_filename: &str,
+) -> CliResult<Map<String, Value>> {
     // invoke cmd::stats
-    let (csv_fields, csv_stats, stats_col_index_map) = get_stats_records(args)?;
+    let (csv_fields, csv_stats, stats_col_index_map) =
+        util::get_stats_records(args, StatsMode::Schema)?;
 
     // amortize memory allocation
     let mut low_cardinality_column_indices: Vec<usize> =
@@ -424,175 +401,6 @@ pub fn infer_schema_from_stats(args: &Args, input_filename: &str) -> CliResult<M
     Ok(properties_map)
 }
 
-/// get stats records from stats.bin file, or if its invalid, by running the stats command
-/// returns tuple (`csv_fields`, `csv_stats`, `stats_col_index_map`)
-fn get_stats_records(args: &Args) -> CliResult<(ByteRecord, Vec<Stats>, AHashMap<String, usize>)> {
-    let stats_args = crate::cmd::stats::Args {
-        arg_input:            args.arg_input.clone(),
-        flag_select:          crate::select::SelectColumns::parse("").unwrap(),
-        flag_everything:      false,
-        flag_typesonly:       false,
-        flag_infer_boolean:   false,
-        flag_mode:            false,
-        flag_cardinality:     true,
-        flag_median:          false,
-        flag_quartiles:       false,
-        flag_mad:             false,
-        flag_nulls:           false,
-        flag_round:           4,
-        flag_infer_dates:     true,
-        flag_dates_whitelist: args.flag_dates_whitelist.to_string(),
-        flag_prefer_dmy:      args.flag_prefer_dmy,
-        flag_force:           args.flag_force,
-        flag_jobs:            Some(util::njobs(args.flag_jobs)),
-        flag_stats_binout:    true,
-        flag_cache_threshold: 1, // force the creation of stats cache files
-        flag_output:          None,
-        flag_no_headers:      args.flag_no_headers,
-        flag_delimiter:       args.flag_delimiter,
-        flag_memcheck:        args.flag_memcheck,
-    };
-
-    let canonical_input_path = Path::new(&args.arg_input.clone().unwrap()).canonicalize()?;
-    let stats_binary_encoded_path = canonical_input_path.with_extension("stats.csv.bin.sz");
-
-    let stats_bin_current = if stats_binary_encoded_path.exists() {
-        let stats_bin_metadata = std::fs::metadata(&stats_binary_encoded_path)?;
-
-        let input_metadata = std::fs::metadata(args.arg_input.clone().unwrap())?;
-
-        if stats_bin_metadata.modified()? > input_metadata.modified()? {
-            info!("Valid stats.csv.bin.sz file found!");
-            true
-        } else {
-            info!("stats.csv.bin.sz file is older than input file. Regenerating stats.bin file.");
-            false
-        }
-    } else {
-        info!("stats.csv.bin.sz file does not exist: {stats_binary_encoded_path:?}");
-        false
-    };
-
-    let mut stats_bin_loaded = false;
-
-    // if stats.bin file exists and is current, use it
-    let mut csv_stats: Vec<Stats> = Vec::new();
-
-    if stats_bin_current && !args.flag_force {
-        let bin_file = BufReader::with_capacity(
-            DEFAULT_RDR_BUFFER_CAPACITY * 4,
-            File::open(stats_binary_encoded_path)?,
-        );
-        let mut buf_binsz_decoder = snap::read::FrameDecoder::new(bin_file);
-        match bincode::deserialize_from(&mut buf_binsz_decoder) {
-            Ok(stats) => {
-                csv_stats = stats;
-                stats_bin_loaded = true;
-            },
-            Err(e) => {
-                wwarn!(
-                    "Error reading stats.csv.bin.sz file: {e:?}. Regenerating stats.bin.sz file."
-                );
-            },
-        }
-    }
-
-    if !stats_bin_loaded {
-        // otherwise, run stats command to generate stats.csv.bin.sz file
-        let tempfile = tempfile::Builder::new()
-            .suffix(".stats.csv")
-            .tempfile()
-            .unwrap();
-        let tempfile_path = tempfile.path().to_str().unwrap().to_string();
-
-        let statsbin_path = canonical_input_path.with_extension("stats.csv.bin.sz");
-
-        let mut stats_args_str = format!(
-            "stats {input} --infer-dates --dates-whitelist {dates_whitelist} --round 4 \
-             --cardinality --output {output} --stats-binout --force",
-            input = {
-                if let Some(arg_input) = stats_args.arg_input.clone() {
-                    arg_input
-                } else {
-                    "-".to_string()
-                }
-            },
-            dates_whitelist = stats_args.flag_dates_whitelist,
-            output = tempfile_path,
-        );
-        if args.flag_prefer_dmy {
-            stats_args_str = format!("{stats_args_str} --prefer-dmy");
-        }
-        if args.flag_no_headers {
-            stats_args_str = format!("{stats_args_str} --no-headers");
-        }
-        if let Some(delimiter) = args.flag_delimiter {
-            let delim = delimiter.as_byte() as char;
-            stats_args_str = format!("{stats_args_str} --delimiter {delim}");
-        }
-        if args.flag_memcheck {
-            stats_args_str = format!("{stats_args_str} --memcheck");
-        }
-        if let Some(mut jobs) = stats_args.flag_jobs {
-            if jobs > 2 {
-                jobs -= 1; // leave one core for the main thread
-            }
-            stats_args_str = format!("{stats_args_str} --jobs {jobs}");
-        }
-
-        let stats_args_vec: Vec<&str> = stats_args_str.split_whitespace().collect();
-
-        let qsv_bin = std::env::current_exe().unwrap();
-        let mut stats_cmd = std::process::Command::new(qsv_bin);
-        stats_cmd.args(stats_args_vec);
-        let _stats_output = stats_cmd.output()?;
-
-        let bin_file =
-            BufReader::with_capacity(DEFAULT_RDR_BUFFER_CAPACITY * 2, File::open(statsbin_path)?);
-        let mut buf_binsz_decoder = snap::read::FrameDecoder::new(bin_file);
-
-        match bincode::deserialize_from(&mut buf_binsz_decoder) {
-            Ok(stats) => {
-                csv_stats = stats;
-            },
-            Err(e) => {
-                return fail_clierror!(
-                    "Error reading stats.csv.bin.sz file: {e:?}. Schema generation aborted."
-                );
-            },
-        }
-    };
-
-    // get the headers from the input file
-    let mut rdr = csv::Reader::from_path(args.arg_input.clone().unwrap()).unwrap();
-    let csv_fields = rdr.byte_headers()?.clone();
-    drop(rdr);
-
-    let stats_columns = if stats_bin_loaded {
-        // if stats.bin file is loaded, we need to get the headers from the stats.csv file
-        let stats_bin_csv_path = canonical_input_path.with_extension("stats.csv");
-        let mut stats_csv_reader = csv::Reader::from_path(stats_bin_csv_path)?;
-        let stats_csv_headers = stats_csv_reader.headers()?.clone();
-        drop(stats_csv_reader);
-        stats_csv_headers
-    } else {
-        // otherwise, we generate the headers from the stats_args struct
-        // as we used the stats_args struct to generate the stats.csv file
-        stats_args.stat_headers()
-    };
-
-    let mut stats_col_index_map = AHashMap::new();
-
-    for (i, col) in stats_columns.iter().enumerate() {
-        if col != "field" {
-            // need offset by 1 due to extra "field" column in headers that's not in stats records
-            stats_col_index_map.insert(col.to_owned(), i - 1);
-        }
-    }
-
-    Ok((csv_fields, csv_stats, stats_col_index_map))
-}
-
 /// get column selector argument string for low cardinality columns
 fn build_low_cardinality_column_selector_arg(
     low_cardinality_column_indices: &mut Vec<usize>,
@@ -634,7 +442,7 @@ fn build_low_cardinality_column_selector_arg(
 /// get frequency tables from `cmd::frequency`
 /// returns map of unique values keyed by header
 fn get_unique_values(
-    args: &Args,
+    args: &util::SchemaArgs,
     column_select_arg: &str,
 ) -> CliResult<AHashMap<String, Vec<String>>> {
     // prepare arg for invoking cmd::frequency
@@ -651,6 +459,8 @@ fn get_unique_values(
         flag_no_nulls:       true,
         flag_no_trim:        false,
         flag_ignore_case:    args.flag_ignore_case,
+        // internal mode for getting frequency tables
+        flag_stats_mode:     "_schema".to_string(),
         flag_jobs:           Some(util::njobs(args.flag_jobs)),
         flag_output:         None,
         flag_no_headers:     args.flag_no_headers,
@@ -731,7 +541,7 @@ fn get_required_fields(properties_map: &Map<String, Value>) -> Vec<Value> {
 
 /// generate map of regex patterns from selected String column of CSV
 fn generate_string_patterns(
-    args: &Args,
+    args: &util::SchemaArgs,
     properties_map: &Map<String, Value>,
 ) -> CliResult<AHashMap<String, String>> {
     // standard boiler-plate for reading CSV
diff --git a/src/cmd/stats.rs b/src/cmd/stats.rs
index 6b1d5d3a0..9f1dae542 100644
--- a/src/cmd/stats.rs
+++ b/src/cmd/stats.rs
@@ -183,12 +183,12 @@ stats options:
                               Note that a file handle is opened for each job.
                               When not set, the number of jobs is set to the
                               number of CPUs detected.
-    --stats-binout            Write the stats in binary format. This is used internally
-                              by other qsv commands (currently `schema` & `tojsonl`) to
-                              load cached stats into memory faster. If set, the snappy compressed
+    --stats-binout            Write the stats in binary format. This is used internally by other
+                              qsv commands (currently `frequency`, `schema` & `tojsonl`) to load
+                              cached stats into memory faster. If set, the snappy compressed
                               binary encoded stats will be written to <FILESTEM>.stats.csv.bin.sz.
-                              You can preemptively create the binary encoded stats file
-                              by using this option BEFORE running the `schema` and `tojsonl`
+                              You can preemptively create the binary encoded stats file by using
+                              this option BEFORE running the `frequency`, `schema` & `tojsonl`
                               commands and they will automatically load the binary encoded
                               stats file if it exists.
  -c, --cache-threshold <arg>  When greater than 1, the threshold in milliseconds before caching
diff --git a/src/cmd/tojsonl.rs b/src/cmd/tojsonl.rs
index b71d65d88..238bc5150 100644
--- a/src/cmd/tojsonl.rs
+++ b/src/cmd/tojsonl.rs
@@ -114,7 +114,7 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
     let record_count = util::count_rows(&conf)?;
 
     // we're calling the schema command to infer data types and enums
-    let schema_args = crate::cmd::schema::Args {
+    let schema_args = util::SchemaArgs {
         // we only do three, as we're only inferring boolean based on enum
         // i.e. we only inspect a field if its boolean if its domain
         // is just two values. if its more than 2, that's all we need know
@@ -166,7 +166,7 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
         args.flag_no_boolean
     };
 
-    let mut lowecase_buffer = String::new();
+    let mut lowercase_buffer = String::new();
 
     // create a vec lookup about inferred field data types
     let mut field_type_vec: Vec<JsonlType> = Vec::with_capacity(headers.len());
@@ -199,7 +199,7 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
                                 }
                             } else if let Some(str_val) = vals[0].as_str() {
                                 // else, if its a string, get the first character of val1 lowercase
-                                boolcheck(str_val, &mut lowecase_buffer)
+                                boolcheck(str_val, &mut lowercase_buffer)
                             } else {
                                 '*'
                             }
@@ -214,7 +214,7 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
                                 _ => '*',
                             }
                         } else if let Some(str_val) = vals[1].as_str() {
-                            boolcheck(str_val, &mut lowecase_buffer)
+                            boolcheck(str_val, &mut lowercase_buffer)
                         } else {
                             '*'
                         };
diff --git a/src/config.rs b/src/config.rs
index 833912429..1f93f40ae 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -95,7 +95,9 @@ pub struct Config {
 }
 
 // Empty trait as an alias for Seek and Read that avoids auto trait errors
+#[cfg(any(feature = "feature_capable", feature = "lite"))]
 pub trait SeekRead: io::Seek + io::Read {}
+#[cfg(any(feature = "feature_capable", feature = "lite"))]
 impl<T: io::Seek + io::Read> SeekRead for T {}
 
 impl Config {
diff --git a/src/util.rs b/src/util.rs
index 87504c179..2a96d6ac8 100644
--- a/src/util.rs
+++ b/src/util.rs
@@ -13,10 +13,12 @@ use std::{
     time::SystemTime,
 };
 
+use ahash::AHashMap;
+use csv::ByteRecord;
 use docopt::Docopt;
 #[cfg(any(feature = "feature_capable", feature = "lite"))]
 use indicatif::{HumanCount, ProgressBar, ProgressDrawTarget, ProgressStyle};
-use log::log_enabled;
+use log::{info, log_enabled, warn};
 use reqwest::Client;
 use serde::de::DeserializeOwned;
 #[cfg(any(feature = "feature_capable", feature = "lite"))]
@@ -27,7 +29,8 @@ use sysinfo::System;
 use crate::cmd::count::polars_count_input;
 use crate::{
     config,
-    config::{Config, Delimiter, DEFAULT_WTR_BUFFER_CAPACITY},
+    config::{Config, Delimiter, DEFAULT_RDR_BUFFER_CAPACITY, DEFAULT_WTR_BUFFER_CAPACITY},
+    select::SelectColumns,
     CliError, CliResult, CURRENT_COMMAND,
 };
 
@@ -46,6 +49,32 @@ static ROW_COUNT: OnceLock<Option<u64>> = OnceLock::new();
 
 pub type ByteString = Vec<u8>;
 
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub enum StatsMode {
+    Schema,
+    Frequency,
+    FrequencyForceStats,
+    None,
+}
+
+#[allow(dead_code)]
+#[derive(serde::Deserialize, Clone)]
+pub struct SchemaArgs {
+    pub flag_enum_threshold:  usize,
+    pub flag_ignore_case:     bool,
+    pub flag_strict_dates:    bool,
+    pub flag_pattern_columns: SelectColumns,
+    pub flag_dates_whitelist: String,
+    pub flag_prefer_dmy:      bool,
+    pub flag_force:           bool,
+    pub flag_stdout:          bool,
+    pub flag_jobs:            Option<usize>,
+    pub flag_no_headers:      bool,
+    pub flag_delimiter:       Option<Delimiter>,
+    pub arg_input:            Option<String>,
+    pub flag_memcheck:        bool,
+}
+
 #[inline]
 pub fn num_cpus() -> usize {
     num_cpus::get()
@@ -1888,6 +1917,206 @@ pub fn trim_bs_whitespace(bytes: &[u8]) -> &[u8] {
     &bytes[start..end]
 }
 
+// get stats records from stats.bin file, or if its invalid, by running the stats command
+/// returns tuple (`csv_fields`, `csv_stats`, `stats_col_index_map`)
+pub fn get_stats_records(
+    args: &SchemaArgs,
+    mode: StatsMode,
+) -> CliResult<(
+    ByteRecord,
+    Vec<crate::cmd::stats::Stats>,
+    AHashMap<String, usize>,
+)> {
+    let stats_args = crate::cmd::stats::Args {
+        arg_input:            args.arg_input.clone(),
+        flag_select:          crate::select::SelectColumns::parse("").unwrap(),
+        flag_everything:      false,
+        flag_typesonly:       false,
+        flag_infer_boolean:   false,
+        flag_mode:            false,
+        flag_cardinality:     true,
+        flag_median:          false,
+        flag_quartiles:       false,
+        flag_mad:             false,
+        flag_nulls:           false,
+        flag_round:           4,
+        flag_infer_dates:     true,
+        flag_dates_whitelist: args.flag_dates_whitelist.to_string(),
+        flag_prefer_dmy:      args.flag_prefer_dmy,
+        flag_force:           args.flag_force,
+        flag_jobs:            Some(njobs(args.flag_jobs)),
+        flag_stats_binout:    true,
+        flag_cache_threshold: 1, // force the creation of stats cache files
+        flag_output:          None,
+        flag_no_headers:      args.flag_no_headers,
+        flag_delimiter:       args.flag_delimiter,
+        flag_memcheck:        args.flag_memcheck,
+    };
+
+    let canonical_input_path = Path::new(&args.arg_input.clone().unwrap()).canonicalize()?;
+    let stats_binary_encoded_path = canonical_input_path.with_extension("stats.csv.bin.sz");
+
+    let stats_bin_current = if stats_binary_encoded_path.exists() {
+        let stats_bin_metadata = std::fs::metadata(&stats_binary_encoded_path)?;
+
+        let input_metadata = std::fs::metadata(args.arg_input.clone().unwrap())?;
+
+        if stats_bin_metadata.modified()? > input_metadata.modified()? {
+            info!("Valid stats.csv.bin.sz file found!");
+            true
+        } else {
+            info!("stats.csv.bin.sz file is older than input file. Regenerating stats.bin file.");
+            false
+        }
+    } else {
+        info!("stats.csv.bin.sz file does not exist: {stats_binary_encoded_path:?}");
+        false
+    };
+
+    if mode == StatsMode::None || (mode == StatsMode::Frequency && !stats_bin_current) {
+        // if the stats.bin file is not present, we're just doing frequency old school
+        // without cardinality
+        return Ok((ByteRecord::new(), Vec::new(), AHashMap::new()));
+    }
+
+    let mut stats_bin_loaded = false;
+
+    // if stats.bin file exists and is current, use it
+    let mut csv_stats: Vec<crate::cmd::stats::Stats> = Vec::new();
+
+    if stats_bin_current && !args.flag_force {
+        let bin_file = BufReader::with_capacity(
+            DEFAULT_RDR_BUFFER_CAPACITY * 4,
+            File::open(stats_binary_encoded_path)?,
+        );
+        let mut buf_binsz_decoder = snap::read::FrameDecoder::new(bin_file);
+        match bincode::deserialize_from(&mut buf_binsz_decoder) {
+            Ok(stats) => {
+                csv_stats = stats;
+                stats_bin_loaded = true;
+            },
+            Err(e) => {
+                wwarn!(
+                    "Error reading stats.csv.bin.sz file: {e:?}. Regenerating stats.bin.sz file."
+                );
+            },
+        }
+    }
+
+    if !stats_bin_loaded {
+        // otherwise, run stats command to generate stats.csv.bin.sz file
+        let tempfile = tempfile::Builder::new()
+            .suffix(".stats.csv")
+            .tempfile()
+            .unwrap();
+        let tempfile_path = tempfile.path().to_str().unwrap().to_string();
+
+        let statsbin_path = canonical_input_path.with_extension("stats.csv.bin.sz");
+
+        let mut stats_args_str = if mode == StatsMode::Schema {
+            // mode is GetStatsMode::Schema
+            // we're generating schema, so we need all the stats
+            format!(
+                "stats {input} --infer-dates --dates-whitelist {dates_whitelist} --round 4 \
+                 --cardinality --output {output} --stats-binout --force",
+                input = {
+                    if let Some(arg_input) = stats_args.arg_input.clone() {
+                        arg_input
+                    } else {
+                        "-".to_string()
+                    }
+                },
+                dates_whitelist = stats_args.flag_dates_whitelist,
+                output = tempfile_path,
+            )
+        } else {
+            // mode is GetStatsMode::Frequency or GetStatsMode::FrequencyForceStats
+            // we're doing frequency, so we just need cardinality
+            format!(
+                "stats {input} --cardinality --stats-binout --output {output}",
+                input = {
+                    if let Some(arg_input) = stats_args.arg_input.clone() {
+                        arg_input
+                    } else {
+                        "-".to_string()
+                    }
+                },
+                output = tempfile_path,
+            )
+        };
+        if args.flag_prefer_dmy {
+            stats_args_str = format!("{stats_args_str} --prefer-dmy");
+        }
+        if args.flag_no_headers {
+            stats_args_str = format!("{stats_args_str} --no-headers");
+        }
+        if let Some(delimiter) = args.flag_delimiter {
+            let delim = delimiter.as_byte() as char;
+            stats_args_str = format!("{stats_args_str} --delimiter {delim}");
+        }
+        if args.flag_memcheck {
+            stats_args_str = format!("{stats_args_str} --memcheck");
+        }
+        if let Some(mut jobs) = stats_args.flag_jobs {
+            if jobs > 2 {
+                jobs -= 1; // leave one core for the main thread
+            }
+            stats_args_str = format!("{stats_args_str} --jobs {jobs}");
+        }
+
+        let stats_args_vec: Vec<&str> = stats_args_str.split_whitespace().collect();
+
+        let qsv_bin = std::env::current_exe().unwrap();
+        let mut stats_cmd = std::process::Command::new(qsv_bin);
+        stats_cmd.args(stats_args_vec);
+        let _stats_output = stats_cmd.output()?;
+
+        let bin_file =
+            BufReader::with_capacity(DEFAULT_RDR_BUFFER_CAPACITY * 2, File::open(statsbin_path)?);
+        let mut buf_binsz_decoder = snap::read::FrameDecoder::new(bin_file);
+
+        match bincode::deserialize_from(&mut buf_binsz_decoder) {
+            Ok(stats) => {
+                csv_stats = stats;
+            },
+            Err(e) => {
+                return fail_clierror!(
+                    "Error reading stats.csv.bin.sz file: {e:?}. Schema generation aborted."
+                );
+            },
+        }
+    };
+
+    // get the headers from the input file
+    let mut rdr = csv::Reader::from_path(args.arg_input.clone().unwrap()).unwrap();
+    let csv_fields = rdr.byte_headers()?.clone();
+    drop(rdr);
+
+    let stats_columns = if stats_bin_loaded {
+        // if stats.bin file is loaded, we need to get the headers from the stats.csv file
+        let stats_bin_csv_path = canonical_input_path.with_extension("stats.csv");
+        let mut stats_csv_reader = csv::Reader::from_path(stats_bin_csv_path)?;
+        let stats_csv_headers = stats_csv_reader.headers()?.clone();
+        drop(stats_csv_reader);
+        stats_csv_headers
+    } else {
+        // otherwise, we generate the headers from the stats_args struct
+        // as we used the stats_args struct to generate the stats.csv file
+        stats_args.stat_headers()
+    };
+
+    let mut stats_col_index_map = AHashMap::new();
+
+    for (i, col) in stats_columns.iter().enumerate() {
+        if col != "field" {
+            // need offset by 1 due to extra "field" column in headers that's not in stats records
+            stats_col_index_map.insert(col.to_owned(), i - 1);
+        }
+    }
+
+    Ok((csv_fields, csv_stats, stats_col_index_map))
+}
+
 // comment out for now as this is still WIP
 // pub fn create_json_record(
 //     no_headers: bool,


Test patch:
diff --git a/tests/test_frequency.rs b/tests/test_frequency.rs
index b583a5b7b..23fa94e29 100644
--- a/tests/test_frequency.rs
+++ b/tests/test_frequency.rs
@@ -111,6 +111,8 @@ fn frequency_trim() {
         .args(["--limit", "0"])
         .args(["--select", "h2"]);
 
+    wrk.assert_success(&mut cmd);
+
     let mut got: Vec<Vec<String>> = wrk.read_stdout(&mut cmd);
     got.sort_unstable();
     let expected = vec![
@@ -471,6 +473,86 @@ fn frequency_all_unique() {
     assert_eq!(got, expected);
 }
 
+#[test]
+fn frequency_all_unique_with_stats_cache() {
+    let wrk = Workdir::new("frequency_all_unique_with_stats_cache");
+    let testdata = wrk.load_test_file("boston311-100.csv");
+
+    let mut stats_cmd = wrk.command("stats");
+    stats_cmd
+        .arg(testdata.clone())
+        .arg("--cardinality")
+        .arg("--stats-binout");
+
+    wrk.assert_success(&mut stats_cmd);
+
+    let mut cmd = wrk.command("frequency");
+    cmd.args(["--select", "1"]).arg(testdata);
+
+    let got: Vec<Vec<String>> = wrk.read_stdout(&mut cmd);
+    let expected = vec![
+        svec!["field", "value", "count", "percentage"],
+        svec!["case_enquiry_id", "ALL_UNIQUE", "100", "100"],
+    ];
+    assert_eq!(got, expected);
+}
+
+#[test]
+fn frequency_all_unique_force_stats_cache() {
+    let wrk = Workdir::new("frequency_all_unique_force_stats_cache");
+    let testdata = wrk.load_test_file("boston311-100.csv");
+
+    let mut cmd = wrk.command("frequency");
+    cmd.args(["--select", "1"])
+        .args(["--stats-mode", "force"])
+        .arg(testdata);
+
+    let got: Vec<Vec<String>> = wrk.read_stdout(&mut cmd);
+    let expected = vec![
+        svec!["field", "value", "count", "percentage"],
+        svec!["case_enquiry_id", "ALL_UNIQUE", "100", "100"],
+    ];
+    assert_eq!(got, expected);
+}
+
+#[test]
+fn frequency_all_unique_stats_mode_none() {
+    let wrk = Workdir::new("frequency_all_unique_stats_mode_none");
+    let testdata = wrk.load_test_file("boston311-100.csv");
+
+    // create stats cache
+    let mut stats_cmd = wrk.command("stats");
+    stats_cmd
+        .arg(testdata.clone())
+        .arg("--cardinality")
+        .arg("--stats-binout");
+
+    wrk.assert_success(&mut stats_cmd);
+
+    // run frequency with stats-mode none, ignoring the stats cache
+    let mut cmd = wrk.command("frequency");
+    cmd.args(["--select", "1"])
+        .args(["--stats-mode", "none"])
+        .arg(testdata);
+
+    let got: Vec<Vec<String>> = wrk.read_stdout(&mut cmd);
+    let expected = vec![
+        svec!["field", "value", "count", "percentage"],
+        svec!["case_enquiry_id", "101004113298", "1", "1"],
+        svec!["case_enquiry_id", "101004113313", "1", "1"],
+        svec!["case_enquiry_id", "101004113348", "1", "1"],
+        svec!["case_enquiry_id", "101004113363", "1", "1"],
+        svec!["case_enquiry_id", "101004113371", "1", "1"],
+        svec!["case_enquiry_id", "101004113385", "1", "1"],
+        svec!["case_enquiry_id", "101004113386", "1", "1"],
+        svec!["case_enquiry_id", "101004113391", "1", "1"],
+        svec!["case_enquiry_id", "101004113394", "1", "1"],
+        svec!["case_enquiry_id", "101004113403", "1", "1"],
+        svec!["case_enquiry_id", "Other (90)", "90", "90"],
+    ];
+    assert_eq!(got, expected);
+}
+
 #[test]
 fn frequency_issue1962() {
     let wrk = Workdir::new("frequency_1962");
@@ -525,7 +607,7 @@ fn prop_frequency() {
     }
     // Run on really small values because we are incredibly careless
     // with allocation.
-    qcheck_sized(p as fn(CsvData) -> bool, 2);
+    qcheck_sized(p as fn(CsvData) -> bool, 5);
 }
 
 // This tests that running the frequency command on a CSV file with these two
@@ -557,7 +639,7 @@ fn prop_frequency_indexed() {
     }
     // Run on really small values because we are incredibly careless
     // with allocation.
-    qcheck_sized(p as fn(CsvData) -> bool, 2);
+    qcheck_sized(p as fn(CsvData) -> bool, 5);
 }
 
 fn param_prop_frequency(name: &str, rows: CsvData, idx: bool) -> bool {