Repository: dathere/qsv (language: rust)

Issue:
feat: "smart" `diff`
When diffing files with a stats cache, compare each files' `qsv_fingerprint_hash` to see if the files are equal and short-circuit the diff. That is, unless the `--force` option is used.


Discussion / hints:

Fix patch:
diff --git a/Cargo.toml b/Cargo.toml
index fcbe4b97b..85f4edab1 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -168,7 +168,7 @@ mlua = { version = "0.10", features = [
 ], optional = true }
 num_cpus = "1"
 odht = "0.3"
-phf = { version = "0.11", features = ["macros"], optional = true }
+phf = { version = "0.11", features = ["macros"]}
 polars = { version = "0.46", features = [
     "asof_join",
     "avro",
@@ -396,7 +396,6 @@ geocode = [
     "cached",
     "geosuggest-core",
     "geosuggest-utils",
-    "phf",
     "sled",
 ]
 luau = ["mlua", "sanitize-filename"]
diff --git a/README.md b/README.md
index 4ad5889f1..d4dfc7ddf 100644
--- a/README.md
+++ b/README.md
@@ -36,7 +36,7 @@
 | [datefmt](/src/cmd/datefmt.rs#L2)<br>📇🚀👆 | Formats recognized date fields ([19 formats recognized](https://docs.rs/qsv-dateparser/latest/qsv_dateparser/#accepted-date-formats)) to a specified date format using [strftime date format specifiers](https://docs.rs/chrono/latest/chrono/format/strftime/). |
 | [dedup](/src/cmd/dedup.rs#L2)<br>🤯🚀👆 | Remove duplicate rows (See also `extdedup`, `extsort`, `sort` & `sortcheck` commands). |
 | [describegpt](/src/cmd/describegpt.rs#L2)<br>🌐🤖🪄 | Infer extended metadata about a CSV using a GPT model from [OpenAI's API](https://platform.openai.com/docs/introduction) or an LLM from another API compatible with the OpenAI API specification such as [Ollama](https://ollama.com) or [Jan](https://jan.ai). |
-| [diff](/src/cmd/diff.rs#L2)<br>🚀 | Find the difference between two CSVs with ludicrous speed!<br/>e.g. _compare two CSVs with 1M rows x 9 columns in under 600ms!_ |
+| [diff](/src/cmd/diff.rs#L2)<br>🚀🪄 | Find the difference between two CSVs with ludicrous speed!<br/>e.g. _compare two CSVs with 1M rows x 9 columns in under 600ms!_ |
 | [edit](/src/cmd/edit.rs#L2) | Replace the value of a cell specified by its row and column. |
 | [enum](/src/cmd/enumerate.rs#L2)<br>👆 | Add a new column enumerating rows by adding a column of incremental or uuid identifiers. Can also be used to copy a column or fill a new column with a constant value.  |
 | [excel](/src/cmd/excel.rs#L2)<br>🚀 | Exports a specified Excel/ODS sheet to a CSV file. |
diff --git a/src/cmd/diff.rs b/src/cmd/diff.rs
index 99229da51..868dfd76b 100644
--- a/src/cmd/diff.rs
+++ b/src/cmd/diff.rs
@@ -3,7 +3,10 @@ Find the difference between two CSVs with ludicrous speed.
 
 NOTE: diff does not support stdin. A file path is required for both arguments.
       Further, PRIMARY KEY VALUES MUST BE UNIQUE WITHIN EACH CSV.
-      Otherwise, diff will produce an incorrect result.
+      When diffing CSVs with just a single --key column and a stats cache is
+      available, diff will automatically validate for primary key uniqueness.
+      If more than one --key column is specified, however, this auto-validation
+      is not done.
 
       To check if a CSV has unique primary key values, use `qsv extdedup`
       with the same key columns using the `--select` option:
@@ -76,7 +79,7 @@ diff options:
     --delimiter-output <arg>    The field delimiter for writing the CSV diff result.
                                 Must be a single character. (default: ,)
     -k, --key <arg...>          The column indices that uniquely identify a record
-                                as a comma separated list of indices, e.g. 0,1,2
+                                as a comma separated list of 0-based indices, e.g. 0,1,2
                                 or column names, e.g. name,age.
                                 Note that when selecting columns by name, only the 
                                 left CSV's headers are used to match the column names
@@ -101,6 +104,12 @@ diff options:
     -j, --jobs <arg>            The number of jobs to run in parallel.
                                 When not set, the number of jobs is set to the number
                                 of CPUs detected.
+    --force                     Force diff and ignore stats caches for the left & right CSVs.
+                                Otherwise, if available, the stats cache will be used to:
+                                 * short-circuit the diff if their fingerprint hashes are
+                                   identical.
+                                 * check for primary key uniqueness when only one --key
+                                   column is specified.
 
 Common options:
     -h, --help                  Display this message
@@ -123,7 +132,10 @@ use super::rename::rename_headers_all_generic;
 use crate::{
     clitypes::CliError,
     config::{Config, Delimiter},
-    util, CliResult,
+    select::SelectColumns,
+    util,
+    util::{get_stats_records, SchemaArgs, StatsMode},
+    CliResult,
 };
 
 #[derive(Deserialize)]
@@ -132,6 +144,7 @@ struct Args {
     arg_input_right:        Option<String>,
     flag_output:            Option<String>,
     flag_jobs:              Option<usize>,
+    flag_force:             bool,
     flag_no_headers_left:   bool,
     flag_no_headers_right:  bool,
     flag_no_headers_output: bool,
@@ -147,6 +160,139 @@ struct Args {
 pub fn run(argv: &[&str]) -> CliResult<()> {
     let mut args: Args = util::get_args(USAGE, argv)?;
 
+    // ===== VALIDATION CHECKS =====
+
+    // If --force is not set and
+    // neither --no-headers-left nor --no-headers-right are set and
+    // the key is a just single column (or unspecified),
+    // perform validation checks on the input files using the stats cache.
+    if !args.flag_force
+        && (!args.flag_no_headers_left && !args.flag_no_headers_right)
+        && args
+            .flag_key
+            .as_ref()
+            .is_none_or(|k| k.split(',').count() == 0)
+    {
+        // ---- STATS CACHE VALIDATION CHECKS ----
+
+        // Get stats for left file
+        let left_schema_args = SchemaArgs {
+            arg_input:            args.arg_input_left.clone(),
+            flag_no_headers:      false,
+            flag_delimiter:       args.flag_delimiter,
+            flag_jobs:            None,
+            flag_memcheck:        false,
+            flag_force:           args.flag_force,
+            flag_prefer_dmy:      false,
+            flag_dates_whitelist: String::new(),
+            flag_enum_threshold:  0,
+            flag_ignore_case:     false,
+            flag_strict_dates:    false,
+            flag_pattern_columns: SelectColumns::parse("")?,
+            flag_stdout:          false,
+        };
+
+        // Get stats for right file using same args
+        let right_schema_args = SchemaArgs {
+            arg_input: args.arg_input_right.clone(),
+            ..left_schema_args.clone()
+        };
+
+        // Get stats records for both files
+        if let (
+            Ok((left_csv_fields, left_stats, left_dataset_stats)),
+            Ok((_, right_stats, right_dataset_stats)),
+        ) = (
+            get_stats_records(&left_schema_args, StatsMode::FrequencyForceStats),
+            get_stats_records(&right_schema_args, StatsMode::FrequencyForceStats),
+        ) {
+            // If both files fingerprint hashes match, files are identical short-circuit diff
+            if left_dataset_stats.get("qsv__fingerprint_hash")
+                == right_dataset_stats.get("qsv__fingerprint_hash")
+            {
+                return Ok(());
+            }
+
+            // Check if row counts match
+            let left_dataset_rowcount = left_dataset_stats
+                .get("qsv__rowcount")
+                .unwrap()
+                .parse::<f64>()
+                .unwrap_or_default() as u64;
+            let right_dataset_rowcount = right_dataset_stats
+                .get("qsv__rowcount")
+                .unwrap()
+                .parse::<f64>()
+                .unwrap_or_default() as u64;
+
+            if left_dataset_rowcount != right_dataset_rowcount {
+                return fail_incorrectusage_clierror!(
+                    "The number of rows in the left ({left_dataset_rowcount}) and right \
+                     ({right_dataset_rowcount}) CSVs do not match."
+                );
+            }
+
+            // If key column specified, check if it has all unique values in both files
+            let mut colname_used_for_key = false;
+            if let Some(key_col) = &args.flag_key {
+                let idx = if key_col.chars().all(char::is_numeric) {
+                    key_col
+                        .parse::<usize>()
+                        .map_err(|err| CliError::Other(err.to_string()))?
+                } else {
+                    // Handle column name case...
+                    colname_used_for_key = true;
+                    left_csv_fields
+                        .iter()
+                        .position(|field| field == key_col.as_bytes())
+                        .unwrap_or_default()
+                };
+
+                // Check cardinality equals row count for key column in left file
+                if let Some(left_col) = left_stats.get(idx) {
+                    if left_col.cardinality != left_dataset_rowcount {
+                        return fail_incorrectusage_clierror!(
+                            "Primary key values in left CSV are not unique in column {colname} \
+                             (cardinality: {left_cardinality} != rowcount: {left_rowcount}). Use \
+                             `qsv extdedup --select {colname} {left_input} --no-output` to check \
+                             duplicates.",
+                            colname = if colname_used_for_key {
+                                key_col.to_string()
+                            } else {
+                                idx.to_string()
+                            },
+                            left_cardinality = left_col.cardinality,
+                            left_rowcount = left_dataset_rowcount,
+                            left_input = args.arg_input_left.as_ref().unwrap()
+                        );
+                    }
+                }
+
+                // Check cardinality equals row count for key column in right file
+                if let Some(right_col) = right_stats.get(idx) {
+                    if right_col.cardinality != right_dataset_rowcount {
+                        return fail_incorrectusage_clierror!(
+                            "Primary key values in right CSV are not unique in column {colname} \
+                             (cardinality: {right_cardinality} != rowcount: {right_rowcount}). \
+                             Use `qsv extdedup --select {colname} {right_input} --no-output` to \
+                             check duplicates.",
+                            colname = if colname_used_for_key {
+                                key_col.to_string()
+                            } else {
+                                idx.to_string()
+                            },
+                            right_cardinality = right_col.cardinality,
+                            right_rowcount = right_dataset_rowcount,
+                            right_input = args.arg_input_right.as_ref().unwrap()
+                        );
+                    }
+                }
+            }
+        }
+    }
+
+    // ---- SETUP and OTHER VALIDATION CHECKS ----
+
     if let Some(delim) = args.flag_delimiter {
         [
             args.flag_delimiter_left,
@@ -241,6 +387,8 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
 
     util::njobs(args.flag_jobs);
 
+    // ===== DIFF PROCESSING =====
+
     let Ok(csv_diff) = CsvByteDiffBuilder::new()
         .primary_key_columns(primary_key_cols.clone())
         .build()
diff --git a/src/cmd/frequency.rs b/src/cmd/frequency.rs
index 7a2c1e098..85e8dc226 100644
--- a/src/cmd/frequency.rs
+++ b/src/cmd/frequency.rs
@@ -537,7 +537,7 @@ impl Args {
             "_schema" => StatsMode::Schema, // only meant for internal use by schema command
             _ => return fail_incorrectusage_clierror!("Invalid stats mode"),
         };
-        let (csv_fields, csv_stats) = get_stats_records(&schema_args, stats_mode)?;
+        let (csv_fields, csv_stats, dataset_stats) = get_stats_records(&schema_args, stats_mode)?;
 
         if stats_mode == StatsMode::None || stats_mode == StatsMode::Schema || csv_fields.is_empty()
         {
@@ -552,8 +552,11 @@ impl Args {
         // in the following hot iterator loop
         assert!(
             csv_fields.len() == csv_stats.len(),
-            "Mismatch between the number of fields and stats records"
+            "Mismatch between the number of fields: {} and stats records: {}",
+            csv_fields.len(),
+            csv_stats.len()
         );
+
         let col_cardinality_vec: Vec<(String, u64)> = csv_stats
             .iter()
             .enumerate()
@@ -571,7 +574,11 @@ impl Args {
             .collect();
 
         // now, get the unique headers, where cardinality == rowcount
-        let row_count = util::count_rows(&self.rconfig())?;
+        let row_count = if let Some(row_count) = dataset_stats.get("qsv__rowcount") {
+            row_count.parse::<u64>().unwrap()
+        } else {
+            util::count_rows(&self.rconfig())?
+        };
         FREQ_ROW_COUNT.set(row_count as u64).unwrap();
 
         // Most datasets have relatively few columns with all unique values (e.g. ID columns)
diff --git a/src/cmd/joinp.rs b/src/cmd/joinp.rs
index 320dfdec9..b43a11d43 100644
--- a/src/cmd/joinp.rs
+++ b/src/cmd/joinp.rs
@@ -887,7 +887,7 @@ impl Args {
                 flag_memcheck:        false,
             };
 
-            let (csv_fields, csv_stats) =
+            let (csv_fields, csv_stats, _) =
                 get_stats_records(&schema_args, util::StatsMode::PolarsSchema)?;
 
             let mut schema = Schema::with_capacity(csv_stats.len());
diff --git a/src/cmd/pivotp.rs b/src/cmd/pivotp.rs
index 3c303e2e4..665efedf1 100644
--- a/src/cmd/pivotp.rs
+++ b/src/cmd/pivotp.rs
@@ -62,7 +62,7 @@ Common options:
     -q, --quiet             Do not return smart aggregation chosen nor pivot result shape to stderr.
 "#;
 
-use std::{fs::File, io, io::Write, path::Path, sync::OnceLock};
+use std::{collections::HashMap, fs::File, io, io::Write, path::Path, sync::OnceLock};
 
 use csv::ByteRecord;
 use indicatif::HumanCount;
@@ -78,7 +78,8 @@ use crate::{
     CliResult,
 };
 
-static STATS_RECORDS: OnceLock<(ByteRecord, Vec<StatsData>)> = OnceLock::new();
+static STATS_RECORDS: OnceLock<(ByteRecord, Vec<StatsData>, HashMap<String, String>)> =
+    OnceLock::new();
 
 #[derive(Deserialize)]
 struct Args {
@@ -128,9 +129,10 @@ fn calculate_pivot_metadata(
         flag_memcheck:        false,
     };
 
-    let (csv_fields, csv_stats) = STATS_RECORDS.get_or_init(|| {
+    #[allow(unused_variables)]
+    let (csv_fields, csv_stats, dataset_stats) = STATS_RECORDS.get_or_init(|| {
         get_stats_records(&schema_args, StatsMode::FrequencyForceStats)
-            .unwrap_or_else(|_| (ByteRecord::new(), Vec::new()))
+            .unwrap_or_else(|_| (ByteRecord::new(), Vec::new(), HashMap::new()))
     });
 
     if csv_stats.is_empty() {
@@ -231,14 +233,17 @@ fn suggest_agg_function(
         flag_memcheck:        false,
     };
 
-    let (csv_fields, csv_stats) = STATS_RECORDS.get_or_init(|| {
+    let (csv_fields, csv_stats, dataset_stats) = STATS_RECORDS.get_or_init(|| {
         get_stats_records(&schema_args, StatsMode::FrequencyForceStats)
-            .unwrap_or_else(|_| (ByteRecord::new(), Vec::new()))
+            .unwrap_or_else(|_| (ByteRecord::new(), Vec::new(), HashMap::new()))
     });
 
     let rconfig = Config::new(Some(&args.arg_input));
-    let row_count = util::count_rows(&rconfig)? as u64;
-    // eprintln!("row_count: {}\nstats: {:#?}", row_count, csv_stats);
+    let row_count = if let Some(row_count) = dataset_stats.get("qsv__rowcount") {
+        row_count.parse::<u64>().unwrap()
+    } else {
+        util::count_rows(&rconfig)?
+    };
 
     // Analyze pivot column characteristics
     let mut high_cardinality_pivot = false;
diff --git a/src/cmd/schema.rs b/src/cmd/schema.rs
index ad44dc33e..8bb4c04db 100644
--- a/src/cmd/schema.rs
+++ b/src/cmd/schema.rs
@@ -213,7 +213,7 @@ pub fn infer_schema_from_stats(
     quiet: bool,
 ) -> CliResult<Map<String, Value>> {
     // invoke cmd::stats
-    let (csv_fields, csv_stats) = util::get_stats_records(args, StatsMode::Schema)?;
+    let (csv_fields, csv_stats, _) = util::get_stats_records(args, StatsMode::Schema)?;
 
     // amortize memory allocation
     let mut low_cardinality_column_indices: Vec<u64> =
diff --git a/src/cmd/sqlp.rs b/src/cmd/sqlp.rs
index 594230608..e5c358ae1 100644
--- a/src/cmd/sqlp.rs
+++ b/src/cmd/sqlp.rs
@@ -805,7 +805,7 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
                         arg_input:            Some(table.to_string_lossy().into_owned()),
                         flag_memcheck:        false,
                     };
-                    let (csv_fields, csv_stats) =
+                    let (csv_fields, csv_stats, _) =
                         get_stats_records(&schema_args, util::StatsMode::PolarsSchema)?;
 
                     let mut schema = Schema::with_capacity(csv_stats.len());
diff --git a/src/cmd/stats.rs b/src/cmd/stats.rs
index 6405914ee..22f0239e7 100644
--- a/src/cmd/stats.rs
+++ b/src/cmd/stats.rs
@@ -263,6 +263,7 @@ use std::{
 
 use crossbeam_channel;
 use itertools::Itertools;
+use phf::phf_map;
 use qsv_dateparser::parse_with_preference;
 use serde::{Deserialize, Serialize};
 use simd_json::{prelude::ValueAsScalar, OwnedValue};
@@ -398,6 +399,7 @@ pub struct StatsData {
     // so we escape it as r#type
     // we need to do this for serde to work
     pub r#type:               String,
+    #[serde(default)]
     pub is_ascii:             bool,
     pub sum:                  Option<f64>,
     pub min:                  Option<String>,
@@ -448,51 +450,52 @@ pub enum JsonTypes {
 
 // we use this to serialize the StatsData data structure
 // to a JSONL file using serde_json
-const STATSDATA_TYPES_ARRAY: [JsonTypes; MAX_STAT_COLUMNS] = [
-    JsonTypes::String, //field
-    JsonTypes::String, //type
-    JsonTypes::Bool,   //is_ascii
-    JsonTypes::Float,  //sum
-    JsonTypes::String, //min
-    JsonTypes::String, //max
-    JsonTypes::Float,  //range
-    JsonTypes::String, //sort_order
-    JsonTypes::Float,  //sortiness
-    JsonTypes::Int,    //min_length
-    JsonTypes::Int,    //max_length
-    JsonTypes::Int,    //sum_length
-    JsonTypes::Float,  //avg_length
-    JsonTypes::Float,  //stddev_length
-    JsonTypes::Float,  //variance_length
-    JsonTypes::Float,  //cv_length
-    JsonTypes::Float,  //mean
-    JsonTypes::Float,  //sem
-    JsonTypes::Float,  //geometric_mean
-    JsonTypes::Float,  //harmonic_mean
-    JsonTypes::Float,  //stddev
-    JsonTypes::Float,  //variance
-    JsonTypes::Float,  //cv
-    JsonTypes::Int,    //nullcount
-    JsonTypes::Int,    //max_precision
-    JsonTypes::Float,  //sparsity
-    JsonTypes::Float,  //mad
-    JsonTypes::Float,  //lower_outer_fence
-    JsonTypes::Float,  //lower_inner_fence
-    JsonTypes::Float,  //q1
-    JsonTypes::Float,  //q2_median
-    JsonTypes::Float,  //q3
-    JsonTypes::Float,  //iqr
-    JsonTypes::Float,  //upper_inner_fence
-    JsonTypes::Float,  //upper_outer_fence
-    JsonTypes::Float,  //skewness
-    JsonTypes::Int,    //cardinality
-    JsonTypes::String, //mode
-    JsonTypes::Int,    //mode_count
-    JsonTypes::Int,    //mode_occurrences
-    JsonTypes::String, //antimode
-    JsonTypes::Int,    //antimode_count
-    JsonTypes::Int,    //antimode_occurrences
-];
+pub static STATSDATA_TYPES_MAP: phf::Map<&'static str, JsonTypes> = phf_map! {
+    "field" => JsonTypes::String,
+    "type" => JsonTypes::String,
+    "is_ascii" => JsonTypes::Bool,
+    "sum" => JsonTypes::Float,
+    "min" => JsonTypes::String,
+    "max" => JsonTypes::String,
+    "range" => JsonTypes::Float,
+    "sort_order" => JsonTypes::String,
+    "sortiness" => JsonTypes::Float,
+    "min_length" => JsonTypes::Int,
+    "max_length" => JsonTypes::Int,
+    "sum_length" => JsonTypes::Int,
+    "avg_length" => JsonTypes::Float,
+    "stddev_length" => JsonTypes::Float,
+    "variance_length" => JsonTypes::Float,
+    "cv_length" => JsonTypes::Float,
+    "mean" => JsonTypes::Float,
+    "sem" => JsonTypes::Float,
+    "geometric_mean" => JsonTypes::Float,
+    "harmonic_mean" => JsonTypes::Float,
+    "stddev" => JsonTypes::Float,
+    "variance" => JsonTypes::Float,
+    "cv" => JsonTypes::Float,
+    "nullcount" => JsonTypes::Int,
+    "max_precision" => JsonTypes::Int,
+    "sparsity" => JsonTypes::Float,
+    "mad" => JsonTypes::Float,
+    "lower_outer_fence" => JsonTypes::Float,
+    "lower_inner_fence" => JsonTypes::Float,
+    "q1" => JsonTypes::Float,
+    "q2_median" => JsonTypes::Float,
+    "q3" => JsonTypes::Float,
+    "iqr" => JsonTypes::Float,
+    "upper_inner_fence" => JsonTypes::Float,
+    "upper_outer_fence" => JsonTypes::Float,
+    "skewness" => JsonTypes::Float,
+    "cardinality" => JsonTypes::Int,
+    "mode" => JsonTypes::String,
+    "mode_count" => JsonTypes::Int,
+    "mode_occurrences" => JsonTypes::Int,
+    "antimode" => JsonTypes::String,
+    "antimode_count" => JsonTypes::Int,
+    "antimode_occurrences" => JsonTypes::Int,
+    "qsv__value" => JsonTypes::Int,
+};
 
 static INFER_DATE_FLAGS: OnceLock<SmallVec<[bool; 50]>> = OnceLock::new();
 static RECORD_COUNT: OnceLock<u64> = OnceLock::new();
@@ -523,11 +526,6 @@ const MAX_ANTIMODES: usize = 10;
 const DEFAULT_ANTIMODES_LEN: usize = 100;
 pub const DEFAULT_MODES_SEPARATOR: &str = "|";
 
-// we do this so this is evaluated at compile-time
-pub const fn get_stats_data_types() -> [JsonTypes; MAX_STAT_COLUMNS] {
-    STATSDATA_TYPES_ARRAY
-}
-
 pub fn run(argv: &[&str]) -> CliResult<()> {
     let mut args: Args = util::get_args(USAGE, argv)?;
     if args.flag_typesonly {
@@ -1010,7 +1008,7 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
                 stats_jsonl_pathbuf.set_extension("data.jsonl");
                 util::csv_to_jsonl(
                     &currstats_filename,
-                    &get_stats_data_types(),
+                    &STATSDATA_TYPES_MAP,
                     &stats_jsonl_pathbuf,
                 )?;
             }
diff --git a/src/util.rs b/src/util.rs
index 247679b81..48a683fcc 100644
--- a/src/util.rs
+++ b/src/util.rs
@@ -4,6 +4,7 @@ use std::borrow::Cow;
 use std::os::unix::process::ExitStatusExt;
 use std::{
     cmp::min,
+    collections::HashMap,
     env,
     fmt::Write as _,
     fs,
@@ -31,7 +32,7 @@ use sysinfo::System;
 #[cfg(feature = "polars")]
 use crate::cmd::count::polars_count_input;
 use crate::{
-    cmd::stats::{get_stats_data_types, JsonTypes, StatsData},
+    cmd::stats::{JsonTypes, StatsData, STATSDATA_TYPES_MAP},
     config,
     config::{Config, Delimiter, DEFAULT_RDR_BUFFER_CAPACITY, DEFAULT_WTR_BUFFER_CAPACITY},
     select::SelectColumns,
@@ -2065,7 +2066,7 @@ pub fn write_json_record<W: std::io::Write>(
 pub fn get_stats_records(
     args: &SchemaArgs,
     mode: StatsMode,
-) -> CliResult<(ByteRecord, Vec<StatsData>)> {
+) -> CliResult<(ByteRecord, Vec<StatsData>, HashMap<String, String>)> {
     const DATASET_STATS_PREFIX: &str = r#"{"field":"qsv__"#;
 
     if mode == StatsMode::None
@@ -2074,7 +2075,7 @@ pub fn get_stats_records(
     {
         // if stdin or StatsMode::None,
         // we're just doing frequency old school w/o cardinality
-        return Ok((ByteRecord::new(), Vec::new()));
+        return Ok((ByteRecord::new(), Vec::new(), HashMap::new()));
     }
 
     let canonical_input_path = Path::new(args.arg_input.as_ref().unwrap()).canonicalize()?;
@@ -2102,7 +2103,7 @@ pub fn get_stats_records(
     if mode == StatsMode::Frequency && !stats_data_current {
         // if the stats.data file is not current,
         // we're also doing frequency old school w/o cardinality
-        return Ok((ByteRecord::new(), Vec::new()));
+        return Ok((ByteRecord::new(), Vec::new(), HashMap::new()));
     }
 
     // get the headers from the input file
@@ -2112,6 +2113,7 @@ pub fn get_stats_records(
 
     let mut stats_data_loaded = false;
     let mut csv_stats: Vec<StatsData> = Vec::with_capacity(csv_fields.len());
+    let mut dataset_stats: HashMap<String, String> = HashMap::with_capacity(4);
 
     // if stats_data file exists and is current, use it
     if stats_data_current && !args.flag_force {
@@ -2120,14 +2122,30 @@ pub fn get_stats_records(
 
         let mut curr_line: String;
         let mut s_slice: Vec<u8>;
+
         for line in statsdatajson_rdr.lines() {
             curr_line = line?;
-            if curr_line.starts_with(DATASET_STATS_PREFIX) {
-                break;
-            }
             s_slice = curr_line.as_bytes().to_vec();
-            if let Ok(stats) = simd_json::serde::from_slice(&mut s_slice) {
-                csv_stats.push(stats);
+            if curr_line.starts_with(DATASET_STATS_PREFIX) {
+                // Parse dataset stats record
+                let v: serde_json::Value = simd_json::serde::from_slice(&mut s_slice).unwrap();
+                let field = &v["field"];
+                let value = v["qsv__value"].clone();
+
+                dataset_stats.insert(
+                    field
+                        .as_str()
+                        .unwrap_or_default()
+                        .trim_matches('"')
+                        .to_string(),
+                    value.to_string(),
+                );
+            } else {
+                // Parse regular stats record
+                match simd_json::from_slice::<StatsData>(&mut s_slice) {
+                    Ok(stats) => csv_stats.push(stats),
+                    Err(e) => eprintln!("error parsing stats: {e}"),
+                }
             }
         }
         stats_data_loaded = !csv_stats.is_empty();
@@ -2269,8 +2287,8 @@ pub fn get_stats_records(
             }
         }
 
-        // create a statsdatajon from the output of the stats command
-        csv_to_jsonl(&tempfile_path, &get_stats_data_types(), statsdatajson_path)?;
+        // create a stats data jsonl from the output of the stats command
+        csv_to_jsonl(&tempfile_path, &STATSDATA_TYPES_MAP, statsdatajson_path)?;
 
         let statsdatajson_rdr =
             BufReader::with_capacity(DEFAULT_RDR_BUFFER_CAPACITY, File::open(statsdatajson_path)?);
@@ -2279,32 +2297,44 @@ pub fn get_stats_records(
         let mut s_slice: Vec<u8>;
         for line in statsdatajson_rdr.lines() {
             curr_line = line?;
-            if curr_line.starts_with(DATASET_STATS_PREFIX) {
-                break;
-            }
             s_slice = curr_line.as_bytes().to_vec();
-            if let Ok(stats) = simd_json::serde::from_slice(&mut s_slice) {
-                csv_stats.push(stats);
+            if curr_line.starts_with(DATASET_STATS_PREFIX) {
+                // Parse dataset stats record
+                let v: serde_json::Value = simd_json::serde::from_slice(&mut s_slice).unwrap();
+                let field = &v["field"];
+                let value = v["qsv__value"].clone();
+
+                dataset_stats.insert(
+                    field
+                        .as_str()
+                        .unwrap_or_default()
+                        .trim_matches('"')
+                        .to_string(),
+                    value.to_string(),
+                );
+            } else {
+                // Parse regular stats record
+                match simd_json::from_slice::<StatsData>(&mut s_slice) {
+                    Ok(stats) => csv_stats.push(stats),
+                    Err(e) => eprintln!("error parsing stats: {e}"),
+                }
             }
         }
     }
 
-    Ok((csv_fields, csv_stats))
+    Ok((csv_fields, csv_stats, dataset_stats))
 }
 
-/// simple helper to convert a CSV file to a JSONL file
-/// no type inferencing is done unlike tojsonl, so all fields are strings
 pub fn csv_to_jsonl(
     input_csv: &str,
-    csv_types: &[JsonTypes],
+    csv_types: &phf::Map<&'static str, JsonTypes>,
     output_jsonl: &PathBuf,
 ) -> CliResult<()> {
     let file = File::open(input_csv)?;
     let mut rdr = csv::ReaderBuilder::new()
-        .has_headers(true) // requires headers for keys
+        .has_headers(true)
         .from_reader(file);
 
-    // Get the headers and create a vector of of keys
     let headers = rdr.headers()?;
     let key_vec: Vec<String> = headers
         .iter()
@@ -2314,36 +2344,47 @@ pub fn csv_to_jsonl(
     let output = File::create(output_jsonl)?;
     let mut writer = BufWriter::new(output);
 
-    // amortize allocations
     let mut json_object = serde_json::Map::with_capacity(key_vec.len());
     let mut record = csv::StringRecord::new();
     let mut json_line: String;
 
-    // Iterate over each record in the CSV
     while rdr.read_record(&mut record)? {
         json_object.clear();
 
-        // safety: we know the record length is the same as the key_vec length
         for (i, val) in record.iter().enumerate() {
             let key = unsafe { key_vec.get_unchecked(i) };
-            let data_type = if key == "cardinality" {
-                &JsonTypes::Int
-            } else {
-                csv_types.get(i).unwrap_or(&JsonTypes::String)
-            };
-            let value = if val.is_empty() && data_type != &JsonTypes::Bool {
+            let data_type = csv_types.get(key).unwrap_or(&JsonTypes::String);
+            let value = if val.is_empty() {
                 continue;
             } else {
                 match *data_type {
                     JsonTypes::String => serde_json::Value::String(val.to_owned()),
                     JsonTypes::Int => {
-                        let num = val.parse::<u64>().unwrap_or_default();
-                        serde_json::Value::Number(serde_json::Number::from(num))
+                        if let Ok(num) = val.parse::<u64>() {
+                            serde_json::Value::Number(serde_json::Number::from(num))
+                        } else {
+                            serde_json::Value::String(val.to_owned())
+                        }
+                    },
+                    JsonTypes::Float => {
+                        if let Ok(num) = val.parse::<f64>() {
+                            if let Some(n) = serde_json::Number::from_f64(num) {
+                                serde_json::Value::Number(n)
+                            } else {
+                                serde_json::Value::Number(
+                                    serde_json::Number::from_f64(0.0).unwrap_or_else(|| {
+                                        serde_json::Number::from_f64(0.0).unwrap()
+                                    }),
+                                )
+                            }
+                        } else {
+                            // serde_json::Value::String(val.to_owned())
+                            serde_json::Value::Number(
+                                serde_json::Number::from_f64(0.0)
+                                    .unwrap_or_else(|| serde_json::Number::from_f64(0.0).unwrap()),
+                            )
+                        }
                     },
-                    JsonTypes::Float => serde_json::Value::Number(
-                        serde_json::Number::from_f64(val.parse::<f64>().unwrap_or_default())
-                            .unwrap_or_else(|| serde_json::Number::from(0)),
-                    ),
                     JsonTypes::Bool => {
                         serde_json::Value::Bool(val.parse::<bool>().unwrap_or(false))
                     },


Test patch:
diff --git a/tests/test_diff.rs b/tests/test_diff.rs
index a1b231948..85dca2d94 100644
--- a/tests/test_diff.rs
+++ b/tests/test_diff.rs
@@ -92,6 +92,8 @@ fn diff_diff_left_and_original_right_sort_diff_result_by_lines_by_default() {
     let mut cmd = wrk.command("diff");
     cmd.arg(test_file).arg(test_file2);
 
+    wrk.assert_success(&mut *&mut cmd);
+
     let got: Vec<Vec<String>> = wrk.read_stdout(&mut cmd);
 
     let diff_result_file_name = "diff_result_diff_left_original_right.csv";
@@ -146,6 +148,8 @@ fn diff_sort_diff_result_by_lines_by_default_modified_rows_interleaved() {
     let mut cmd = wrk.command("diff");
     cmd.args(["left.csv", "right.csv"]);
 
+    wrk.assert_success(&mut cmd);
+
     let got: Vec<Vec<String>> = wrk.read_stdout(&mut cmd);
     let expected: Vec<Vec<String>> = vec![
         svec!["diffresult", "h1", "h2", "h3"],
@@ -265,6 +269,8 @@ fn diff_different_delimiters_sort_diff_result_by_first_column() {
         ";",
     ]);
 
+    wrk.assert_success(&mut cmd);
+
     let got: Vec<Vec<String>> = wrk.read_stdout(&mut cmd);
 
     let diff_result_file_name = "diff_result_original_left_diff_right_sort_columns.csv";
@@ -364,6 +370,8 @@ fn diff_key_sort_by_column_name() {
         "h1,h3,h2",
     ]);
 
+    wrk.assert_success(&mut cmd);
+
     let got: Vec<Vec<String>> = wrk.read_stdout(&mut cmd);
     let expected: Vec<Vec<String>> = vec![
         svec!["diffresult", "h1", "h2", "h3"],
@@ -445,6 +453,8 @@ fn diff_only_left_has_headers_headers_in_result() {
     let mut cmd = wrk.command("diff");
     cmd.args(["left.csv", "right.csv", "--no-headers-right"]);
 
+    wrk.assert_success(&mut cmd);
+
     let got: Vec<Vec<String>> = wrk.read_stdout(&mut cmd);
     let expected = vec![
         svec!["diffresult", "h1", "h2", "h3"],
@@ -468,6 +478,8 @@ fn diff_only_right_has_headers_headers_in_result() {
     let mut cmd = wrk.command("diff");
     cmd.args(["left.csv", "right.csv", "--no-headers-left"]);
 
+    wrk.assert_success(&mut cmd);
+
     let got: Vec<Vec<String>> = wrk.read_stdout(&mut cmd);
     let expected = vec![
         svec!["diffresult", "h1", "h2", "h3"],
@@ -552,6 +564,8 @@ fn diff_no_diff_with_generic_headers_in_result() {
         "--no-headers-right",
     ]);
 
+    wrk.assert_success(&mut cmd);
+
     let got: Vec<Vec<String>> = wrk.read_stdout(&mut cmd);
     let expected = vec![svec!["diffresult", "_col_1", "_col_2", "_col_3",]];
 
@@ -665,6 +679,8 @@ fn diff_drop_equal_fields_flag_on_modified_rows_one_row_modified() {
     let mut cmd = wrk.command("diff");
     cmd.args(["left.csv", "right.csv", "--drop-equal-fields"]);
 
+    wrk.assert_success(&mut cmd);
+
     let got: String = wrk.stdout(&mut cmd);
     let expected = "\
 diffresult,h1,h2,h3
@@ -772,6 +788,8 @@ fn diff_drop_equal_fields_flag_on_modified_rows_multiple_key_fields_far_apart()
     // here, first and last columns are our key fields
     cmd.args(["left.csv", "right.csv", "--drop-equal-fields", "-k", "0,3"]);
 
+    wrk.assert_success(&mut cmd);
+
     let got: String = wrk.stdout(&mut cmd);
     let expected = "\
 diffresult,h1,h2,h3,h4
@@ -870,6 +888,8 @@ fn diff_with_mixed_delimiters() {
         "right.csv",
     ]);
 
+    wrk.assert_success(&mut cmd);
+
     let got: String = wrk.stdout(&mut cmd);
     let expected = "\
 diffresult|h1|h2|h3
diff --git a/tests/test_frequency.rs b/tests/test_frequency.rs
index 7160b434d..41e84b1e3 100644
--- a/tests/test_frequency.rs
+++ b/tests/test_frequency.rs
@@ -455,6 +455,8 @@ fn frequency_all_unique() {
     let mut cmd = wrk.command("frequency");
     cmd.args(["--select", "1"]).arg(testdata);
 
+    wrk.assert_success(&mut cmd);
+
     let got: Vec<Vec<String>> = wrk.read_stdout(&mut cmd);
     let expected = vec![
         svec!["field", "value", "count", "percentage"],