Repository: dathere/qsv (language: rust)

Issue:
`stats`: rename `--stats-json` option to `--stats-jsonl`
As it actually creates a jsonl file.

Also the name of the generated json file should have the `.jsonl` extension.


Discussion / hints:

Fix patch:
diff --git a/src/cmd/schema.rs b/src/cmd/schema.rs
index fc30ce521..ddc8c9a2d 100644
--- a/src/cmd/schema.rs
+++ b/src/cmd/schema.rs
@@ -21,9 +21,9 @@ a `stdin.csv` file will created with stdin's contents as well.
 Note that `stdin.csv` will be overwritten if it already exists.
 
 Schema generation can be a compute-intensive process, especially for large CSV files.
-To speed up generation, the `schema` command will reuse a `stats.csv.data.json` file if it
+To speed up generation, the `schema` command will reuse a `stats.csv.data.jsonl` file if it
 exists and is current (i.e. stats generated with --cardinality and --infer-dates options).
-Otherwise, it will run the `stats` command to generate the `stats.csv.data.json` file first,
+Otherwise, it will run the `stats` command to generate the `stats.csv.data.jsonl` file first,
 and then use that to generate the schema file.
 
 For examples, see https://github.com/jqnatividad/qsv/blob/master/tests/test_schema.rs.
diff --git a/src/cmd/stats.rs b/src/cmd/stats.rs
index 00632666c..4e0277cf1 100644
--- a/src/cmd/stats.rs
+++ b/src/cmd/stats.rs
@@ -51,9 +51,9 @@ chunks and each chunk is processed in parallel. The number of chunks is determin
 number of logical CPUs detected. You can override this by setting the --jobs option.
 
 As stats is a central command in qsv, and can be expensive to compute, `stats` caches results
-in <FILESTEM>.stats.csv & if the --stats-json option is used, <FILESTEM>.stats.csv.data.json 
-(e.g., qsv stats nyc311.csv will create nyc311.stats.csv & nyc311.stats.csv.data.json).
-The arguments used to generate the cached stats are saved in <FILESTEM>.stats.csv.json.
+in <FILESTEM>.stats.csv & if the --stats-json option is used, <FILESTEM>.stats.csv.data.jsonl
+(e.g., qsv stats nyc311.csv will create nyc311.stats.csv & nyc311.stats.csv.data.jsonl).
+The arguments used to generate the cached stats are saved in <FILESTEM>.stats.csv.jsonl.
 
 If stats have already been computed for the input file with similar arguments and the file
 hasn't changed, the stats will be loaded from the cache instead of recomputing it.
@@ -182,11 +182,11 @@ stats options:
                               Note that a file handle is opened for each job.
                               When not set, the number of jobs is set to the
                               number of CPUs detected.
-    --stats-json              Also write the stats in json format. 
-                              If set, the stats will be written to <FILESTEM>.stats.csv.data.json.
+    --stats-jsonl             Also write the stats in JSONL format. 
+                              If set, the stats will be written to <FILESTEM>.stats.csv.data.jsonl.
                               Note that this option used internally by other qsv commands
                               (currently `frequency`, `schema` & `tojsonl`) to load cached stats. 
-                              You can preemptively create the stats-json file by using
+                              You can preemptively create the stats-jsonl file by using
                               this option BEFORE running the `frequency`, `schema` & `tojsonl`
                               commands and they will automatically use it.
  -c, --cache-threshold <arg>  When greater than 1, the threshold in milliseconds before caching
@@ -227,7 +227,9 @@ a JSONschema based on a CSV's summary statistics; and use the generated JSONsche
 quickly validate complex CSVs (NYC's 311 data) at almost 930,000 records/sec.
 
 It's type inferences are also used by the `tojsonl` command to generate properly typed
-JSONL files.
+JSONL files; the `frequency` command to short-circuit frequency table generation for columns
+with all unique values; and the `schema` command to set the data type of a column and identify
+low-cardinality columns for enum generation in the JSONschema.
 
 To safeguard against undefined behavior, `stats` is the most extensively tested command,
 with ~500 tests.
@@ -278,7 +280,7 @@ pub struct Args {
     pub flag_prefer_dmy:      bool,
     pub flag_force:           bool,
     pub flag_jobs:            Option<usize>,
-    pub flag_stats_json:      bool,
+    pub flag_stats_jsonl:     bool,
     pub flag_cache_threshold: isize,
     pub flag_output:          Option<String>,
     pub flag_no_headers:      bool,
@@ -520,10 +522,10 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
     }
 
     let mut compute_stats = true;
-    let mut create_cache = args.flag_cache_threshold > 0 || args.flag_stats_json;
+    let mut create_cache = args.flag_cache_threshold > 0 || args.flag_stats_jsonl;
     let mut autoindex_set = false;
 
-    let write_stats_json = args.flag_stats_json;
+    let write_stats_jsonl = args.flag_stats_jsonl;
 
     if let Some(path) = fconfig.path.clone() {
         let path_file_stem = path.file_stem().unwrap().to_str().unwrap();
@@ -768,9 +770,9 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
                 serde_json::to_string_pretty(&current_stats_args).unwrap(),
             )?;
 
-            // save the stats data to "<FILESTEM>.stats.csv.data.json"
-            if write_stats_json {
-                stats_pathbuf.set_extension("data.json");
+            // save the stats data to "<FILESTEM>.stats.csv.data.jsonl"
+            if write_stats_jsonl {
+                stats_pathbuf.set_extension("data.jsonl");
                 util::csv_to_jsonl(&currstats_filename, &STATSDATA_TYPES_ARRAY, stats_pathbuf)?;
             }
         }
diff --git a/src/cmd/tojsonl.rs b/src/cmd/tojsonl.rs
index 9d66e8855..53b9f58d6 100644
--- a/src/cmd/tojsonl.rs
+++ b/src/cmd/tojsonl.rs
@@ -9,9 +9,9 @@ It will infer a column as boolean if its cardinality is 2, and the first charact
 the values are one of the following case-insensitive combinations:
   t/f; t/null; 1/0; 1/null; y/n & y/null are treated as true/false.
 
-The `tojsonl` command will reuse a `stats.csv.data.json` file if it exists and is current
-(i.e. stats generated with --cardinality and --infer-dates options) and will skip
-recomputing stats.
+The `tojsonl` command will reuse a `stats.csv.data.jsonl` file if it exists and is
+current (i.e. stats generated with --cardinality and --infer-dates options) and will
+skip recomputing stats.
 
 For examples, see https://github.com/jqnatividad/qsv/blob/master/tests/test_tojsonl.rs.
 
diff --git a/src/util.rs b/src/util.rs
index d30a345de..93f1a8894 100644
--- a/src/util.rs
+++ b/src/util.rs
@@ -1915,8 +1915,8 @@ pub fn trim_bs_whitespace(bytes: &[u8]) -> &[u8] {
     &bytes[start..end]
 }
 
-/// get stats records from stats.csv.data.json file, or if its invalid, by running the stats command
-/// returns tuple (`csv_fields`, `csv_stats`, `stats_col_index_map`)
+/// get stats records from stats.csv.data.jsonl file, or if its invalid, by running the stats
+/// command returns tuple (`csv_fields`, `csv_stats`, `stats_col_index_map`)
 pub fn get_stats_records(
     args: &SchemaArgs,
     mode: StatsMode,
@@ -1931,7 +1931,7 @@ pub fn get_stats_records(
     };
 
     let canonical_input_path = Path::new(&args.arg_input.clone().unwrap()).canonicalize()?;
-    let statsdata_path = canonical_input_path.with_extension("stats.csv.data.json");
+    let statsdata_path = canonical_input_path.with_extension("stats.csv.data.jsonl");
 
     let stats_data_current = if statsdata_path.exists() {
         let statsdata_metadata = std::fs::metadata(&statsdata_path)?;
@@ -1939,16 +1939,17 @@ pub fn get_stats_records(
         let input_metadata = std::fs::metadata(args.arg_input.clone().unwrap())?;
 
         if statsdata_metadata.modified()? > input_metadata.modified()? {
-            info!("Valid stats.csv.data.json file found!");
+            info!("Valid stats.csv.data.jsonl file found!");
             true
         } else {
             info!(
-                "stats.csv.data.json file is older than input file. Regenerating stats json file."
+                "stats.csv.data.jsonl file is older than input file. Regenerating stats jsonl \
+                 file."
             );
             false
         }
     } else {
-        info!("stats.csv.data.json file does not exist: {statsdata_path:?}");
+        info!("stats.csv.data.jsonl file does not exist: {statsdata_path:?}");
         false
     };
 
@@ -1995,7 +1996,7 @@ pub fn get_stats_records(
             flag_prefer_dmy:      args.flag_prefer_dmy,
             flag_force:           args.flag_force,
             flag_jobs:            Some(njobs(args.flag_jobs)),
-            flag_stats_json:      true,
+            flag_stats_jsonl:     true,
             flag_cache_threshold: 1, // force the creation of stats cache files
             flag_output:          None,
             flag_no_headers:      args.flag_no_headers,
@@ -2003,21 +2004,21 @@ pub fn get_stats_records(
             flag_memcheck:        args.flag_memcheck,
         };
 
-        // otherwise, run stats command to generate stats.csv.data.json file
+        // otherwise, run stats command to generate stats.csv.data.jsonl file
         let tempfile = tempfile::Builder::new()
             .suffix(".stats.csv")
             .tempfile()
             .unwrap();
         let tempfile_path = tempfile.path().to_str().unwrap().to_string();
 
-        let statsdatajson_path = canonical_input_path.with_extension("stats.csv.data.json");
+        let statsdatajson_path = canonical_input_path.with_extension("stats.csv.data.jsonl");
 
         let mut stats_args_str = if mode == StatsMode::Schema {
             // mode is GetStatsMode::Schema
             // we're generating schema, so we cardinality and to infer-dates
             format!(
                 "stats {input} --infer-dates --dates-whitelist {dates_whitelist} --round 4 \
-                 --cardinality --output {output} --stats-json --force",
+                 --cardinality --stats-jsonl --force --output {output}",
                 input = {
                     if let Some(arg_input) = stats_args.arg_input.clone() {
                         arg_input
@@ -2032,7 +2033,7 @@ pub fn get_stats_records(
             // mode is GetStatsMode::Frequency or GetStatsMode::FrequencyForceStats
             // we're doing frequency, so we just need cardinality
             format!(
-                "stats {input} --cardinality --stats-json --output {output}",
+                "stats {input} --cardinality --stats-jsonl --output {output}",
                 input = {
                     if let Some(arg_input) = stats_args.arg_input.clone() {
                         arg_input


Test patch:
diff --git a/tests/test_frequency.rs b/tests/test_frequency.rs
index 2dc0ce0d2..4d32fa9f4 100644
--- a/tests/test_frequency.rs
+++ b/tests/test_frequency.rs
@@ -482,7 +482,7 @@ fn frequency_all_unique_with_stats_cache() {
     stats_cmd
         .arg(testdata.clone())
         .arg("--cardinality")
-        .arg("--stats-json");
+        .arg("--stats-jsonl");
 
     wrk.assert_success(&mut stats_cmd);
 
@@ -506,7 +506,7 @@ fn frequency_all_unique_with_stats_cache_alt_all_unique_text() {
     stats_cmd
         .arg(testdata.clone())
         .arg("--cardinality")
-        .arg("--stats-json");
+        .arg("--stats-jsonl");
 
     wrk.assert_success(&mut stats_cmd);
 
@@ -553,7 +553,7 @@ fn frequency_all_unique_stats_mode_none() {
     stats_cmd
         .arg(testdata.clone())
         .arg("--cardinality")
-        .arg("--stats-json");
+        .arg("--stats-jsonl");
 
     wrk.assert_success(&mut stats_cmd);