Repository: dathere/qsv (language: rust)

Issue:
`stats`: add `sortiness` score
When computing `sort_order` of a column, we can maintain a "sortiness"[^1] score.

Right now, `sort_order` has three states - `Ascending`, `Descending` and `Unsorted`.

`sortiness` will allow us to assess the general sort order of an unsorted column.

The score will range from -1.0 to 1.0.

-1.0 is equivalent to Descending. 1.0 is Ascending. 0.0 indicates that the column only has one value.

When sortiness is 0.0, we can even short-circuit and skip cardinality computation.

[^1]: Inspired by Stephen Colbert's [Truthiness](https://en.wikipedia.org/wiki/Truthiness) 😉 


Discussion / hints:

Fix patch:
diff --git a/src/cmd/stats.rs b/src/cmd/stats.rs
index 1d25e9f11..0602a3297 100644
--- a/src/cmd/stats.rs
+++ b/src/cmd/stats.rs
@@ -450,6 +450,7 @@ const STATSDATA_TYPES_ARRAY: [JsonTypes; MAX_STAT_COLUMNS] = [
     JsonTypes::String, //max
     JsonTypes::Float,  //range
     JsonTypes::String, //sort_order
+    JsonTypes::Float,  //sortiness
     JsonTypes::Int,    //min_length
     JsonTypes::Int,    //max_length
     JsonTypes::Int,    //sum_length
@@ -504,10 +505,10 @@ const MS_IN_DAY_INT: i64 = 86_400_000;
 const DAY_DECIMAL_PLACES: u32 = 5;
 
 // maximum number of output columns
-const MAX_STAT_COLUMNS: usize = 42;
+const MAX_STAT_COLUMNS: usize = 43;
 
 // the first N columns are fingerprint hash columns
-const FINGERPRINT_HASH_COLUMNS: usize = 25;
+const FINGERPRINT_HASH_COLUMNS: usize = 26;
 
 // maximum number of antimodes to display
 const MAX_ANTIMODES: usize = 10;
@@ -1200,7 +1201,7 @@ impl Args {
 
         // these are the standard stats columns that are always output
         // the "streaming" stats that are always included in stats output
-        // aka the 25 FINGERPRINT_HASH_COLUMNS
+        // aka the 26 FINGERPRINT_HASH_COLUMNS
         fields.extend_from_slice(&[
             "field",
             "type",
@@ -1210,6 +1211,7 @@ impl Args {
             "max",
             "range",
             "sort_order",
+            "sortiness",
             "min_length",
             "max_length",
             "sum_length",
@@ -1569,15 +1571,16 @@ impl Stats {
 
         let empty = String::new;
 
-        // min/max/range/sort_order
+        // min/max/range/sort_order/sortiness
         // we do this first as we want to get the sort_order, so we can skip sorting if not
         // required. We also need to do this before --infer-boolean because we need to know
         // the min/max values to determine if the range is equal to the supported boolean
         // ranges (0/1, f/t, n/y)
-        let mut minmax_range_sortorder_pieces = Vec::with_capacity(4);
+        let mut minmax_range_sortorder_pieces = Vec::with_capacity(5);
         let mut minval_lower: char = '\0';
         let mut maxval_lower: char = '\0';
         let mut column_sorted = false;
+        // let mut sortiness = 0_f64;
         if let Some(mm) = self
             .minmax
             .as_ref()
@@ -1589,9 +1592,15 @@ impl Stats {
             if mm.3.starts_with("Ascending") {
                 column_sorted = true;
             }
-            minmax_range_sortorder_pieces.extend_from_slice(&[mm.0, mm.1, mm.2, mm.3]);
+            minmax_range_sortorder_pieces.extend_from_slice(&[mm.0, mm.1, mm.2, mm.3, mm.4]);
         } else {
-            minmax_range_sortorder_pieces.extend_from_slice(&[empty(), empty(), empty(), empty()]);
+            minmax_range_sortorder_pieces.extend_from_slice(&[
+                empty(),
+                empty(),
+                empty(),
+                empty(),
+                empty(),
+            ]);
         }
 
         let record_count = *RECORD_COUNT.get().unwrap_or(&1);
@@ -2341,59 +2350,72 @@ impl TypedMinMax {
     }
 
     #[inline]
-    fn show(&self, typ: FieldType, round_places: u32) -> Option<(String, String, String, String)> {
+    fn show(
+        &self,
+        typ: FieldType,
+        round_places: u32,
+    ) -> Option<(String, String, String, String, String)> {
         match typ {
             TNull => None,
             TString => {
-                if let (Some(min), Some(max), sort_order) = (
+                if let (Some(min), Some(max), sort_order, sortiness) = (
                     self.strings.min(),
                     self.strings.max(),
                     self.strings.sort_order(),
+                    self.strings.sortiness(),
                 ) {
                     let min = String::from_utf8_lossy(min).to_string();
                     let max = String::from_utf8_lossy(max).to_string();
                     let sort_order = sort_order.to_string();
-                    Some((min, max, String::new(), sort_order))
+                    let sortiness = util::round_num(sortiness, round_places);
+                    Some((min, max, String::new(), sort_order, sortiness))
                 } else {
                     None
                 }
             },
             TInteger => {
-                if let (Some(min), Some(max), sort_order) = (
+                if let (Some(min), Some(max), sort_order, sortiness) = (
                     self.integers.min(),
                     self.integers.max(),
                     self.integers.sort_order(),
+                    self.integers.sortiness(),
                 ) {
                     Some((
                         itoa::Buffer::new().format(*min).to_owned(),
                         itoa::Buffer::new().format(*max).to_owned(),
                         itoa::Buffer::new().format(*max - *min).to_owned(),
                         sort_order.to_string(),
+                        util::round_num(sortiness, round_places),
                     ))
                 } else {
                     None
                 }
             },
             TFloat => {
-                if let (Some(min), Some(max), sort_order) = (
+                if let (Some(min), Some(max), sort_order, sortiness) = (
                     self.floats.min(),
                     self.floats.max(),
                     self.floats.sort_order(),
+                    self.floats.sortiness(),
                 ) {
                     Some((
                         ryu::Buffer::new().format(*min).to_owned(),
                         ryu::Buffer::new().format(*max).to_owned(),
                         util::round_num(*max - *min, round_places),
                         sort_order.to_string(),
+                        util::round_num(sortiness, round_places),
                     ))
                 } else {
                     None
                 }
             },
             TDateTime | TDate => {
-                if let (Some(min), Some(max), sort_order) =
-                    (self.dates.min(), self.dates.max(), self.dates.sort_order())
-                {
+                if let (Some(min), Some(max), sort_order, sortiness) = (
+                    self.dates.min(),
+                    self.dates.max(),
+                    self.dates.sort_order(),
+                    self.dates.sortiness(),
+                ) {
                     Some((
                         timestamp_ms_to_rfc3339(*min, typ),
                         timestamp_ms_to_rfc3339(*max, typ),
@@ -2404,6 +2426,7 @@ impl TypedMinMax {
                             u32::max(round_places, 5),
                         ),
                         sort_order.to_string(),
+                        util::round_num(sortiness, round_places),
                     ))
                 } else {
                     None


Test patch:
diff --git a/resources/test/boston311-10-boolean-1or0-stats.csv b/resources/test/boston311-10-boolean-1or0-stats.csv
index 0a4f396bc..6f772b40f 100644
--- a/resources/test/boston311-10-boolean-1or0-stats.csv
+++ b/resources/test/boston311-10-boolean-1or0-stats.csv
@@ -1,35 +1,35 @@
-field,type,is_ascii,sum,min,max,range,sort_order,min_length,max_length,sum_length,avg_length,stddev_length,variance_length,cv_length,mean,sem,geometric_mean,harmonic_mean,stddev,variance,cv,nullcount,max_precision,sparsity,cardinality,qsv__value
-case_enquiry_id,Integer,,1010041354742,101004113298,101004155594,42296,Unsorted,,,,,,,,101004135474.2,4663.4961,101004135474.1991,101004135474.1978,14747.2697,217481962.3498,0,0,,0,10,
-open_dt,String,true,,2022-01-01 00:16:00,2022-01-31 11:46:00,,Unsorted,19,19,190,19,0,0,0,,,,,,,,0,,0,10,
-target_dt,String,true,,2022-01-11 08:30:00,2022-05-20 13:03:21,,Unsorted,0,19,114,11.4,9.3081,86.64,0.8165,,,,,,,,4,,0.4,6,
-closed_dt,String,true,,2022-01-09 06:43:06,2022-01-20 08:45:12,,Unsorted,0,19,95,9.5,9.4412,89.1358,0.9938,,,,,,,,5,,0.5,6,
-ontime,String,true,,ONTIME,OVERDUE,,Unsorted,6,7,62,6.2,0.4,0.16,0.0645,,,,,,,,0,,0,2,
-case_status,String,true,,Closed,Open,,Unsorted,4,6,50,5,1,1,0.2,,,,,,,,0,,0,2,
-case_status_boolean,Boolean,,5,0,1,1,Unsorted,,,,,,,,0.5,0.1581,0,,0.5,0.25,100,0,,0,2,
-closure_reason,String,true,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,1,82,350,35,34.5543,1194,0.9873,,,,,,,,0,,0,6,
-case_title,String,true,,BTDT: Complaint,Sidewalk Cover / Manhole,,Unsorted,13,57,235,23.5,14.1156,199.25,0.6007,,,,,,,,0,,0,8,
-subject,String,true,,Boston Police Department,Public Works Department,,Unsorted,21,31,235,23.5,2.6552,7.05,0.113,,,,,,,,0,,0,5,
-reason,String,true,,Administrative & General Requests,Street Cleaning,,Unsorted,7,33,174,17.4,7.9019,62.44,0.4541,,,,,,,,0,,0,7,
-type,String,true,,CE Collection,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,13,47,240,24,11.619,135,0.4841,,,,,,,,0,,0,8,
-queue,String,true,,BTDT_Parking Enforcement,PWDx_Snow Cases,,Unsorted,15,46,272,27.2,10.1272,102.56,0.3723,,,,,,,,0,,0,7,
-department,String,true,,BTDT,PWDx,,Unsorted,3,4,38,3.8,0.4,0.16,0.1053,,,,,,,,0,,0,5,
-submittedphoto,NULL,,,,,,,,,,,,,,,,,,,,,10,,1,1,
-closedphoto,NULL,,,,,,,,,,,,,,,,,,,,,10,,1,1,
-location,String,true,, ,850 South St  Roslindale  MA  02131,,Unsorted,1,40,309,30.9,10.4062,108.29,0.3368,,,,,,,,0,,0,10,
-fire_district,String,true,, ,9,,Unsorted,1,1,10,1,0,0,0,,,,,,,,0,,0,4,
-pwd_district,String,true,, ,1C,,Unsorted,1,2,19,1.9,0.3,0.09,0.1579,,,,,,,,0,,0,6,
-city_council_district,String,true,, ,8,,Unsorted,1,1,10,1,0,0,0,,,,,,,,0,,0,6,
-police_district,String,true,, ,E5,,Unsorted,1,3,21,2.1,0.5385,0.29,0.2564,,,,,,,,0,,0,6,
-neighborhood,String,true,, ,South End,,Unsorted,1,13,91,9.1,3.2696,10.69,0.3593,,,,,,,,0,,0,8,
-neighborhood_services_district,String,true,, ,6,,Unsorted,1,2,14,1.4,0.4899,0.24,0.3499,,,,,,,,0,,0,7,
-ward,String,true,, ,Ward 9,,Unsorted,1,7,53,5.3,1.9519,3.81,0.3683,,,,,,,,0,,0,8,
-precinct,String,true,, ,2004,,Unsorted,1,4,37,3.7,0.9,0.81,0.2432,,,,,,,,0,,0,9,
-location_street_name,String,true,,12 Derne St,850 South St,,Unsorted,0,20,120,12,2.7889,7.7778,0.2324,,,,,,,,1,,0.1,10,
-location_zipcode,String,true,,02113,02131,,Unsorted,0,5,45,4.5,0,0,0,,,,,,,,1,,0.1,8,
-latitude,Float,,423.4656,42.2884,42.3735,0.0851,Unsorted,,,,,,,,42.3466,0.008,42.3466,42.3465,0.0252,0.0006,0.0595,0,4,0,9,
-longitude,Float,,-710.782,-71.133,-71.0566,0.0764,Unsorted,,,,,,,,-71.0782,0.0078,,,0.0246,0.0006,-0.0346,0,4,0,10,
-source,String,true,,City Worker App,Constituent Call,,Unsorted,15,16,157,15.7,0.4583,0.21,0.0292,,,,,,,,0,,0,2,
-qsv__rowcount,,,,,,,,,,,,,,,,,,,,,,,,,,10
-qsv__columncount,,,,,,,,,,,,,,,,,,,,,,,,,,30
-qsv__filesize_bytes,,,,,,,,,,,,,,,,,,,,,,,,,,3887
-qsv__fingerprint_hash,,,,,,,,,,,,,,,,,,,,,,,,,,c68663e33baa4ab6ad8719ee9d15dc1b84423ea08bf9aafeba4b1c0b18eb3de2
+field,type,is_ascii,sum,min,max,range,sort_order,sortiness,min_length,max_length,sum_length,avg_length,stddev_length,variance_length,cv_length,mean,sem,geometric_mean,harmonic_mean,stddev,variance,cv,nullcount,max_precision,sparsity,cardinality,qsv__value
+case_enquiry_id,Integer,,1010041354742,101004113298,101004155594,42296,Unsorted,-0.1111,,,,,,,,101004135474.2,4663.4961,101004135474.1991,101004135474.1978,14747.2697,217481962.3498,0,0,,0,10,
+open_dt,String,true,,2022-01-01 00:16:00,2022-01-31 11:46:00,,Unsorted,-0.1111,19,19,190,19,0,0,0,,,,,,,,0,,0,10,
+target_dt,String,true,,2022-01-11 08:30:00,2022-05-20 13:03:21,,Unsorted,0.2,0,19,114,11.4,9.3081,86.64,0.8165,,,,,,,,4,,0.4,6,
+closed_dt,String,true,,2022-01-09 06:43:06,2022-01-20 08:45:12,,Unsorted,0.5,0,19,95,9.5,9.4412,89.1358,0.9938,,,,,,,,5,,0.5,6,
+ontime,String,true,,ONTIME,OVERDUE,,Unsorted,0.5556,6,7,62,6.2,0.4,0.16,0.0645,,,,,,,,0,,0,2,
+case_status,String,true,,Closed,Open,,Unsorted,0.1111,4,6,50,5,1,1,0.2,,,,,,,,0,,0,2,
+case_status_boolean,Boolean,,5,0,1,1,Unsorted,0.1111,,,,,,,,0.5,0.1581,0,,0.5,0.25,100,0,,0,2,
+closure_reason,String,true,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,0.1111,1,82,350,35,34.5543,1194,0.9873,,,,,,,,0,,0,6,
+case_title,String,true,,BTDT: Complaint,Sidewalk Cover / Manhole,,Unsorted,-0.1111,13,57,235,23.5,14.1156,199.25,0.6007,,,,,,,,0,,0,8,
+subject,String,true,,Boston Police Department,Public Works Department,,Unsorted,0.5556,21,31,235,23.5,2.6552,7.05,0.113,,,,,,,,0,,0,5,
+reason,String,true,,Administrative & General Requests,Street Cleaning,,Unsorted,0.1111,7,33,174,17.4,7.9019,62.44,0.4541,,,,,,,,0,,0,7,
+type,String,true,,CE Collection,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,0.1111,13,47,240,24,11.619,135,0.4841,,,,,,,,0,,0,8,
+queue,String,true,,BTDT_Parking Enforcement,PWDx_Snow Cases,,Unsorted,0.3333,15,46,272,27.2,10.1272,102.56,0.3723,,,,,,,,0,,0,7,
+department,String,true,,BTDT,PWDx,,Unsorted,0.5556,3,4,38,3.8,0.4,0.16,0.1053,,,,,,,,0,,0,5,
+submittedphoto,NULL,,,,,,,,,,,,,,,,,,,,,,10,,1,1,
+closedphoto,NULL,,,,,,,,,,,,,,,,,,,,,,10,,1,1,
+location,String,true,, ,850 South St  Roslindale  MA  02131,,Unsorted,-0.1111,1,40,309,30.9,10.4062,108.29,0.3368,,,,,,,,0,,0,10,
+fire_district,String,true,, ,9,,Unsorted,0.3333,1,1,10,1,0,0,0,,,,,,,,0,,0,4,
+pwd_district,String,true,, ,1C,,Unsorted,0.3333,1,2,19,1.9,0.3,0.09,0.1579,,,,,,,,0,,0,6,
+city_council_district,String,true,, ,8,,Unsorted,0.3333,1,1,10,1,0,0,0,,,,,,,,0,,0,6,
+police_district,String,true,, ,E5,,Unsorted,0.1111,1,3,21,2.1,0.5385,0.29,0.2564,,,,,,,,0,,0,6,
+neighborhood,String,true,, ,South End,,Unsorted,0.1111,1,13,91,9.1,3.2696,10.69,0.3593,,,,,,,,0,,0,8,
+neighborhood_services_district,String,true,, ,6,,Unsorted,-0.1111,1,2,14,1.4,0.4899,0.24,0.3499,,,,,,,,0,,0,7,
+ward,String,true,, ,Ward 9,,Unsorted,0.5556,1,7,53,5.3,1.9519,3.81,0.3683,,,,,,,,0,,0,8,
+precinct,String,true,, ,2004,,Unsorted,0.3333,1,4,37,3.7,0.9,0.81,0.2432,,,,,,,,0,,0,9,
+location_street_name,String,true,,12 Derne St,850 South St,,Unsorted,-0.25,0,20,120,12,2.7889,7.7778,0.2324,,,,,,,,1,,0.1,10,
+location_zipcode,String,true,,02113,02131,,Unsorted,0.25,0,5,45,4.5,0,0,0,,,,,,,,1,,0.1,8,
+latitude,Float,,423.4656,42.2884,42.3735,0.0851,Unsorted,-0.1111,,,,,,,,42.3466,0.008,42.3466,42.3465,0.0252,0.0006,0.0595,0,4,0,9,
+longitude,Float,,-710.782,-71.133,-71.0566,0.0764,Unsorted,0.1111,,,,,,,,-71.0782,0.0078,,,0.0246,0.0006,-0.0346,0,4,0,10,
+source,String,true,,City Worker App,Constituent Call,,Unsorted,0.5556,15,16,157,15.7,0.4583,0.21,0.0292,,,,,,,,0,,0,2,
+qsv__rowcount,,,,,,,,,,,,,,,,,,,,,,,,,,,10
+qsv__columncount,,,,,,,,,,,,,,,,,,,,,,,,,,,30
+qsv__filesize_bytes,,,,,,,,,,,,,,,,,,,,,,,,,,,3887
+qsv__fingerprint_hash,,,,,,,,,,,,,,,,,,,,,,,,,,,2de69db140b01aae7ac3ed110469cd5d454e990baaf3dc73e2e6e26909ac3c4e
diff --git a/resources/test/boston311-10-boolean-tf-stats.csv b/resources/test/boston311-10-boolean-tf-stats.csv
index 6ed3fc14d..8c8fa4205 100644
--- a/resources/test/boston311-10-boolean-tf-stats.csv
+++ b/resources/test/boston311-10-boolean-tf-stats.csv
@@ -1,35 +1,35 @@
-field,type,is_ascii,sum,min,max,range,sort_order,min_length,max_length,sum_length,avg_length,stddev_length,variance_length,cv_length,mean,sem,geometric_mean,harmonic_mean,stddev,variance,cv,nullcount,max_precision,sparsity,cardinality,qsv__value
-case_enquiry_id,Integer,,1010041354742,101004113298,101004155594,42296,Unsorted,,,,,,,,101004135474.2,4663.4961,101004135474.1991,101004135474.1978,14747.2697,217481962.3498,0,0,,0,10,
-open_dt,String,true,,2022-01-01 00:16:00,2022-01-31 11:46:00,,Unsorted,19,19,190,19,0,0,0,,,,,,,,0,,0,10,
-target_dt,String,true,,2022-01-11 08:30:00,2022-05-20 13:03:21,,Unsorted,0,19,114,11.4,9.3081,86.64,0.8165,,,,,,,,4,,0.4,6,
-closed_dt,String,true,,2022-01-09 06:43:06,2022-01-20 08:45:12,,Unsorted,0,19,95,9.5,9.4412,89.1358,0.9938,,,,,,,,5,,0.5,6,
-ontime,String,true,,ONTIME,OVERDUE,,Unsorted,6,7,62,6.2,0.4,0.16,0.0645,,,,,,,,0,,0,2,
-case_status,String,true,,Closed,Open,,Unsorted,4,6,50,5,1,1,0.2,,,,,,,,0,,0,2,
-case_status_boolean,Boolean,true,,False,True,,Unsorted,4,5,45,4.5,0.5,0.25,0.1111,,,,,,,,0,,0,2,
-closure_reason,String,true,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,1,82,350,35,34.5543,1194,0.9873,,,,,,,,0,,0,6,
-case_title,String,true,,BTDT: Complaint,Sidewalk Cover / Manhole,,Unsorted,13,57,235,23.5,14.1156,199.25,0.6007,,,,,,,,0,,0,8,
-subject,String,true,,Boston Police Department,Public Works Department,,Unsorted,21,31,235,23.5,2.6552,7.05,0.113,,,,,,,,0,,0,5,
-reason,String,true,,Administrative & General Requests,Street Cleaning,,Unsorted,7,33,174,17.4,7.9019,62.44,0.4541,,,,,,,,0,,0,7,
-type,String,true,,CE Collection,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,13,47,240,24,11.619,135,0.4841,,,,,,,,0,,0,8,
-queue,String,true,,BTDT_Parking Enforcement,PWDx_Snow Cases,,Unsorted,15,46,272,27.2,10.1272,102.56,0.3723,,,,,,,,0,,0,7,
-department,String,true,,BTDT,PWDx,,Unsorted,3,4,38,3.8,0.4,0.16,0.1053,,,,,,,,0,,0,5,
-submittedphoto,NULL,,,,,,,,,,,,,,,,,,,,,10,,1,1,
-closedphoto,NULL,,,,,,,,,,,,,,,,,,,,,10,,1,1,
-location,String,true,, ,850 South St  Roslindale  MA  02131,,Unsorted,1,40,309,30.9,10.4062,108.29,0.3368,,,,,,,,0,,0,10,
-fire_district,String,true,, ,9,,Unsorted,1,1,10,1,0,0,0,,,,,,,,0,,0,4,
-pwd_district,String,true,, ,1C,,Unsorted,1,2,19,1.9,0.3,0.09,0.1579,,,,,,,,0,,0,6,
-city_council_district,String,true,, ,8,,Unsorted,1,1,10,1,0,0,0,,,,,,,,0,,0,6,
-police_district,String,true,, ,E5,,Unsorted,1,3,21,2.1,0.5385,0.29,0.2564,,,,,,,,0,,0,6,
-neighborhood,String,true,, ,South End,,Unsorted,1,13,91,9.1,3.2696,10.69,0.3593,,,,,,,,0,,0,8,
-neighborhood_services_district,String,true,, ,6,,Unsorted,1,2,14,1.4,0.4899,0.24,0.3499,,,,,,,,0,,0,7,
-ward,String,true,, ,Ward 9,,Unsorted,1,7,53,5.3,1.9519,3.81,0.3683,,,,,,,,0,,0,8,
-precinct,String,true,, ,2004,,Unsorted,1,4,37,3.7,0.9,0.81,0.2432,,,,,,,,0,,0,9,
-location_street_name,String,true,,12 Derne St,850 South St,,Unsorted,0,20,120,12,2.7889,7.7778,0.2324,,,,,,,,1,,0.1,10,
-location_zipcode,String,true,,02113,02131,,Unsorted,0,5,45,4.5,0,0,0,,,,,,,,1,,0.1,8,
-latitude,Float,,423.4656,42.2884,42.3735,0.0851,Unsorted,,,,,,,,42.3466,0.008,42.3466,42.3465,0.0252,0.0006,0.0595,0,4,0,9,
-longitude,Float,,-710.782,-71.133,-71.0566,0.0764,Unsorted,,,,,,,,-71.0782,0.0078,,,0.0246,0.0006,-0.0346,0,4,0,10,
-source,String,true,,City Worker App,Constituent Call,,Unsorted,15,16,157,15.7,0.4583,0.21,0.0292,,,,,,,,0,,0,2,
-qsv__rowcount,,,,,,,,,,,,,,,,,,,,,,,,,,10
-qsv__columncount,,,,,,,,,,,,,,,,,,,,,,,,,,30
-qsv__filesize_bytes,,,,,,,,,,,,,,,,,,,,,,,,,,3922
-qsv__fingerprint_hash,,,,,,,,,,,,,,,,,,,,,,,,,,2c18181b0593e8f8e5729741fe2da2d06b28a6d59714dbd600009190d1920400
+field,type,is_ascii,sum,min,max,range,sort_order,sortiness,min_length,max_length,sum_length,avg_length,stddev_length,variance_length,cv_length,mean,sem,geometric_mean,harmonic_mean,stddev,variance,cv,nullcount,max_precision,sparsity,cardinality,qsv__value
+case_enquiry_id,Integer,,1010041354742,101004113298,101004155594,42296,Unsorted,-0.1111,,,,,,,,101004135474.2,4663.4961,101004135474.1991,101004135474.1978,14747.2697,217481962.3498,0,0,,0,10,
+open_dt,String,true,,2022-01-01 00:16:00,2022-01-31 11:46:00,,Unsorted,-0.1111,19,19,190,19,0,0,0,,,,,,,,0,,0,10,
+target_dt,String,true,,2022-01-11 08:30:00,2022-05-20 13:03:21,,Unsorted,0.2,0,19,114,11.4,9.3081,86.64,0.8165,,,,,,,,4,,0.4,6,
+closed_dt,String,true,,2022-01-09 06:43:06,2022-01-20 08:45:12,,Unsorted,0.5,0,19,95,9.5,9.4412,89.1358,0.9938,,,,,,,,5,,0.5,6,
+ontime,String,true,,ONTIME,OVERDUE,,Unsorted,0.5556,6,7,62,6.2,0.4,0.16,0.0645,,,,,,,,0,,0,2,
+case_status,String,true,,Closed,Open,,Unsorted,0.1111,4,6,50,5,1,1,0.2,,,,,,,,0,,0,2,
+case_status_boolean,Boolean,true,,False,True,,Unsorted,0.1111,4,5,45,4.5,0.5,0.25,0.1111,,,,,,,,0,,0,2,
+closure_reason,String,true,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,0.1111,1,82,350,35,34.5543,1194,0.9873,,,,,,,,0,,0,6,
+case_title,String,true,,BTDT: Complaint,Sidewalk Cover / Manhole,,Unsorted,-0.1111,13,57,235,23.5,14.1156,199.25,0.6007,,,,,,,,0,,0,8,
+subject,String,true,,Boston Police Department,Public Works Department,,Unsorted,0.5556,21,31,235,23.5,2.6552,7.05,0.113,,,,,,,,0,,0,5,
+reason,String,true,,Administrative & General Requests,Street Cleaning,,Unsorted,0.1111,7,33,174,17.4,7.9019,62.44,0.4541,,,,,,,,0,,0,7,
+type,String,true,,CE Collection,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,0.1111,13,47,240,24,11.619,135,0.4841,,,,,,,,0,,0,8,
+queue,String,true,,BTDT_Parking Enforcement,PWDx_Snow Cases,,Unsorted,0.3333,15,46,272,27.2,10.1272,102.56,0.3723,,,,,,,,0,,0,7,
+department,String,true,,BTDT,PWDx,,Unsorted,0.5556,3,4,38,3.8,0.4,0.16,0.1053,,,,,,,,0,,0,5,
+submittedphoto,NULL,,,,,,,,,,,,,,,,,,,,,,10,,1,1,
+closedphoto,NULL,,,,,,,,,,,,,,,,,,,,,,10,,1,1,
+location,String,true,, ,850 South St  Roslindale  MA  02131,,Unsorted,-0.1111,1,40,309,30.9,10.4062,108.29,0.3368,,,,,,,,0,,0,10,
+fire_district,String,true,, ,9,,Unsorted,0.3333,1,1,10,1,0,0,0,,,,,,,,0,,0,4,
+pwd_district,String,true,, ,1C,,Unsorted,0.3333,1,2,19,1.9,0.3,0.09,0.1579,,,,,,,,0,,0,6,
+city_council_district,String,true,, ,8,,Unsorted,0.3333,1,1,10,1,0,0,0,,,,,,,,0,,0,6,
+police_district,String,true,, ,E5,,Unsorted,0.1111,1,3,21,2.1,0.5385,0.29,0.2564,,,,,,,,0,,0,6,
+neighborhood,String,true,, ,South End,,Unsorted,0.1111,1,13,91,9.1,3.2696,10.69,0.3593,,,,,,,,0,,0,8,
+neighborhood_services_district,String,true,, ,6,,Unsorted,-0.1111,1,2,14,1.4,0.4899,0.24,0.3499,,,,,,,,0,,0,7,
+ward,String,true,, ,Ward 9,,Unsorted,0.5556,1,7,53,5.3,1.9519,3.81,0.3683,,,,,,,,0,,0,8,
+precinct,String,true,, ,2004,,Unsorted,0.3333,1,4,37,3.7,0.9,0.81,0.2432,,,,,,,,0,,0,9,
+location_street_name,String,true,,12 Derne St,850 South St,,Unsorted,-0.25,0,20,120,12,2.7889,7.7778,0.2324,,,,,,,,1,,0.1,10,
+location_zipcode,String,true,,02113,02131,,Unsorted,0.25,0,5,45,4.5,0,0,0,,,,,,,,1,,0.1,8,
+latitude,Float,,423.4656,42.2884,42.3735,0.0851,Unsorted,-0.1111,,,,,,,,42.3466,0.008,42.3466,42.3465,0.0252,0.0006,0.0595,0,4,0,9,
+longitude,Float,,-710.782,-71.133,-71.0566,0.0764,Unsorted,0.1111,,,,,,,,-71.0782,0.0078,,,0.0246,0.0006,-0.0346,0,4,0,10,
+source,String,true,,City Worker App,Constituent Call,,Unsorted,0.5556,15,16,157,15.7,0.4583,0.21,0.0292,,,,,,,,0,,0,2,
+qsv__rowcount,,,,,,,,,,,,,,,,,,,,,,,,,,,10
+qsv__columncount,,,,,,,,,,,,,,,,,,,,,,,,,,,30
+qsv__filesize_bytes,,,,,,,,,,,,,,,,,,,,,,,,,,,3922
+qsv__fingerprint_hash,,,,,,,,,,,,,,,,,,,,,,,,,,,55eabf294ec7529ad75755db9cd482e094ccef0df7bde083426f4e8582ba1c52
diff --git a/resources/test/boston311-100-antimodes-len500-stats.csv b/resources/test/boston311-100-antimodes-len500-stats.csv
index 3c24fc747..c94ba970d 100644
--- a/resources/test/boston311-100-antimodes-len500-stats.csv
+++ b/resources/test/boston311-100-antimodes-len500-stats.csv
@@ -1,34 +1,34 @@
-field,type,is_ascii,sum,min,max,range,sort_order,min_length,max_length,sum_length,avg_length,stddev_length,variance_length,cv_length,mean,sem,geometric_mean,harmonic_mean,stddev,variance,cv,nullcount,max_precision,sparsity,mad,lower_outer_fence,lower_inner_fence,q1,q2_median,q3,iqr,upper_inner_fence,upper_outer_fence,skewness,cardinality,mode,mode_count,mode_occurrences,antimode,antimode_count,antimode_occurrences,qsv__value
-case_enquiry_id,Integer,,10100411645180,101004113298,101004155594,42296,Unsorted,,,,,,,,101004116451.8,790.552,101004116451.8012,101004116451.7994,7905.5202,62497248.9138,0,0,,0,673,101004109567,101004111646,101004113725,101004114353,101004115111,1386,101004117190,101004119269,0.0938,100,,0,0,*ALL,0,1,
-open_dt,String,true,,2022-01-01 00:16:00,2022-01-31 11:46:00,,Unsorted,19,19,1900,19,0,0,0,,,,,,,,0,,0,,,,,,,,,,,100,,0,0,*ALL,0,1,
-target_dt,String,true,,2022-01-03 10:32:34,2022-05-20 13:03:21,,Unsorted,0,19,1691,16.91,5.9449,35.3419,0.3516,,,,,,,,11,,0.11,,,,,,,,,,,42,2022-01-04 08:30:00,1,25,*PREVIEW: 2022-01-03 10:32:34|2022-01-03 11:58:12|2022-01-04 09:58:36|2022-01-04 10:41:29|2022-01-04 10:42:18|2022-01-04 11:26:45|2022-01-04 11:31:57|2022-01-04 12:13:47|2022-01-04 12:19:43|2022-01-04 15:51:30,34,1,
-closed_dt,String,true,,2022-01-01 12:56:14,2022-04-25 14:30:31,,Unsorted,0,19,1615,16.15,6.6205,43.8312,0.4099,,,,,,,,15,,0.15,,,,,,,,,,,86,,1,15,*PREVIEW: 2022-01-01 12:56:14|2022-01-01 14:17:15|2022-01-01 14:59:41|2022-01-01 15:10:16|2022-01-01 19:07:41|2022-01-02 06:43:42|2022-01-02 06:49:07|2022-01-02 08:54:11|2022-01-02 11:03:10|2022-01-02 12:45:49,85,1,
-ontime,String,true,,ONTIME,OVERDUE,,Unsorted,6,7,617,6.17,0.3756,0.1411,0.0609,,,,,,,,0,,0,,,,,,,,,,,2,ONTIME,1,83,OVERDUE,1,17,
-case_status,String,true,,Closed,Open,,Unsorted,4,6,570,5.7,0.7141,0.51,0.1253,,,,,,,,0,,0,,,,,,,,,,,2,Closed,1,85,Open,1,15,
-closure_reason,String,true,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,1,284,8314,83.14,55.0262,3027.8804,0.6618,,,,,,,,0,,0,,,,,,,,,,,86, ,1,15,*PREVIEW: Case Closed Case Resolved  NEW CART#21026466 DELV ON 1/11/22  |Case Closed Case Resolved  NO OVERFLOW NO ADDITIONAL CART DELV 1/11/22  |Case Closed. Closed date : 2022-01-02 23:40:14.32 Case Resolved CLEAR |Case Closed. Closed date : 2022-01-02 23:54:13.31 Case Resolved BOSTON POLICE WERE THERE TO TICKET |Case Closed. Closed date : 2022-01-02 23:55:44.4 Case Resolved CLEAR |Case Closed. Closed date : 2022-01-02 23:59:18.367 Case Resolved CLEAR |Case Closed. Closed date : 2022-01-03 00:...,85,1,
-case_title,String,true,,Abandoned Vehicles,Traffic Signal Inspection,,Unsorted,10,57,2386,23.86,9.452,89.3404,0.3961,,,,,,,,0,,0,,,,,,,,,,,42,Parking Enforcement,1,20,"*PREVIEW: Animal Generic Request|BTDT: Complaint|City/State Snow Issues|DISPATCHED Short Term Rental|Electrical|Graffiti: Ward 8  0803  |Loud Parties/Music/People|Misc. Snow Complaint|Missed ""Other"" Trash: District 07|Missed Trash/Recycling/Yard Waste/Bulk Item",24,1,
-subject,String,true,,Animal Control,Transportation - Traffic Division,,Unsorted,14,33,2570,25.7,4.9041,24.05,0.1908,,,,,,,,0,,0,,,,,,,,,,,9,Public Works Department,1,51,Animal Control|Boston Police Department|Boston Water & Sewer Commission,3,1,
-reason,String,true,,Administrative & General Requests,Street Lights,,Unsorted,7,33,1892,18.92,8.4056,70.6536,0.4443,,,,,,,,0,,0,,,,,,,,,,,20,Enforcement & Abandoned Vehicles,1,23,Administrative & General Requests|Animal Issues|Building|Employee & General Comments|Noise Disturbance|Programs|Sidewalk Cover / Manhole,7,1,
-type,String,true,,Abandoned Vehicles,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,10,47,2266,22.66,8.034,64.5444,0.3545,,,,,,,,0,,0,,,,,,,,,,,36,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|City/State Snow Issues|Electrical|General Comments For a Program or Policy|General Lighting Request|Knockdown Replacement|Loud Parties/Music/People|Misc. Snow Complaint|New Sign  Crosswalk or Pavement Marking|Schedule a Bulk Item Pickup SS,15,1,
-queue,String,true,,BTDT_AVRS Interface Queue,PWDx_Street Light_General Lighting Request,,Unsorted,13,55,2802,28.02,8.0224,64.3596,0.2863,,,,,,,,0,,0,,,,,,,,,,,35,BTDT_Parking Enforcement,1,21,*PREVIEW: BTDT_BostonBikes|BTDT_Engineering_New Sign and Pavement Marking Requests|BTDT_Sign Shop_Sign Installation|BTDT_Traffic Signal_Graffiti|ISD_Building (INTERNAL)|PARK_Maintenance_Region 6|PWDx_Contractor Complaints|PWDx_District 04: Allston/Brighton|PWDx_District 06: West Roxbury and Roslindale|PWDx_District 09: East Boston,15,1,
-department,String,true,,BTDT,PWDx,,Unsorted,3,4,392,3.92,0.2713,0.0736,0.0692,,,,,,,,0,,0,,,,,,,,,,,7,PWDx,1,49,GEN_,1,2,
-submittedphoto,String,true,,https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg,https://311.boston.gov/media/boston/report/photos/61d75bba05bbcf180c2d41de/report.jpg,,Unsorted,0,100,3633,36.33,43.2585,1871.2989,1.1907,,,,,,,,58,,0.58,,,,,,,,,,,43,,1,58,*PREVIEW: https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg|https://311.boston.gov/media/boston/report/photos/61d0627a05bbcf180c297a39/photo_20220101_091656.jpg|https://311.boston.gov/media/boston/report/photos/61d063e505bbcf180c297b6a/photo_20220101_092319.jpg|https://311.boston.gov/media/boston/report/photos/61d0688a05bbcf180c297be9/report.jpg|https://311.boston.gov/media/boston/report/photos/61d074c005bbcf180c298048/report.jpg|https://311.boston.gov/media/b...,42,1,
-closedphoto,NULL,,,,,,,,,,,,,,,,,,,,,100,,1,,,,,,,,,,,1,,1,100,,0,0,
-location,String,true,, ,INTERSECTION of Verdun St & Gallivan Blvd  Dorchester  MA  ,,Unsorted,1,63,3938,39.38,9.6247,92.6356,0.2444,,,,,,,,0,,0,,,,,,,,,,,98,563 Columbus Ave  Roxbury  MA  02118|INTERSECTION of Gallivan Blvd & Washington St  Dorchester  MA  ,2,2,*PREVIEW:  |103 N Beacon St  Brighton  MA  02135|11 Aberdeen St  Boston  MA  02215|1148 Hyde Park Ave  Hyde Park  MA  02136|119 L St  South Boston  MA  02127|12 Derne St  Boston  MA  02114|126 Elm St  Charlestown  MA  02129|1270 Commonwealth Ave  Allston  MA  02134|130 Shirley St  Roxbury  MA  02119|131 Arlington St  Boston  MA  02116,96,1,
-fire_district,String,true,, ,9,,Unsorted,1,2,113,1.13,0.3363,0.1131,0.2976,,,,,,,,0,,0,,,,,,,,,,,10,3,1,19, ,1,1,
-pwd_district,String,true,, ,1C,,Unsorted,1,3,209,2.09,0.3192,0.1019,0.1527,,,,,,,,0,,0,,,,,,,,,,,14,1B,1,16, ,1,1,
-city_council_district,String,true,, ,9,,Unsorted,1,1,100,1,0,0,0,,,,,,,,0,,0,,,,,,,,,,,10,1,1,22, ,1,1,
-police_district,String,true,, ,E5,,Unsorted,1,3,223,2.23,0.444,0.1971,0.1991,,,,,,,,0,,0,,,,,,,,,,,13,A1,1,20, ,1,1,
-neighborhood,String,true,, ,West Roxbury,,Unsorted,1,38,1486,14.86,9.8671,97.3604,0.664,,,,,,,,0,,0,,,,,,,,,,,19,Dorchester,1,15, |Brighton|Mission Hill,3,1,
-neighborhood_services_district,String,true,, ,9,,Unsorted,1,2,139,1.39,0.4877,0.2379,0.3509,,,,,,,,0,,0,,,,,,,,,,,16,3,1,15, |12,2,1,
-ward,String,true,, ,Ward 9,,Unsorted,1,7,499,4.99,2.2293,4.9699,0.4468,,,,,,,,0,,0,,,,,,,,,,,42,Ward 3,1,10,*PREVIEW:  |01|02|04|06|07|1|10|16|18,23,1,
-precinct,String,true,, ,2210,,Unsorted,0,4,393,3.93,0.4951,0.2451,0.126,,,,,,,,1,,0.01,,,,,,,,,,,76,0306,1,5,*PREVIEW: NULL| |0102|0105|0108|0109|0201|0204|0305|0307,61,1,
-location_street_name,String,true,,103 N Beacon St,INTERSECTION Verdun St & Gallivan Blvd,,Unsorted,0,45,1800,18,9.3995,88.3508,0.5222,,,,,,,,1,,0.01,,,,,,,,,,,97,20 Washington St|563 Columbus Ave|INTERSECTION Gallivan Blvd & Washington St,3,2,*PREVIEW: NULL|103 N Beacon St|11 Aberdeen St|1148 Hyde Park Ave|119 L St|12 Derne St|126 Elm St|1270 Commonwealth Ave|130 Shirley St|131 Arlington St,94,1,
-location_zipcode,String,true,,02109,02215,,Unsorted,0,5,415,4.15,1.8405,3.3874,0.4435,,,,,,,,17,,0.17,,,,,,,,,,,24,,1,17,02126|02134|02210|02215,4,1,
-latitude,Float,,4233.6674,42.2553,42.3806,0.1253,Unsorted,,,,,,,,42.3367,0.0031,42.3367,42.3367,0.0305,0.0009,0.072,0,4,0,0.0163,42.2034,42.2619,42.3204,42.3432,42.3594,0.039,42.4179,42.4764,-0.1667,78,42.3594,1,20,*PREVIEW: 42.2553|42.2601|42.2609|42.2645|42.2674|42.2789|42.2797|42.2804|42.2821|42.2878,74,1,
-longitude,Float,,-7107.2688,-71.1626,-71.0298,0.1328,Unsorted,,,,,,,,-71.0727,0.0031,,,0.0311,0.001,-0.0437,0,4,0,0.0121,-71.1741,-71.1294,-71.0848,-71.0609,-71.055,0.0298,-71.0104,-70.9658,-0.6101,77,-71.0587,1,19,*PREVIEW: -71.0298|-71.0301|-71.0309|-71.0323|-71.0325|-71.0329|-71.0336|-71.0338|-71.034|-71.0355,72,1,
-source,String,true,,Citizens Connect App,Self Service,,Unsorted,12,20,1801,18.01,2.3473,5.5099,0.1303,,,,,,,,0,,0,,,,,,,,,,,4,Citizens Connect App,1,56,Self Service,1,3,
-qsv__rowcount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,100
-qsv__columncount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,29
-qsv__filesize_bytes,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,47702
-qsv__fingerprint_hash,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,53018bee30100ff6bb30214ed7dbb6d7c1a40115a6c91b47b451fb76dd211e59
+field,type,is_ascii,sum,min,max,range,sort_order,sortiness,min_length,max_length,sum_length,avg_length,stddev_length,variance_length,cv_length,mean,sem,geometric_mean,harmonic_mean,stddev,variance,cv,nullcount,max_precision,sparsity,mad,lower_outer_fence,lower_inner_fence,q1,q2_median,q3,iqr,upper_inner_fence,upper_outer_fence,skewness,cardinality,mode,mode_count,mode_occurrences,antimode,antimode_count,antimode_occurrences,qsv__value
+case_enquiry_id,Integer,,10100411645180,101004113298,101004155594,42296,Unsorted,0.798,,,,,,,,101004116451.8,790.552,101004116451.8012,101004116451.7994,7905.5202,62497248.9138,0,0,,0,673,101004109567,101004111646,101004113725,101004114353,101004115111,1386,101004117190,101004119269,0.0938,100,,0,0,*ALL,0,1,
+open_dt,String,true,,2022-01-01 00:16:00,2022-01-31 11:46:00,,Unsorted,0.798,19,19,1900,19,0,0,0,,,,,,,,0,,0,,,,,,,,,,,100,,0,0,*ALL,0,1,
+target_dt,String,true,,2022-01-03 10:32:34,2022-05-20 13:03:21,,Unsorted,0.2273,0,19,1691,16.91,5.9449,35.3419,0.3516,,,,,,,,11,,0.11,,,,,,,,,,,42,2022-01-04 08:30:00,1,25,*PREVIEW: 2022-01-03 10:32:34|2022-01-03 11:58:12|2022-01-04 09:58:36|2022-01-04 10:41:29|2022-01-04 10:42:18|2022-01-04 11:26:45|2022-01-04 11:31:57|2022-01-04 12:13:47|2022-01-04 12:19:43|2022-01-04 15:51:30,34,1,
+closed_dt,String,true,,2022-01-01 12:56:14,2022-04-25 14:30:31,,Unsorted,-0.0714,0,19,1615,16.15,6.6205,43.8312,0.4099,,,,,,,,15,,0.15,,,,,,,,,,,86,,1,15,*PREVIEW: 2022-01-01 12:56:14|2022-01-01 14:17:15|2022-01-01 14:59:41|2022-01-01 15:10:16|2022-01-01 19:07:41|2022-01-02 06:43:42|2022-01-02 06:49:07|2022-01-02 08:54:11|2022-01-02 11:03:10|2022-01-02 12:45:49,85,1,
+ontime,String,true,,ONTIME,OVERDUE,,Unsorted,0.6768,6,7,617,6.17,0.3756,0.1411,0.0609,,,,,,,,0,,0,,,,,,,,,,,2,ONTIME,1,83,OVERDUE,1,17,
+case_status,String,true,,Closed,Open,,Unsorted,0.7172,4,6,570,5.7,0.7141,0.51,0.1253,,,,,,,,0,,0,,,,,,,,,,,2,Closed,1,85,Open,1,15,
+closure_reason,String,true,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,-0.0909,1,284,8314,83.14,55.0262,3027.8804,0.6618,,,,,,,,0,,0,,,,,,,,,,,86, ,1,15,*PREVIEW: Case Closed Case Resolved  NEW CART#21026466 DELV ON 1/11/22  |Case Closed Case Resolved  NO OVERFLOW NO ADDITIONAL CART DELV 1/11/22  |Case Closed. Closed date : 2022-01-02 23:40:14.32 Case Resolved CLEAR |Case Closed. Closed date : 2022-01-02 23:54:13.31 Case Resolved BOSTON POLICE WERE THERE TO TICKET |Case Closed. Closed date : 2022-01-02 23:55:44.4 Case Resolved CLEAR |Case Closed. Closed date : 2022-01-02 23:59:18.367 Case Resolved CLEAR |Case Closed. Closed date : 2022-01-03 00:...,85,1,
+case_title,String,true,,Abandoned Vehicles,Traffic Signal Inspection,,Unsorted,0.0101,10,57,2386,23.86,9.452,89.3404,0.3961,,,,,,,,0,,0,,,,,,,,,,,42,Parking Enforcement,1,20,"*PREVIEW: Animal Generic Request|BTDT: Complaint|City/State Snow Issues|DISPATCHED Short Term Rental|Electrical|Graffiti: Ward 8  0803  |Loud Parties/Music/People|Misc. Snow Complaint|Missed ""Other"" Trash: District 07|Missed Trash/Recycling/Yard Waste/Bulk Item",24,1,
+subject,String,true,,Animal Control,Transportation - Traffic Division,,Unsorted,0.3737,14,33,2570,25.7,4.9041,24.05,0.1908,,,,,,,,0,,0,,,,,,,,,,,9,Public Works Department,1,51,Animal Control|Boston Police Department|Boston Water & Sewer Commission,3,1,
+reason,String,true,,Administrative & General Requests,Street Lights,,Unsorted,0.1717,7,33,1892,18.92,8.4056,70.6536,0.4443,,,,,,,,0,,0,,,,,,,,,,,20,Enforcement & Abandoned Vehicles,1,23,Administrative & General Requests|Animal Issues|Building|Employee & General Comments|Noise Disturbance|Programs|Sidewalk Cover / Manhole,7,1,
+type,String,true,,Abandoned Vehicles,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,0.0707,10,47,2266,22.66,8.034,64.5444,0.3545,,,,,,,,0,,0,,,,,,,,,,,36,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|City/State Snow Issues|Electrical|General Comments For a Program or Policy|General Lighting Request|Knockdown Replacement|Loud Parties/Music/People|Misc. Snow Complaint|New Sign  Crosswalk or Pavement Marking|Schedule a Bulk Item Pickup SS,15,1,
+queue,String,true,,BTDT_AVRS Interface Queue,PWDx_Street Light_General Lighting Request,,Unsorted,0.2121,13,55,2802,28.02,8.0224,64.3596,0.2863,,,,,,,,0,,0,,,,,,,,,,,35,BTDT_Parking Enforcement,1,21,*PREVIEW: BTDT_BostonBikes|BTDT_Engineering_New Sign and Pavement Marking Requests|BTDT_Sign Shop_Sign Installation|BTDT_Traffic Signal_Graffiti|ISD_Building (INTERNAL)|PARK_Maintenance_Region 6|PWDx_Contractor Complaints|PWDx_District 04: Allston/Brighton|PWDx_District 06: West Roxbury and Roslindale|PWDx_District 09: East Boston,15,1,
+department,String,true,,BTDT,PWDx,,Unsorted,0.3737,3,4,392,3.92,0.2713,0.0736,0.0692,,,,,,,,0,,0,,,,,,,,,,,7,PWDx,1,49,GEN_,1,2,
+submittedphoto,String,true,,https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg,https://311.boston.gov/media/boston/report/photos/61d75bba05bbcf180c2d41de/report.jpg,,Unsorted,0.8537,0,100,3633,36.33,43.2585,1871.2989,1.1907,,,,,,,,58,,0.58,,,,,,,,,,,43,,1,58,*PREVIEW: https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg|https://311.boston.gov/media/boston/report/photos/61d0627a05bbcf180c297a39/photo_20220101_091656.jpg|https://311.boston.gov/media/boston/report/photos/61d063e505bbcf180c297b6a/photo_20220101_092319.jpg|https://311.boston.gov/media/boston/report/photos/61d0688a05bbcf180c297be9/report.jpg|https://311.boston.gov/media/boston/report/photos/61d074c005bbcf180c298048/report.jpg|https://311.boston.gov/media/b...,42,1,
+closedphoto,NULL,,,,,,,,,,,,,,,,,,,,,,100,,1,,,,,,,,,,,1,,1,100,,0,0,
+location,String,true,, ,INTERSECTION of Verdun St & Gallivan Blvd  Dorchester  MA  ,,Unsorted,-0.0303,1,63,3938,39.38,9.6247,92.6356,0.2444,,,,,,,,0,,0,,,,,,,,,,,98,563 Columbus Ave  Roxbury  MA  02118|INTERSECTION of Gallivan Blvd & Washington St  Dorchester  MA  ,2,2,*PREVIEW:  |103 N Beacon St  Brighton  MA  02135|11 Aberdeen St  Boston  MA  02215|1148 Hyde Park Ave  Hyde Park  MA  02136|119 L St  South Boston  MA  02127|12 Derne St  Boston  MA  02114|126 Elm St  Charlestown  MA  02129|1270 Commonwealth Ave  Allston  MA  02134|130 Shirley St  Roxbury  MA  02119|131 Arlington St  Boston  MA  02116,96,1,
+fire_district,String,true,, ,9,,Unsorted,0.1515,1,2,113,1.13,0.3363,0.1131,0.2976,,,,,,,,0,,0,,,,,,,,,,,10,3,1,19, ,1,1,
+pwd_district,String,true,, ,1C,,Unsorted,0.0707,1,3,209,2.09,0.3192,0.1019,0.1527,,,,,,,,0,,0,,,,,,,,,,,14,1B,1,16, ,1,1,
+city_council_district,String,true,, ,9,,Unsorted,0.1313,1,1,100,1,0,0,0,,,,,,,,0,,0,,,,,,,,,,,10,1,1,22, ,1,1,
+police_district,String,true,, ,E5,,Unsorted,0.1717,1,3,223,2.23,0.444,0.1971,0.1991,,,,,,,,0,,0,,,,,,,,,,,13,A1,1,20, ,1,1,
+neighborhood,String,true,, ,West Roxbury,,Unsorted,0.0303,1,38,1486,14.86,9.8671,97.3604,0.664,,,,,,,,0,,0,,,,,,,,,,,19,Dorchester,1,15, |Brighton|Mission Hill,3,1,
+neighborhood_services_district,String,true,, ,9,,Unsorted,0.0303,1,2,139,1.39,0.4877,0.2379,0.3509,,,,,,,,0,,0,,,,,,,,,,,16,3,1,15, |12,2,1,
+ward,String,true,, ,Ward 9,,Unsorted,0.0505,1,7,499,4.99,2.2293,4.9699,0.4468,,,,,,,,0,,0,,,,,,,,,,,42,Ward 3,1,10,*PREVIEW:  |01|02|04|06|07|1|10|16|18,23,1,
+precinct,String,true,, ,2210,,Unsorted,0.0408,0,4,393,3.93,0.4951,0.2451,0.126,,,,,,,,1,,0.01,,,,,,,,,,,76,0306,1,5,*PREVIEW: NULL| |0102|0105|0108|0109|0201|0204|0305|0307,61,1,
+location_street_name,String,true,,103 N Beacon St,INTERSECTION Verdun St & Gallivan Blvd,,Unsorted,-0.0204,0,45,1800,18,9.3995,88.3508,0.5222,,,,,,,,1,,0.01,,,,,,,,,,,97,20 Washington St|563 Columbus Ave|INTERSECTION Gallivan Blvd & Washington St,3,2,*PREVIEW: NULL|103 N Beacon St|11 Aberdeen St|1148 Hyde Park Ave|119 L St|12 Derne St|126 Elm St|1270 Commonwealth Ave|130 Shirley St|131 Arlington St,94,1,
+location_zipcode,String,true,,02109,02215,,Unsorted,0.0488,0,5,415,4.15,1.8405,3.3874,0.4435,,,,,,,,17,,0.17,,,,,,,,,,,24,,1,17,02126|02134|02210|02215,4,1,
+latitude,Float,,4233.6674,42.2553,42.3806,0.1253,Unsorted,0.0505,,,,,,,,42.3367,0.0031,42.3367,42.3367,0.0305,0.0009,0.072,0,4,0,0.0163,42.2034,42.2619,42.3204,42.3432,42.3594,0.039,42.4179,42.4764,-0.1667,78,42.3594,1,20,*PREVIEW: 42.2553|42.2601|42.2609|42.2645|42.2674|42.2789|42.2797|42.2804|42.2821|42.2878,74,1,
+longitude,Float,,-7107.2688,-71.1626,-71.0298,0.1328,Unsorted,0.1515,,,,,,,,-71.0727,0.0031,,,0.0311,0.001,-0.0437,0,4,0,0.0121,-71.1741,-71.1294,-71.0848,-71.0609,-71.055,0.0298,-71.0104,-70.9658,-0.6101,77,-71.0587,1,19,*PREVIEW: -71.0298|-71.0301|-71.0309|-71.0323|-71.0325|-71.0329|-71.0336|-71.0338|-71.034|-71.0355,72,1,
+source,String,true,,Citizens Connect App,Self Service,,Unsorted,0.5354,12,20,1801,18.01,2.3473,5.5099,0.1303,,,,,,,,0,,0,,,,,,,,,,,4,Citizens Connect App,1,56,Self Service,1,3,
+qsv__rowcount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,100
+qsv__columncount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,29
+qsv__filesize_bytes,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,47702
+qsv__fingerprint_hash,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2427f44545835ae925b85f6c5fc713f9aaf81756146c770e19369ab90387e062
diff --git a/resources/test/boston311-100-everything-8places-stats.csv b/resources/test/boston311-100-everything-8places-stats.csv
index f562b4b5e..c3b4a61ab 100644
--- a/resources/test/boston311-100-everything-8places-stats.csv
+++ b/resources/test/boston311-100-everything-8places-stats.csv
@@ -1,34 +1,34 @@
-field,type,is_ascii,sum,min,max,range,sort_order,min_length,max_length,sum_length,avg_length,mean,nullcount,max_precision,sparsity,mad,lower_outer_fence,lower_inner_fence,q1,q2_median,q3,iqr,upper_inner_fence,upper_outer_fence,skewness,cardinality,mode,mode_count,mode_occurrences,antimode,antimode_count,antimode_occurrences,qsv__value
-case_enquiry_id,Integer,,10100411645180,101004113298,101004155594,42296,Unsorted,,,,,101004116451.80003357,0,,0,673,101004109567,101004111646,101004113725,101004114353,101004115111,1386,101004117190,101004119269,0.09379509,100,,0,0,*ALL,0,1,
-open_dt,String,true,,2022-01-01 00:16:00,2022-01-31 11:46:00,,Unsorted,19,19,1900,19,,0,,0,,,,,,,,,,,100,,0,0,*ALL,0,1,
-target_dt,String,true,,2022-01-03 10:32:34,2022-05-20 13:03:21,,Unsorted,0,19,1691,16.91,,11,,0.11,,,,,,,,,,,42,2022-01-04 08:30:00,1,25,*PREVIEW: 2022-01-03 10:32:34|2022-01-03 11:58:12|2022-01-04 09:58:36|2022-01-04 10:41:29|2022-01-04...,34,1,
-closed_dt,String,true,,2022-01-01 12:56:14,2022-04-25 14:30:31,,Unsorted,0,19,1615,16.15,,15,,0.15,,,,,,,,,,,86,,1,15,*PREVIEW: 2022-01-01 12:56:14|2022-01-01 14:17:15|2022-01-01 14:59:41|2022-01-01 15:10:16|2022-01-01...,85,1,
-ontime,String,true,,ONTIME,OVERDUE,,Unsorted,6,7,617,6.17,,0,,0,,,,,,,,,,,2,ONTIME,1,83,OVERDUE,1,17,
-case_status,String,true,,Closed,Open,,Unsorted,4,6,570,5.7,,0,,0,,,,,,,,,,,2,Closed,1,85,Open,1,15,
-closure_reason,String,true,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,1,284,8314,83.14,,0,,0,,,,,,,,,,,86, ,1,15,*PREVIEW: Case Closed Case Resolved  NEW CART#21026466 DELV ON 1/11/22  |Case Closed Case Resolved  ...,85,1,
-case_title,String,true,,Abandoned Vehicles,Traffic Signal Inspection,,Unsorted,10,57,2386,23.86,,0,,0,,,,,,,,,,,42,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|BTDT: Complaint|City/State Snow Issues|DISPATCHED Short Term Rental...,24,1,
-subject,String,true,,Animal Control,Transportation - Traffic Division,,Unsorted,14,33,2570,25.7,,0,,0,,,,,,,,,,,9,Public Works Department,1,51,Animal Control|Boston Police Department|Boston Water & Sewer Commission,3,1,
-reason,String,true,,Administrative & General Requests,Street Lights,,Unsorted,7,33,1892,18.92,,0,,0,,,,,,,,,,,20,Enforcement & Abandoned Vehicles,1,23,Administrative & General Requests|Animal Issues|Building|Employee & General Comments|Noise Disturban...,7,1,
-type,String,true,,Abandoned Vehicles,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,10,47,2266,22.66,,0,,0,,,,,,,,,,,36,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|City/State Snow Issues|Electrical|General Comments For a Program or...,15,1,
-queue,String,true,,BTDT_AVRS Interface Queue,PWDx_Street Light_General Lighting Request,,Unsorted,13,55,2802,28.02,,0,,0,,,,,,,,,,,35,BTDT_Parking Enforcement,1,21,*PREVIEW: BTDT_BostonBikes|BTDT_Engineering_New Sign and Pavement Marking Requests|BTDT_Sign Shop_Si...,15,1,
-department,String,true,,BTDT,PWDx,,Unsorted,3,4,392,3.92,,0,,0,,,,,,,,,,,7,PWDx,1,49,GEN_,1,2,
-submittedphoto,String,true,,https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg,https://311.boston.gov/media/boston/report/photos/61d75bba05bbcf180c2d41de/report.jpg,,Unsorted,0,100,3633,36.33,,58,,0.58,,,,,,,,,,,43,,1,58,*PREVIEW: https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg|http...,42,1,
-closedphoto,NULL,,,,,,,,,,,,100,,1,,,,,,,,,,,1,,1,100,,0,0,
-location,String,true,, ,INTERSECTION of Verdun St & Gallivan Blvd  Dorchester  MA  ,,Unsorted,1,63,3938,39.38,,0,,0,,,,,,,,,,,98,563 Columbus Ave  Roxbury  MA  02118|INTERSECTION of Gallivan Blvd & Washington St  Dorchester  MA  ,2,2,*PREVIEW:  |103 N Beacon St  Brighton  MA  02135|11 Aberdeen St  Boston  MA  02215|1148 Hyde Park Av...,96,1,
-fire_district,String,true,, ,9,,Unsorted,1,2,113,1.13,,0,,0,,,,,,,,,,,10,3,1,19, ,1,1,
-pwd_district,String,true,, ,1C,,Unsorted,1,3,209,2.09,,0,,0,,,,,,,,,,,14,1B,1,16, ,1,1,
-city_council_district,String,true,, ,9,,Unsorted,1,1,100,1,,0,,0,,,,,,,,,,,10,1,1,22, ,1,1,
-police_district,String,true,, ,E5,,Unsorted,1,3,223,2.23,,0,,0,,,,,,,,,,,13,A1,1,20, ,1,1,
-neighborhood,String,true,, ,West Roxbury,,Unsorted,1,38,1486,14.86,,0,,0,,,,,,,,,,,19,Dorchester,1,15, |Brighton|Mission Hill,3,1,
-neighborhood_services_district,String,true,, ,9,,Unsorted,1,2,139,1.39,,0,,0,,,,,,,,,,,16,3,1,15, |12,2,1,
-ward,String,true,, ,Ward 9,,Unsorted,1,7,499,4.99,,0,,0,,,,,,,,,,,42,Ward 3,1,10,*PREVIEW:  |01|02|04|06|07|1|10|16|18,23,1,
-precinct,String,true,, ,2210,,Unsorted,0,4,393,3.93,,1,,0.01,,,,,,,,,,,76,0306,1,5,*PREVIEW: NULL| |0102|0105|0108|0109|0201|0204|0305|0307,61,1,
-location_street_name,String,true,,103 N Beacon St,INTERSECTION Verdun St & Gallivan Blvd,,Unsorted,0,45,1800,18,,1,,0.01,,,,,,,,,,,97,20 Washington St|563 Columbus Ave|INTERSECTION Gallivan Blvd & Washington St,3,2,*PREVIEW: NULL|103 N Beacon St|11 Aberdeen St|1148 Hyde Park Ave|119 L St|12 Derne St|126 Elm St|127...,94,1,
-location_zipcode,String,true,,02109,02215,,Unsorted,0,5,415,4.15,,17,,0.17,,,,,,,,,,,24,,1,17,02126|02134|02210|02215,4,1,
-latitude,Float,,4233.6674,42.2553,42.3806,0.1253,Unsorted,,,,,42.336674,0,4,0,0.0163,42.2034,42.2619,42.3204,42.34315,42.3594,0.039,42.4179,42.4764,-0.16666667,78,42.3594,1,20,*PREVIEW: 42.2553|42.2601|42.2609|42.2645|42.2674|42.2789|42.2797|42.2804|42.2821|42.2878,74,1,
-longitude,Float,,-7107.2688,-71.1626,-71.0298,0.1328,Unsorted,,,,,-71.072688,0,4,0,0.01205,-71.17405,-71.129425,-71.0848,-71.06085,-71.05505,0.02975,-71.010425,-70.9658,-0.61008403,77,-71.0587,1,19,*PREVIEW: -71.0298|-71.0301|-71.0309|-71.0323|-71.0325|-71.0329|-71.0336|-71.0338|-71.034|-71.0355,72,1,
-source,String,true,,Citizens Connect App,Self Service,,Unsorted,12,20,1801,18.01,,0,,0,,,,,,,,,,,4,Citizens Connect App,1,56,Self Service,1,3,
-qsv__rowcount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,100
-qsv__columncount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,29
-qsv__filesize_bytes,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,47702
-qsv__fingerprint_hash,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,9e791e33efb6c780c4b18003de559313719e23f16288dbe391c0dcf5e35cfb22
+field,type,is_ascii,sum,min,max,range,sort_order,sortiness,min_length,max_length,sum_length,avg_length,mean,nullcount,max_precision,sparsity,mad,lower_outer_fence,lower_inner_fence,q1,q2_median,q3,iqr,upper_inner_fence,upper_outer_fence,skewness,cardinality,mode,mode_count,mode_occurrences,antimode,antimode_count,antimode_occurrences,qsv__value
+case_enquiry_id,Integer,,10100411645180,101004113298,101004155594,42296,Unsorted,0.7979798,,,,,101004116451.80003357,0,,0,673,101004109567,101004111646,101004113725,101004114353,101004115111,1386,101004117190,101004119269,0.09379509,100,,0,0,*ALL,0,1,
+open_dt,String,true,,2022-01-01 00:16:00,2022-01-31 11:46:00,,Unsorted,0.7979798,19,19,1900,19,,0,,0,,,,,,,,,,,100,,0,0,*ALL,0,1,
+target_dt,String,true,,2022-01-03 10:32:34,2022-05-20 13:03:21,,Unsorted,0.22727273,0,19,1691,16.91,,11,,0.11,,,,,,,,,,,42,2022-01-04 08:30:00,1,25,*PREVIEW: 2022-01-03 10:32:34|2022-01-03 11:58:12|2022-01-04 09:58:36|2022-01-04 10:41:29|2022-01-04...,34,1,
+closed_dt,String,true,,2022-01-01 12:56:14,2022-04-25 14:30:31,,Unsorted,-0.07142857,0,19,1615,16.15,,15,,0.15,,,,,,,,,,,86,,1,15,*PREVIEW: 2022-01-01 12:56:14|2022-01-01 14:17:15|2022-01-01 14:59:41|2022-01-01 15:10:16|2022-01-01...,85,1,
+ontime,String,true,,ONTIME,OVERDUE,,Unsorted,0.67676768,6,7,617,6.17,,0,,0,,,,,,,,,,,2,ONTIME,1,83,OVERDUE,1,17,
+case_status,String,true,,Closed,Open,,Unsorted,0.71717172,4,6,570,5.7,,0,,0,,,,,,,,,,,2,Closed,1,85,Open,1,15,
+closure_reason,String,true,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,-0.09090909,1,284,8314,83.14,,0,,0,,,,,,,,,,,86, ,1,15,*PREVIEW: Case Closed Case Resolved  NEW CART#21026466 DELV ON 1/11/22  |Case Closed Case Resolved  ...,85,1,
+case_title,String,true,,Abandoned Vehicles,Traffic Signal Inspection,,Unsorted,0.01010101,10,57,2386,23.86,,0,,0,,,,,,,,,,,42,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|BTDT: Complaint|City/State Snow Issues|DISPATCHED Short Term Rental...,24,1,
+subject,String,true,,Animal Control,Transportation - Traffic Division,,Unsorted,0.37373737,14,33,2570,25.7,,0,,0,,,,,,,,,,,9,Public Works Department,1,51,Animal Control|Boston Police Department|Boston Water & Sewer Commission,3,1,
+reason,String,true,,Administrative & General Requests,Street Lights,,Unsorted,0.17171717,7,33,1892,18.92,,0,,0,,,,,,,,,,,20,Enforcement & Abandoned Vehicles,1,23,Administrative & General Requests|Animal Issues|Building|Employee & General Comments|Noise Disturban...,7,1,
+type,String,true,,Abandoned Vehicles,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,0.07070707,10,47,2266,22.66,,0,,0,,,,,,,,,,,36,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|City/State Snow Issues|Electrical|General Comments For a Program or...,15,1,
+queue,String,true,,BTDT_AVRS Interface Queue,PWDx_Street Light_General Lighting Request,,Unsorted,0.21212121,13,55,2802,28.02,,0,,0,,,,,,,,,,,35,BTDT_Parking Enforcement,1,21,*PREVIEW: BTDT_BostonBikes|BTDT_Engineering_New Sign and Pavement Marking Requests|BTDT_Sign Shop_Si...,15,1,
+department,String,true,,BTDT,PWDx,,Unsorted,0.37373737,3,4,392,3.92,,0,,0,,,,,,,,,,,7,PWDx,1,49,GEN_,1,2,
+submittedphoto,String,true,,https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg,https://311.boston.gov/media/boston/report/photos/61d75bba05bbcf180c2d41de/report.jpg,,Unsorted,0.85365854,0,100,3633,36.33,,58,,0.58,,,,,,,,,,,43,,1,58,*PREVIEW: https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg|http...,42,1,
+closedphoto,NULL,,,,,,,,,,,,,100,,1,,,,,,,,,,,1,,1,100,,0,0,
+location,String,true,, ,INTERSECTION of Verdun St & Gallivan Blvd  Dorchester  MA  ,,Unsorted,-0.03030303,1,63,3938,39.38,,0,,0,,,,,,,,,,,98,563 Columbus Ave  Roxbury  MA  02118|INTERSECTION of Gallivan Blvd & Washington St  Dorchester  MA  ,2,2,*PREVIEW:  |103 N Beacon St  Brighton  MA  02135|11 Aberdeen St  Boston  MA  02215|1148 Hyde Park Av...,96,1,
+fire_district,String,true,, ,9,,Unsorted,0.15151515,1,2,113,1.13,,0,,0,,,,,,,,,,,10,3,1,19, ,1,1,
+pwd_district,String,true,, ,1C,,Unsorted,0.07070707,1,3,209,2.09,,0,,0,,,,,,,,,,,14,1B,1,16, ,1,1,
+city_council_district,String,true,, ,9,,Unsorted,0.13131313,1,1,100,1,,0,,0,,,,,,,,,,,10,1,1,22, ,1,1,
+police_district,String,true,, ,E5,,Unsorted,0.17171717,1,3,223,2.23,,0,,0,,,,,,,,,,,13,A1,1,20, ,1,1,
+neighborhood,String,true,, ,West Roxbury,,Unsorted,0.03030303,1,38,1486,14.86,,0,,0,,,,,,,,,,,19,Dorchester,1,15, |Brighton|Mission Hill,3,1,
+neighborhood_services_district,String,true,, ,9,,Unsorted,0.03030303,1,2,139,1.39,,0,,0,,,,,,,,,,,16,3,1,15, |12,2,1,
+ward,String,true,, ,Ward 9,,Unsorted,0.05050505,1,7,499,4.99,,0,,0,,,,,,,,,,,42,Ward 3,1,10,*PREVIEW:  |01|02|04|06|07|1|10|16|18,23,1,
+precinct,String,true,, ,2210,,Unsorted,0.04081633,0,4,393,3.93,,1,,0.01,,,,,,,,,,,76,0306,1,5,*PREVIEW: NULL| |0102|0105|0108|0109|0201|0204|0305|0307,61,1,
+location_street_name,String,true,,103 N Beacon St,INTERSECTION Verdun St & Gallivan Blvd,,Unsorted,-0.02040816,0,45,1800,18,,1,,0.01,,,,,,,,,,,97,20 Washington St|563 Columbus Ave|INTERSECTION Gallivan Blvd & Washington St,3,2,*PREVIEW: NULL|103 N Beacon St|11 Aberdeen St|1148 Hyde Park Ave|119 L St|12 Derne St|126 Elm St|127...,94,1,
+location_zipcode,String,true,,02109,02215,,Unsorted,0.04878049,0,5,415,4.15,,17,,0.17,,,,,,,,,,,24,,1,17,02126|02134|02210|02215,4,1,
+latitude,Float,,4233.6674,42.2553,42.3806,0.1253,Unsorted,0.05050505,,,,,42.336674,0,4,0,0.0163,42.2034,42.2619,42.3204,42.34315,42.3594,0.039,42.4179,42.4764,-0.16666667,78,42.3594,1,20,*PREVIEW: 42.2553|42.2601|42.2609|42.2645|42.2674|42.2789|42.2797|42.2804|42.2821|42.2878,74,1,
+longitude,Float,,-7107.2688,-71.1626,-71.0298,0.1328,Unsorted,0.15151515,,,,,-71.072688,0,4,0,0.01205,-71.17405,-71.129425,-71.0848,-71.06085,-71.05505,0.02975,-71.010425,-70.9658,-0.61008403,77,-71.0587,1,19,*PREVIEW: -71.0298|-71.0301|-71.0309|-71.0323|-71.0325|-71.0329|-71.0336|-71.0338|-71.034|-71.0355,72,1,
+source,String,true,,Citizens Connect App,Self Service,,Unsorted,0.53535354,12,20,1801,18.01,,0,,0,,,,,,,,,,,4,Citizens Connect App,1,56,Self Service,1,3,
+qsv__rowcount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,100
+qsv__columncount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,29
+qsv__filesize_bytes,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,47702
+qsv__fingerprint_hash,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,b26048ae4592f5f2f3631ece8d1264a03de76d9d6da80fa133bb300969b347be
diff --git a/resources/test/boston311-100-everything-date-stats-variance-stddev.csv b/resources/test/boston311-100-everything-date-stats-variance-stddev.csv
index 9553a32f5..b990d703c 100644
--- a/resources/test/boston311-100-everything-date-stats-variance-stddev.csv
+++ b/resources/test/boston311-100-everything-date-stats-variance-stddev.csv
@@ -1,34 +1,34 @@
-field,type,is_ascii,sum,min,max,range,sort_order,min_length,max_length,sum_length,avg_length,stddev_length,variance_length,cv_length,mean,sem,geometric_mean,harmonic_mean,stddev,variance,cv,nullcount,max_precision,sparsity,mad,lower_outer_fence,lower_inner_fence,q1,q2_median,q3,iqr,upper_inner_fence,upper_outer_fence,skewness,cardinality,mode,mode_count,mode_occurrences,antimode,antimode_count,antimode_occurrences,qsv__value
-case_enquiry_id,Integer,,10100411645180,101004113298,101004155594,42296,Unsorted,,,,,,,,101004116451.8,790.552,101004116451.8012,101004116451.7994,7905.5202,62497248.9138,0,0,,0,673,101004109567,101004111646,101004113725,101004114353,101004115111,1386,101004117190,101004119269,0.0938,100,,0,0,*ALL,0,1,
-open_dt,DateTime,,,2022-01-01T00:16:00+00:00,2022-01-31T11:46:00+00:00,30.47917,Unsorted,,,,,,,,2022-01-04T07:07:45.050+00:00,0.5568,18996.29623,18996.29542,5.568,31.00259,0.0293,0,,0,0.76261,2021-12-27T14:16:49+00:00,2021-12-30T06:00:07+00:00,2022-01-01T21:43:25+00:00,2022-01-03T07:02:14+00:00,2022-01-03T16:12:17+00:00,1.77005,2022-01-06T07:55:35+00:00,2022-01-08T23:38:53+00:00,-0.5684,100,,0,0,*ALL,0,1,
-target_dt,DateTime,,,2022-01-03T10:32:34+00:00,2022-05-20T13:03:21+00:00,137.10471,Unsorted,,,,,,,,2022-01-17T03:14:16.404+00:00,2.86258,19009.11578,19009.0967,27.00551,729.29774,0.1421,11,,0.11,1,2021-11-26T08:30:00+00:00,2021-12-15T20:30:00+00:00,2022-01-04T08:30:00+00:00,2022-01-05T08:30:00+00:00,2022-01-17T08:30:00+00:00,13,2022-02-05T20:30:00+00:00,2022-02-25T08:30:00+00:00,0.8462,42,2022-01-04 08:30:00,1,25,*PREVIEW: 2022-01-03 10:32:34|2022-01-03 11:58:12|2022-01-04 09:58:36|2022-01-04 10:41:29|2022-01-04...,34,1,
-closed_dt,DateTime,,,2022-01-01T12:56:14+00:00,2022-04-25T14:30:31+00:00,114.06547,Unsorted,,,,,,,,2022-01-08T01:10:44.411+00:00,1.71655,19000.04255,19000.036,15.82577,250.4549,0.0833,15,,0.15,0.77213,2021-12-29T15:13:29+00:00,2021-12-31T19:50:08.750+00:00,2022-01-03T00:26:48.500+00:00,2022-01-03T12:15:23+00:00,2022-01-04T11:31:15+00:00,1.46142,2022-01-06T16:07:54.750+00:00,2022-01-08T20:44:34.500+00:00,0.3266,86,,1,15,*PREVIEW: 2022-01-01 12:56:14|2022-01-01 14:17:15|2022-01-01 14:59:41|2022-01-01 15:10:16|2022-01-01...,85,1,
-ontime,String,true,,ONTIME,OVERDUE,,Unsorted,6,7,617,6.17,0.3756,0.1411,0.0609,,,,,,,,0,,0,,,,,,,,,,,2,ONTIME,1,83,OVERDUE,1,17,
-case_status,String,true,,Closed,Open,,Unsorted,4,6,570,5.7,0.7141,0.51,0.1253,,,,,,,,0,,0,,,,,,,,,,,2,Closed,1,85,Open,1,15,
-closure_reason,String,true,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,1,284,8314,83.14,55.0262,3027.8804,0.6618,,,,,,,,0,,0,,,,,,,,,,,86, ,1,15,*PREVIEW: Case Closed Case Resolved  NEW CART#21026466 DELV ON 1/11/22  |Case Closed Case Resolved  ...,85,1,
-case_title,String,true,,Abandoned Vehicles,Traffic Signal Inspection,,Unsorted,10,57,2386,23.86,9.452,89.3404,0.3961,,,,,,,,0,,0,,,,,,,,,,,42,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|BTDT: Complaint|City/State Snow Issues|DISPATCHED Short Term Rental...,24,1,
-subject,String,true,,Animal Control,Transportation - Traffic Division,,Unsorted,14,33,2570,25.7,4.9041,24.05,0.1908,,,,,,,,0,,0,,,,,,,,,,,9,Public Works Department,1,51,Animal Control|Boston Police Department|Boston Water & Sewer Commission,3,1,
-reason,String,true,,Administrative & General Requests,Street Lights,,Unsorted,7,33,1892,18.92,8.4056,70.6536,0.4443,,,,,,,,0,,0,,,,,,,,,,,20,Enforcement & Abandoned Vehicles,1,23,Administrative & General Requests|Animal Issues|Building|Employee & General Comments|Noise Disturban...,7,1,
-type,String,true,,Abandoned Vehicles,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,10,47,2266,22.66,8.034,64.5444,0.3545,,,,,,,,0,,0,,,,,,,,,,,36,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|City/State Snow Issues|Electrical|General Comments For a Program or...,15,1,
-queue,String,true,,BTDT_AVRS Interface Queue,PWDx_Street Light_General Lighting Request,,Unsorted,13,55,2802,28.02,8.0224,64.3596,0.2863,,,,,,,,0,,0,,,,,,,,,,,35,BTDT_Parking Enforcement,1,21,*PREVIEW: BTDT_BostonBikes|BTDT_Engineering_New Sign and Pavement Marking Requests|BTDT_Sign Shop_Si...,15,1,
-department,String,true,,BTDT,PWDx,,Unsorted,3,4,392,3.92,0.2713,0.0736,0.0692,,,,,,,,0,,0,,,,,,,,,,,7,PWDx,1,49,GEN_,1,2,
-submittedphoto,String,true,,https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg,https://311.boston.gov/media/boston/report/photos/61d75bba05bbcf180c2d41de/report.jpg,,Unsorted,0,100,3633,36.33,43.2585,1871.2989,1.1907,,,,,,,,58,,0.58,,,,,,,,,,,43,,1,58,*PREVIEW: https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg|http...,42,1,
-closedphoto,NULL,,,,,,,,,,,,,,,,,,,,,100,,1,,,,,,,,,,,1,,1,100,,0,0,
-location,String,true,, ,INTERSECTION of Verdun St & Gallivan Blvd  Dorchester  MA  ,,Unsorted,1,63,3938,39.38,9.6247,92.6356,0.2444,,,,,,,,0,,0,,,,,,,,,,,98,563 Columbus Ave  Roxbury  MA  02118|INTERSECTION of Gallivan Blvd & Washington St  Dorchester  MA  ,2,2,*PREVIEW:  |103 N Beacon St  Brighton  MA  02135|11 Aberdeen St  Boston  MA  02215|1148 Hyde Park Av...,96,1,
-fire_district,String,true,, ,9,,Unsorted,1,2,113,1.13,0.3363,0.1131,0.2976,,,,,,,,0,,0,,,,,,,,,,,10,3,1,19, ,1,1,
-pwd_district,String,true,, ,1C,,Unsorted,1,3,209,2.09,0.3192,0.1019,0.1527,,,,,,,,0,,0,,,,,,,,,,,14,1B,1,16, ,1,1,
-city_council_district,String,true,, ,9,,Unsorted,1,1,100,1,0,0,0,,,,,,,,0,,0,,,,,,,,,,,10,1,1,22, ,1,1,
-police_district,String,true,, ,E5,,Unsorted,1,3,223,2.23,0.444,0.1971,0.1991,,,,,,,,0,,0,,,,,,,,,,,13,A1,1,20, ,1,1,
-neighborhood,String,true,, ,West Roxbury,,Unsorted,1,38,1486,14.86,9.8671,97.3604,0.664,,,,,,,,0,,0,,,,,,,,,,,19,Dorchester,1,15, |Brighton|Mission Hill,3,1,
-neighborhood_services_district,String,true,, ,9,,Unsorted,1,2,139,1.39,0.4877,0.2379,0.3509,,,,,,,,0,,0,,,,,,,,,,,16,3,1,15, |12,2,1,
-ward,String,true,, ,Ward 9,,Unsorted,1,7,499,4.99,2.2293,4.9699,0.4468,,,,,,,,0,,0,,,,,,,,,,,42,Ward 3,1,10,*PREVIEW:  |01|02|04|06|07|1|10|16|18,23,1,
-precinct,String,true,, ,2210,,Unsorted,0,4,393,3.93,0.4951,0.2451,0.126,,,,,,,,1,,0.01,,,,,,,,,,,76,0306,1,5,*PREVIEW: NULL| |0102|0105|0108|0109|0201|0204|0305|0307,61,1,
-location_street_name,String,true,,103 N Beacon St,INTERSECTION Verdun St & Gallivan Blvd,,Unsorted,0,45,1800,18,9.3995,88.3508,0.5222,,,,,,,,1,,0.01,,,,,,,,,,,97,20 Washington St|563 Columbus Ave|INTERSECTION Gallivan Blvd & Washington St,3,2,*PREVIEW: NULL|103 N Beacon St|11 Aberdeen St|1148 Hyde Park Ave|119 L St|12 Derne St|126 Elm St|127...,94,1,
-location_zipcode,String,true,,02109,02215,,Unsorted,0,5,415,4.15,1.8405,3.3874,0.4435,,,,,,,,17,,0.17,,,,,,,,,,,24,,1,17,02126|02134|02210|02215,4,1,
-latitude,Float,,4233.6674,42.2553,42.3806,0.1253,Unsorted,,,,,,,,42.3367,0.0031,42.3367,42.3367,0.0305,0.0009,0.072,0,4,0,0.0163,42.2034,42.2619,42.3204,42.3432,42.3594,0.039,42.4179,42.4764,-0.1667,78,42.3594,1,20,*PREVIEW: 42.2553|42.2601|42.2609|42.2645|42.2674|42.2789|42.2797|42.2804|42.2821|42.2878,74,1,
-longitude,Float,,-7107.2688,-71.1626,-71.0298,0.1328,Unsorted,,,,,,,,-71.0727,0.0031,,,0.0311,0.001,-0.0437,0,4,0,0.0121,-71.1741,-71.1294,-71.0848,-71.0609,-71.055,0.0298,-71.0104,-70.9658,-0.6101,77,-71.0587,1,19,*PREVIEW: -71.0298|-71.0301|-71.0309|-71.0323|-71.0325|-71.0329|-71.0336|-71.0338|-71.034|-71.0355,72,1,
-source,String,true,,Citizens Connect App,Self Service,,Unsorted,12,20,1801,18.01,2.3473,5.5099,0.1303,,,,,,,,0,,0,,,,,,,,,,,4,Citizens Connect App,1,56,Self Service,1,3,
-qsv__rowcount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,100
-qsv__columncount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,29
-qsv__filesize_bytes,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,47702
-qsv__fingerprint_hash,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,85611c6f33acdc78001619e131cbf15cd0e2cc41abb8865ec879a0b529ce2642
+field,type,is_ascii,sum,min,max,range,sort_order,sortiness,min_length,max_length,sum_length,avg_length,stddev_length,variance_length,cv_length,mean,sem,geometric_mean,harmonic_mean,stddev,variance,cv,nullcount,max_precision,sparsity,mad,lower_outer_fence,lower_inner_fence,q1,q2_median,q3,iqr,upper_inner_fence,upper_outer_fence,skewness,cardinality,mode,mode_count,mode_occurrences,antimode,antimode_count,antimode_occurrences,qsv__value
+case_enquiry_id,Integer,,10100411645180,101004113298,101004155594,42296,Unsorted,0.798,,,,,,,,101004116451.8,790.552,101004116451.8012,101004116451.7994,7905.5202,62497248.9138,0,0,,0,673,101004109567,101004111646,101004113725,101004114353,101004115111,1386,101004117190,101004119269,0.0938,100,,0,0,*ALL,0,1,
+open_dt,DateTime,,,2022-01-01T00:16:00+00:00,2022-01-31T11:46:00+00:00,30.47917,Unsorted,0.798,,,,,,,,2022-01-04T07:07:45.050+00:00,0.5568,18996.29623,18996.29542,5.568,31.00259,0.0293,0,,0,0.76261,2021-12-27T14:16:49+00:00,2021-12-30T06:00:07+00:00,2022-01-01T21:43:25+00:00,2022-01-03T07:02:14+00:00,2022-01-03T16:12:17+00:00,1.77005,2022-01-06T07:55:35+00:00,2022-01-08T23:38:53+00:00,-0.5684,100,,0,0,*ALL,0,1,
+target_dt,DateTime,,,2022-01-03T10:32:34+00:00,2022-05-20T13:03:21+00:00,137.10471,Unsorted,0.2273,,,,,,,,2022-01-17T03:14:16.404+00:00,2.86258,19009.11578,19009.0967,27.00551,729.29774,0.1421,11,,0.11,1,2021-11-26T08:30:00+00:00,2021-12-15T20:30:00+00:00,2022-01-04T08:30:00+00:00,2022-01-05T08:30:00+00:00,2022-01-17T08:30:00+00:00,13,2022-02-05T20:30:00+00:00,2022-02-25T08:30:00+00:00,0.8462,42,2022-01-04 08:30:00,1,25,*PREVIEW: 2022-01-03 10:32:34|2022-01-03 11:58:12|2022-01-04 09:58:36|2022-01-04 10:41:29|2022-01-04...,34,1,
+closed_dt,DateTime,,,2022-01-01T12:56:14+00:00,2022-04-25T14:30:31+00:00,114.06547,Unsorted,-0.0714,,,,,,,,2022-01-08T01:10:44.411+00:00,1.71655,19000.04255,19000.036,15.82577,250.4549,0.0833,15,,0.15,0.77213,2021-12-29T15:13:29+00:00,2021-12-31T19:50:08.750+00:00,2022-01-03T00:26:48.500+00:00,2022-01-03T12:15:23+00:00,2022-01-04T11:31:15+00:00,1.46142,2022-01-06T16:07:54.750+00:00,2022-01-08T20:44:34.500+00:00,0.3266,86,,1,15,*PREVIEW: 2022-01-01 12:56:14|2022-01-01 14:17:15|2022-01-01 14:59:41|2022-01-01 15:10:16|2022-01-01...,85,1,
+ontime,String,true,,ONTIME,OVERDUE,,Unsorted,0.6768,6,7,617,6.17,0.3756,0.1411,0.0609,,,,,,,,0,,0,,,,,,,,,,,2,ONTIME,1,83,OVERDUE,1,17,
+case_status,String,true,,Closed,Open,,Unsorted,0.7172,4,6,570,5.7,0.7141,0.51,0.1253,,,,,,,,0,,0,,,,,,,,,,,2,Closed,1,85,Open,1,15,
+closure_reason,String,true,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,-0.0909,1,284,8314,83.14,55.0262,3027.8804,0.6618,,,,,,,,0,,0,,,,,,,,,,,86, ,1,15,*PREVIEW: Case Closed Case Resolved  NEW CART#21026466 DELV ON 1/11/22  |Case Closed Case Resolved  ...,85,1,
+case_title,String,true,,Abandoned Vehicles,Traffic Signal Inspection,,Unsorted,0.0101,10,57,2386,23.86,9.452,89.3404,0.3961,,,,,,,,0,,0,,,,,,,,,,,42,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|BTDT: Complaint|City/State Snow Issues|DISPATCHED Short Term Rental...,24,1,
+subject,String,true,,Animal Control,Transportation - Traffic Division,,Unsorted,0.3737,14,33,2570,25.7,4.9041,24.05,0.1908,,,,,,,,0,,0,,,,,,,,,,,9,Public Works Department,1,51,Animal Control|Boston Police Department|Boston Water & Sewer Commission,3,1,
+reason,String,true,,Administrative & General Requests,Street Lights,,Unsorted,0.1717,7,33,1892,18.92,8.4056,70.6536,0.4443,,,,,,,,0,,0,,,,,,,,,,,20,Enforcement & Abandoned Vehicles,1,23,Administrative & General Requests|Animal Issues|Building|Employee & General Comments|Noise Disturban...,7,1,
+type,String,true,,Abandoned Vehicles,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,0.0707,10,47,2266,22.66,8.034,64.5444,0.3545,,,,,,,,0,,0,,,,,,,,,,,36,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|City/State Snow Issues|Electrical|General Comments For a Program or...,15,1,
+queue,String,true,,BTDT_AVRS Interface Queue,PWDx_Street Light_General Lighting Request,,Unsorted,0.2121,13,55,2802,28.02,8.0224,64.3596,0.2863,,,,,,,,0,,0,,,,,,,,,,,35,BTDT_Parking Enforcement,1,21,*PREVIEW: BTDT_BostonBikes|BTDT_Engineering_New Sign and Pavement Marking Requests|BTDT_Sign Shop_Si...,15,1,
+department,String,true,,BTDT,PWDx,,Unsorted,0.3737,3,4,392,3.92,0.2713,0.0736,0.0692,,,,,,,,0,,0,,,,,,,,,,,7,PWDx,1,49,GEN_,1,2,
+submittedphoto,String,true,,https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg,https://311.boston.gov/media/boston/report/photos/61d75bba05bbcf180c2d41de/report.jpg,,Unsorted,0.8537,0,100,3633,36.33,43.2585,1871.2989,1.1907,,,,,,,,58,,0.58,,,,,,,,,,,43,,1,58,*PREVIEW: https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg|http...,42,1,
+closedphoto,NULL,,,,,,,,,,,,,,,,,,,,,,100,,1,,,,,,,,,,,1,,1,100,,0,0,
+location,String,true,, ,INTERSECTION of Verdun St & Gallivan Blvd  Dorchester  MA  ,,Unsorted,-0.0303,1,63,3938,39.38,9.6247,92.6356,0.2444,,,,,,,,0,,0,,,,,,,,,,,98,563 Columbus Ave  Roxbury  MA  02118|INTERSECTION of Gallivan Blvd & Washington St  Dorchester  MA  ,2,2,*PREVIEW:  |103 N Beacon St  Brighton  MA  02135|11 Aberdeen St  Boston  MA  02215|1148 Hyde Park Av...,96,1,
+fire_district,String,true,, ,9,,Unsorted,0.1515,1,2,113,1.13,0.3363,0.1131,0.2976,,,,,,,,0,,0,,,,,,,,,,,10,3,1,19, ,1,1,
+pwd_district,String,true,, ,1C,,Unsorted,0.0707,1,3,209,2.09,0.3192,0.1019,0.1527,,,,,,,,0,,0,,,,,,,,,,,14,1B,1,16, ,1,1,
+city_council_district,String,true,, ,9,,Unsorted,0.1313,1,1,100,1,0,0,0,,,,,,,,0,,0,,,,,,,,,,,10,1,1,22, ,1,1,
+police_district,String,true,, ,E5,,Unsorted,0.1717,1,3,223,2.23,0.444,0.1971,0.1991,,,,,,,,0,,0,,,,,,,,,,,13,A1,1,20, ,1,1,
+neighborhood,String,true,, ,West Roxbury,,Unsorted,0.0303,1,38,1486,14.86,9.8671,97.3604,0.664,,,,,,,,0,,0,,,,,,,,,,,19,Dorchester,1,15, |Brighton|Mission Hill,3,1,
+neighborhood_services_district,String,true,, ,9,,Unsorted,0.0303,1,2,139,1.39,0.4877,0.2379,0.3509,,,,,,,,0,,0,,,,,,,,,,,16,3,1,15, |12,2,1,
+ward,String,true,, ,Ward 9,,Unsorted,0.0505,1,7,499,4.99,2.2293,4.9699,0.4468,,,,,,,,0,,0,,,,,,,,,,,42,Ward 3,1,10,*PREVIEW:  |01|02|04|06|07|1|10|16|18,23,1,
+precinct,String,true,, ,2210,,Unsorted,0.0408,0,4,393,3.93,0.4951,0.2451,0.126,,,,,,,,1,,0.01,,,,,,,,,,,76,0306,1,5,*PREVIEW: NULL| |0102|0105|0108|0109|0201|0204|0305|0307,61,1,
+location_street_name,String,true,,103 N Beacon St,INTERSECTION Verdun St & Gallivan Blvd,,Unsorted,-0.0204,0,45,1800,18,9.3995,88.3508,0.5222,,,,,,,,1,,0.01,,,,,,,,,,,97,20 Washington St|563 Columbus Ave|INTERSECTION Gallivan Blvd & Washington St,3,2,*PREVIEW: NULL|103 N Beacon St|11 Aberdeen St|1148 Hyde Park Ave|119 L St|12 Derne St|126 Elm St|127...,94,1,
+location_zipcode,String,true,,02109,02215,,Unsorted,0.0488,0,5,415,4.15,1.8405,3.3874,0.4435,,,,,,,,17,,0.17,,,,,,,,,,,24,,1,17,02126|02134|02210|02215,4,1,
+latitude,Float,,4233.6674,42.2553,42.3806,0.1253,Unsorted,0.0505,,,,,,,,42.3367,0.0031,42.3367,42.3367,0.0305,0.0009,0.072,0,4,0,0.0163,42.2034,42.2619,42.3204,42.3432,42.3594,0.039,42.4179,42.4764,-0.1667,78,42.3594,1,20,*PREVIEW: 42.2553|42.2601|42.2609|42.2645|42.2674|42.2789|42.2797|42.2804|42.2821|42.2878,74,1,
+longitude,Float,,-7107.2688,-71.1626,-71.0298,0.1328,Unsorted,0.1515,,,,,,,,-71.0727,0.0031,,,0.0311,0.001,-0.0437,0,4,0,0.0121,-71.1741,-71.1294,-71.0848,-71.0609,-71.055,0.0298,-71.0104,-70.9658,-0.6101,77,-71.0587,1,19,*PREVIEW: -71.0298|-71.0301|-71.0309|-71.0323|-71.0325|-71.0329|-71.0336|-71.0338|-71.034|-71.0355,72,1,
+source,String,true,,Citizens Connect App,Self Service,,Unsorted,0.5354,12,20,1801,18.01,2.3473,5.5099,0.1303,,,,,,,,0,,0,,,,,,,,,,,4,Citizens Connect App,1,56,Self Service,1,3,
+qsv__rowcount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,100
+qsv__columncount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,29
+qsv__filesize_bytes,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,47702
+qsv__fingerprint_hash,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6baac425a955df5a52aa16e21a329a68bae1ebeca05226468cd32f859bd7bd7a
diff --git a/resources/test/boston311-100-everything-date-stats.csv b/resources/test/boston311-100-everything-date-stats.csv
index 13d8805b8..22d243578 100644
--- a/resources/test/boston311-100-everything-date-stats.csv
+++ b/resources/test/boston311-100-everything-date-stats.csv
@@ -1,34 +1,34 @@
-field,type,is_ascii,sum,min,max,range,sort_order,min_length,max_length,sum_length,avg_length,mean,nullcount,max_precision,sparsity,mad,lower_outer_fence,lower_inner_fence,q1,q2_median,q3,iqr,upper_inner_fence,upper_outer_fence,skewness,cardinality,mode,mode_count,mode_occurrences,antimode,antimode_count,antimode_occurrences,qsv__value
-case_enquiry_id,Integer,,10100411645180,101004113298,101004155594,42296,Unsorted,,,,,101004116451.8,0,,0,673,101004109567,101004111646,101004113725,101004114353,101004115111,1386,101004117190,101004119269,0.0938,100,,0,0,*ALL,0,1,
-open_dt,DateTime,,,2022-01-01T00:16:00+00:00,2022-01-31T11:46:00+00:00,30.47917,Unsorted,,,,,2022-01-04T07:07:45.050+00:00,0,,0,0.76261,2021-12-27T14:16:49+00:00,2021-12-30T06:00:07+00:00,2022-01-01T21:43:25+00:00,2022-01-03T07:02:14+00:00,2022-01-03T16:12:17+00:00,1.77005,2022-01-06T07:55:35+00:00,2022-01-08T23:38:53+00:00,-0.5684,100,,0,0,*ALL,0,1,
-target_dt,DateTime,,,2022-01-03T10:32:34+00:00,2022-05-20T13:03:21+00:00,137.10471,Unsorted,,,,,2022-01-17T03:14:16.404+00:00,11,,0.11,1,2021-11-26T08:30:00+00:00,2021-12-15T20:30:00+00:00,2022-01-04T08:30:00+00:00,2022-01-05T08:30:00+00:00,2022-01-17T08:30:00+00:00,13,2022-02-05T20:30:00+00:00,2022-02-25T08:30:00+00:00,0.8462,42,2022-01-04 08:30:00,1,25,*PREVIEW: 2022-01-03 10:32:34|2022-01-03 11:58:12|2022-01-04 09:58:36|2022-01-04 10:41:29|2022-01-04...,34,1,
-closed_dt,DateTime,,,2022-01-01T12:56:14+00:00,2022-04-25T14:30:31+00:00,114.06547,Unsorted,,,,,2022-01-08T01:10:44.411+00:00,15,,0.15,0.77213,2021-12-29T15:13:29+00:00,2021-12-31T19:50:08.750+00:00,2022-01-03T00:26:48.500+00:00,2022-01-03T12:15:23+00:00,2022-01-04T11:31:15+00:00,1.46142,2022-01-06T16:07:54.750+00:00,2022-01-08T20:44:34.500+00:00,0.3266,86,,1,15,*PREVIEW: 2022-01-01 12:56:14|2022-01-01 14:17:15|2022-01-01 14:59:41|2022-01-01 15:10:16|2022-01-01...,85,1,
-ontime,String,true,,ONTIME,OVERDUE,,Unsorted,6,7,617,6.17,,0,,0,,,,,,,,,,,2,ONTIME,1,83,OVERDUE,1,17,
-case_status,String,true,,Closed,Open,,Unsorted,4,6,570,5.7,,0,,0,,,,,,,,,,,2,Closed,1,85,Open,1,15,
-closure_reason,String,true,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,1,284,8314,83.14,,0,,0,,,,,,,,,,,86, ,1,15,*PREVIEW: Case Closed Case Resolved  NEW CART#21026466 DELV ON 1/11/22  |Case Closed Case Resolved  ...,85,1,
-case_title,String,true,,Abandoned Vehicles,Traffic Signal Inspection,,Unsorted,10,57,2386,23.86,,0,,0,,,,,,,,,,,42,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|BTDT: Complaint|City/State Snow Issues|DISPATCHED Short Term Rental...,24,1,
-subject,String,true,,Animal Control,Transportation - Traffic Division,,Unsorted,14,33,2570,25.7,,0,,0,,,,,,,,,,,9,Public Works Department,1,51,Animal Control|Boston Police Department|Boston Water & Sewer Commission,3,1,
-reason,String,true,,Administrative & General Requests,Street Lights,,Unsorted,7,33,1892,18.92,,0,,0,,,,,,,,,,,20,Enforcement & Abandoned Vehicles,1,23,Administrative & General Requests|Animal Issues|Building|Employee & General Comments|Noise Disturban...,7,1,
-type,String,true,,Abandoned Vehicles,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,10,47,2266,22.66,,0,,0,,,,,,,,,,,36,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|City/State Snow Issues|Electrical|General Comments For a Program or...,15,1,
-queue,String,true,,BTDT_AVRS Interface Queue,PWDx_Street Light_General Lighting Request,,Unsorted,13,55,2802,28.02,,0,,0,,,,,,,,,,,35,BTDT_Parking Enforcement,1,21,*PREVIEW: BTDT_BostonBikes|BTDT_Engineering_New Sign and Pavement Marking Requests|BTDT_Sign Shop_Si...,15,1,
-department,String,true,,BTDT,PWDx,,Unsorted,3,4,392,3.92,,0,,0,,,,,,,,,,,7,PWDx,1,49,GEN_,1,2,
-submittedphoto,String,true,,https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg,https://311.boston.gov/media/boston/report/photos/61d75bba05bbcf180c2d41de/report.jpg,,Unsorted,0,100,3633,36.33,,58,,0.58,,,,,,,,,,,43,,1,58,*PREVIEW: https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg|http...,42,1,
-closedphoto,NULL,,,,,,,,,,,,100,,1,,,,,,,,,,,1,,1,100,,0,0,
-location,String,true,, ,INTERSECTION of Verdun St & Gallivan Blvd  Dorchester  MA  ,,Unsorted,1,63,3938,39.38,,0,,0,,,,,,,,,,,98,563 Columbus Ave  Roxbury  MA  02118|INTERSECTION of Gallivan Blvd & Washington St  Dorchester  MA  ,2,2,*PREVIEW:  |103 N Beacon St  Brighton  MA  02135|11 Aberdeen St  Boston  MA  02215|1148 Hyde Park Av...,96,1,
-fire_district,String,true,, ,9,,Unsorted,1,2,113,1.13,,0,,0,,,,,,,,,,,10,3,1,19, ,1,1,
-pwd_district,String,true,, ,1C,,Unsorted,1,3,209,2.09,,0,,0,,,,,,,,,,,14,1B,1,16, ,1,1,
-city_council_district,String,true,, ,9,,Unsorted,1,1,100,1,,0,,0,,,,,,,,,,,10,1,1,22, ,1,1,
-police_district,String,true,, ,E5,,Unsorted,1,3,223,2.23,,0,,0,,,,,,,,,,,13,A1,1,20, ,1,1,
-neighborhood,String,true,, ,West Roxbury,,Unsorted,1,38,1486,14.86,,0,,0,,,,,,,,,,,19,Dorchester,1,15, |Brighton|Mission Hill,3,1,
-neighborhood_services_district,String,true,, ,9,,Unsorted,1,2,139,1.39,,0,,0,,,,,,,,,,,16,3,1,15, |12,2,1,
-ward,String,true,, ,Ward 9,,Unsorted,1,7,499,4.99,,0,,0,,,,,,,,,,,42,Ward 3,1,10,*PREVIEW:  |01|02|04|06|07|1|10|16|18,23,1,
-precinct,String,true,, ,2210,,Unsorted,0,4,393,3.93,,1,,0.01,,,,,,,,,,,76,0306,1,5,*PREVIEW: NULL| |0102|0105|0108|0109|0201|0204|0305|0307,61,1,
-location_street_name,String,true,,103 N Beacon St,INTERSECTION Verdun St & Gallivan Blvd,,Unsorted,0,45,1800,18,,1,,0.01,,,,,,,,,,,97,20 Washington St|563 Columbus Ave|INTERSECTION Gallivan Blvd & Washington St,3,2,*PREVIEW: NULL|103 N Beacon St|11 Aberdeen St|1148 Hyde Park Ave|119 L St|12 Derne St|126 Elm St|127...,94,1,
-location_zipcode,String,true,,02109,02215,,Unsorted,0,5,415,4.15,,17,,0.17,,,,,,,,,,,24,,1,17,02126|02134|02210|02215,4,1,
-latitude,Float,,4233.6674,42.2553,42.3806,0.1253,Unsorted,,,,,42.3367,0,4,0,0.0163,42.2034,42.2619,42.3204,42.3432,42.3594,0.039,42.4179,42.4764,-0.1667,78,42.3594,1,20,*PREVIEW: 42.2553|42.2601|42.2609|42.2645|42.2674|42.2789|42.2797|42.2804|42.2821|42.2878,74,1,
-longitude,Float,,-7107.2688,-71.1626,-71.0298,0.1328,Unsorted,,,,,-71.0727,0,4,0,0.0121,-71.1741,-71.1294,-71.0848,-71.0609,-71.055,0.0298,-71.0104,-70.9658,-0.6101,77,-71.0587,1,19,*PREVIEW: -71.0298|-71.0301|-71.0309|-71.0323|-71.0325|-71.0329|-71.0336|-71.0338|-71.034|-71.0355,72,1,
-source,String,true,,Citizens Connect App,Self Service,,Unsorted,12,20,1801,18.01,,0,,0,,,,,,,,,,,4,Citizens Connect App,1,56,Self Service,1,3,
-qsv__rowcount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,100
-qsv__columncount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,29
-qsv__filesize_bytes,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,47702
-qsv__fingerprint_hash,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,85611c6f33acdc78001619e131cbf15cd0e2cc41abb8865ec879a0b529ce2642
+field,type,is_ascii,sum,min,max,range,sort_order,sortiness,min_length,max_length,sum_length,avg_length,mean,nullcount,max_precision,sparsity,mad,lower_outer_fence,lower_inner_fence,q1,q2_median,q3,iqr,upper_inner_fence,upper_outer_fence,skewness,cardinality,mode,mode_count,mode_occurrences,antimode,antimode_count,antimode_occurrences,qsv__value
+case_enquiry_id,Integer,,10100411645180,101004113298,101004155594,42296,Unsorted,0.798,,,,,101004116451.8,0,,0,673,101004109567,101004111646,101004113725,101004114353,101004115111,1386,101004117190,101004119269,0.0938,100,,0,0,*ALL,0,1,
+open_dt,DateTime,,,2022-01-01T00:16:00+00:00,2022-01-31T11:46:00+00:00,30.47917,Unsorted,0.798,,,,,2022-01-04T07:07:45.050+00:00,0,,0,0.76261,2021-12-27T14:16:49+00:00,2021-12-30T06:00:07+00:00,2022-01-01T21:43:25+00:00,2022-01-03T07:02:14+00:00,2022-01-03T16:12:17+00:00,1.77005,2022-01-06T07:55:35+00:00,2022-01-08T23:38:53+00:00,-0.5684,100,,0,0,*ALL,0,1,
+target_dt,DateTime,,,2022-01-03T10:32:34+00:00,2022-05-20T13:03:21+00:00,137.10471,Unsorted,0.2273,,,,,2022-01-17T03:14:16.404+00:00,11,,0.11,1,2021-11-26T08:30:00+00:00,2021-12-15T20:30:00+00:00,2022-01-04T08:30:00+00:00,2022-01-05T08:30:00+00:00,2022-01-17T08:30:00+00:00,13,2022-02-05T20:30:00+00:00,2022-02-25T08:30:00+00:00,0.8462,42,2022-01-04 08:30:00,1,25,*PREVIEW: 2022-01-03 10:32:34|2022-01-03 11:58:12|2022-01-04 09:58:36|2022-01-04 10:41:29|2022-01-04...,34,1,
+closed_dt,DateTime,,,2022-01-01T12:56:14+00:00,2022-04-25T14:30:31+00:00,114.06547,Unsorted,-0.0714,,,,,2022-01-08T01:10:44.411+00:00,15,,0.15,0.77213,2021-12-29T15:13:29+00:00,2021-12-31T19:50:08.750+00:00,2022-01-03T00:26:48.500+00:00,2022-01-03T12:15:23+00:00,2022-01-04T11:31:15+00:00,1.46142,2022-01-06T16:07:54.750+00:00,2022-01-08T20:44:34.500+00:00,0.3266,86,,1,15,*PREVIEW: 2022-01-01 12:56:14|2022-01-01 14:17:15|2022-01-01 14:59:41|2022-01-01 15:10:16|2022-01-01...,85,1,
+ontime,String,true,,ONTIME,OVERDUE,,Unsorted,0.6768,6,7,617,6.17,,0,,0,,,,,,,,,,,2,ONTIME,1,83,OVERDUE,1,17,
+case_status,String,true,,Closed,Open,,Unsorted,0.7172,4,6,570,5.7,,0,,0,,,,,,,,,,,2,Closed,1,85,Open,1,15,
+closure_reason,String,true,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,-0.0909,1,284,8314,83.14,,0,,0,,,,,,,,,,,86, ,1,15,*PREVIEW: Case Closed Case Resolved  NEW CART#21026466 DELV ON 1/11/22  |Case Closed Case Resolved  ...,85,1,
+case_title,String,true,,Abandoned Vehicles,Traffic Signal Inspection,,Unsorted,0.0101,10,57,2386,23.86,,0,,0,,,,,,,,,,,42,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|BTDT: Complaint|City/State Snow Issues|DISPATCHED Short Term Rental...,24,1,
+subject,String,true,,Animal Control,Transportation - Traffic Division,,Unsorted,0.3737,14,33,2570,25.7,,0,,0,,,,,,,,,,,9,Public Works Department,1,51,Animal Control|Boston Police Department|Boston Water & Sewer Commission,3,1,
+reason,String,true,,Administrative & General Requests,Street Lights,,Unsorted,0.1717,7,33,1892,18.92,,0,,0,,,,,,,,,,,20,Enforcement & Abandoned Vehicles,1,23,Administrative & General Requests|Animal Issues|Building|Employee & General Comments|Noise Disturban...,7,1,
+type,String,true,,Abandoned Vehicles,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,0.0707,10,47,2266,22.66,,0,,0,,,,,,,,,,,36,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|City/State Snow Issues|Electrical|General Comments For a Program or...,15,1,
+queue,String,true,,BTDT_AVRS Interface Queue,PWDx_Street Light_General Lighting Request,,Unsorted,0.2121,13,55,2802,28.02,,0,,0,,,,,,,,,,,35,BTDT_Parking Enforcement,1,21,*PREVIEW: BTDT_BostonBikes|BTDT_Engineering_New Sign and Pavement Marking Requests|BTDT_Sign Shop_Si...,15,1,
+department,String,true,,BTDT,PWDx,,Unsorted,0.3737,3,4,392,3.92,,0,,0,,,,,,,,,,,7,PWDx,1,49,GEN_,1,2,
+submittedphoto,String,true,,https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg,https://311.boston.gov/media/boston/report/photos/61d75bba05bbcf180c2d41de/report.jpg,,Unsorted,0.8537,0,100,3633,36.33,,58,,0.58,,,,,,,,,,,43,,1,58,*PREVIEW: https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg|http...,42,1,
+closedphoto,NULL,,,,,,,,,,,,,100,,1,,,,,,,,,,,1,,1,100,,0,0,
+location,String,true,, ,INTERSECTION of Verdun St & Gallivan Blvd  Dorchester  MA  ,,Unsorted,-0.0303,1,63,3938,39.38,,0,,0,,,,,,,,,,,98,563 Columbus Ave  Roxbury  MA  02118|INTERSECTION of Gallivan Blvd & Washington St  Dorchester  MA  ,2,2,*PREVIEW:  |103 N Beacon St  Brighton  MA  02135|11 Aberdeen St  Boston  MA  02215|1148 Hyde Park Av...,96,1,
+fire_district,String,true,, ,9,,Unsorted,0.1515,1,2,113,1.13,,0,,0,,,,,,,,,,,10,3,1,19, ,1,1,
+pwd_district,String,true,, ,1C,,Unsorted,0.0707,1,3,209,2.09,,0,,0,,,,,,,,,,,14,1B,1,16, ,1,1,
+city_council_district,String,true,, ,9,,Unsorted,0.1313,1,1,100,1,,0,,0,,,,,,,,,,,10,1,1,22, ,1,1,
+police_district,String,true,, ,E5,,Unsorted,0.1717,1,3,223,2.23,,0,,0,,,,,,,,,,,13,A1,1,20, ,1,1,
+neighborhood,String,true,, ,West Roxbury,,Unsorted,0.0303,1,38,1486,14.86,,0,,0,,,,,,,,,,,19,Dorchester,1,15, |Brighton|Mission Hill,3,1,
+neighborhood_services_district,String,true,, ,9,,Unsorted,0.0303,1,2,139,1.39,,0,,0,,,,,,,,,,,16,3,1,15, |12,2,1,
+ward,String,true,, ,Ward 9,,Unsorted,0.0505,1,7,499,4.99,,0,,0,,,,,,,,,,,42,Ward 3,1,10,*PREVIEW:  |01|02|04|06|07|1|10|16|18,23,1,
+precinct,String,true,, ,2210,,Unsorted,0.0408,0,4,393,3.93,,1,,0.01,,,,,,,,,,,76,0306,1,5,*PREVIEW: NULL| |0102|0105|0108|0109|0201|0204|0305|0307,61,1,
+location_street_name,String,true,,103 N Beacon St,INTERSECTION Verdun St & Gallivan Blvd,,Unsorted,-0.0204,0,45,1800,18,,1,,0.01,,,,,,,,,,,97,20 Washington St|563 Columbus Ave|INTERSECTION Gallivan Blvd & Washington St,3,2,*PREVIEW: NULL|103 N Beacon St|11 Aberdeen St|1148 Hyde Park Ave|119 L St|12 Derne St|126 Elm St|127...,94,1,
+location_zipcode,String,true,,02109,02215,,Unsorted,0.0488,0,5,415,4.15,,17,,0.17,,,,,,,,,,,24,,1,17,02126|02134|02210|02215,4,1,
+latitude,Float,,4233.6674,42.2553,42.3806,0.1253,Unsorted,0.0505,,,,,42.3367,0,4,0,0.0163,42.2034,42.2619,42.3204,42.3432,42.3594,0.039,42.4179,42.4764,-0.1667,78,42.3594,1,20,*PREVIEW: 42.2553|42.2601|42.2609|42.2645|42.2674|42.2789|42.2797|42.2804|42.2821|42.2878,74,1,
+longitude,Float,,-7107.2688,-71.1626,-71.0298,0.1328,Unsorted,0.1515,,,,,-71.0727,0,4,0,0.0121,-71.1741,-71.1294,-71.0848,-71.0609,-71.055,0.0298,-71.0104,-70.9658,-0.6101,77,-71.0587,1,19,*PREVIEW: -71.0298|-71.0301|-71.0309|-71.0323|-71.0325|-71.0329|-71.0336|-71.0338|-71.034|-71.0355,72,1,
+source,String,true,,Citizens Connect App,Self Service,,Unsorted,0.5354,12,20,1801,18.01,,0,,0,,,,,,,,,,,4,Citizens Connect App,1,56,Self Service,1,3,
+qsv__rowcount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,100
+qsv__columncount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,29
+qsv__filesize_bytes,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,47702
+qsv__fingerprint_hash,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6baac425a955df5a52aa16e21a329a68bae1ebeca05226468cd32f859bd7bd7a
diff --git a/resources/test/boston311-100-everything-datenotime-stats.csv b/resources/test/boston311-100-everything-datenotime-stats.csv
index 159409b04..8ef3c3283 100644
--- a/resources/test/boston311-100-everything-datenotime-stats.csv
+++ b/resources/test/boston311-100-everything-datenotime-stats.csv
@@ -1,34 +1,34 @@
-field,type,is_ascii,sum,min,max,range,sort_order,min_length,max_length,sum_length,avg_length,mean,nullcount,max_precision,sparsity,mad,lower_outer_fence,lower_inner_fence,q1,q2_median,q3,iqr,upper_inner_fence,upper_outer_fence,skewness,cardinality,mode,mode_count,mode_occurrences,antimode,antimode_count,antimode_occurrences,qsv__value
-case_enquiry_id,Integer,,10100411645180,101004113298,101004155594,42296,Unsorted,,,,,101004116451.8,0,,0,673,101004109567,101004111646,101004113725,101004114353,101004115111,1386,101004117190,101004119269,0.0938,100,,0,0,*ALL,0,1,
-open_dt,Date,,,2022-01-01,2022-01-31,30,Unsorted,,,,,2022-01-03,0,,0,1,2021-12-26,2021-12-29,2022-01-01,2022-01-03,2022-01-03,2,2022-01-06,2022-01-09,-1,10,2022-01-03,1,38,2022-01-06|2022-01-08|2022-01-19|2022-01-21,4,1,
-target_dt,Date,,,2022-01-03,2022-05-20,137,Unsorted,,,,,2022-01-16,11,,0.11,1,2021-11-26,2021-12-15,2022-01-04,2022-01-05,2022-01-17,13,2022-02-05,2022-02-25,0.8462,22,2022-01-04,1,37,2022-01-11|2022-01-19|2022-02-04|2022-02-14|2022-02-17|2022-03-02|2022-03-10|2022-04-01|2022-05-20,9,1,
-closed_dt,Date,,,2022-01-01,2022-04-25,114,Unsorted,,,,,2022-01-07,15,,0.15,1,2021-12-31,2022-01-01,2022-01-03,2022-01-03,2022-01-04,1,2022-01-05,2022-01-07,1,18,2022-01-03,1,35,2022-01-07|2022-01-09|2022-01-13|2022-01-19|2022-02-12|2022-02-28|2022-03-09|2022-04-25,8,1,
-ontime,String,true,,ONTIME,OVERDUE,,Unsorted,6,7,617,6.17,,0,,0,,,,,,,,,,,2,ONTIME,1,83,OVERDUE,1,17,
-case_status,String,true,,Closed,Open,,Unsorted,4,6,570,5.7,,0,,0,,,,,,,,,,,2,Closed,1,85,Open,1,15,
-closure_reason,String,true,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,1,284,8314,83.14,,0,,0,,,,,,,,,,,86, ,1,15,*PREVIEW: Case Closed Case Resolved  NEW CART#21026466 DELV ON 1/11/22  |Case Closed Case Resolved  ...,85,1,
-case_title,String,true,,Abandoned Vehicles,Traffic Signal Inspection,,Unsorted,10,57,2386,23.86,,0,,0,,,,,,,,,,,42,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|BTDT: Complaint|City/State Snow Issues|DISPATCHED Short Term Rental...,24,1,
-subject,String,true,,Animal Control,Transportation - Traffic Division,,Unsorted,14,33,2570,25.7,,0,,0,,,,,,,,,,,9,Public Works Department,1,51,Animal Control|Boston Police Department|Boston Water & Sewer Commission,3,1,
-reason,String,true,,Administrative & General Requests,Street Lights,,Unsorted,7,33,1892,18.92,,0,,0,,,,,,,,,,,20,Enforcement & Abandoned Vehicles,1,23,Administrative & General Requests|Animal Issues|Building|Employee & General Comments|Noise Disturban...,7,1,
-type,String,true,,Abandoned Vehicles,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,10,47,2266,22.66,,0,,0,,,,,,,,,,,36,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|City/State Snow Issues|Electrical|General Comments For a Program or...,15,1,
-queue,String,true,,BTDT_AVRS Interface Queue,PWDx_Street Light_General Lighting Request,,Unsorted,13,55,2802,28.02,,0,,0,,,,,,,,,,,35,BTDT_Parking Enforcement,1,21,*PREVIEW: BTDT_BostonBikes|BTDT_Engineering_New Sign and Pavement Marking Requests|BTDT_Sign Shop_Si...,15,1,
-department,String,true,,BTDT,PWDx,,Unsorted,3,4,392,3.92,,0,,0,,,,,,,,,,,7,PWDx,1,49,GEN_,1,2,
-submittedphoto,String,true,,https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg,https://311.boston.gov/media/boston/report/photos/61d75bba05bbcf180c2d41de/report.jpg,,Unsorted,0,100,3633,36.33,,58,,0.58,,,,,,,,,,,43,,1,58,*PREVIEW: https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg|http...,42,1,
-closedphoto,NULL,,,,,,,,,,,,100,,1,,,,,,,,,,,1,,1,100,,0,0,
-location,String,true,, ,INTERSECTION of Verdun St & Gallivan Blvd  Dorchester  MA  ,,Unsorted,1,63,3938,39.38,,0,,0,,,,,,,,,,,98,563 Columbus Ave  Roxbury  MA  02118|INTERSECTION of Gallivan Blvd & Washington St  Dorchester  MA  ,2,2,*PREVIEW:  |103 N Beacon St  Brighton  MA  02135|11 Aberdeen St  Boston  MA  02215|1148 Hyde Park Av...,96,1,
-fire_district,String,true,, ,9,,Unsorted,1,2,113,1.13,,0,,0,,,,,,,,,,,10,3,1,19, ,1,1,
-pwd_district,String,true,, ,1C,,Unsorted,1,3,209,2.09,,0,,0,,,,,,,,,,,14,1B,1,16, ,1,1,
-city_council_district,String,true,, ,9,,Unsorted,1,1,100,1,,0,,0,,,,,,,,,,,10,1,1,22, ,1,1,
-police_district,String,true,, ,E5,,Unsorted,1,3,223,2.23,,0,,0,,,,,,,,,,,13,A1,1,20, ,1,1,
-neighborhood,String,true,, ,West Roxbury,,Unsorted,1,38,1486,14.86,,0,,0,,,,,,,,,,,19,Dorchester,1,15, |Brighton|Mission Hill,3,1,
-neighborhood_services_district,String,true,, ,9,,Unsorted,1,2,139,1.39,,0,,0,,,,,,,,,,,16,3,1,15, |12,2,1,
-ward,String,true,, ,Ward 9,,Unsorted,1,7,499,4.99,,0,,0,,,,,,,,,,,42,Ward 3,1,10,*PREVIEW:  |01|02|04|06|07|1|10|16|18,23,1,
-precinct,String,true,, ,2210,,Unsorted,0,4,393,3.93,,1,,0.01,,,,,,,,,,,76,0306,1,5,*PREVIEW: NULL| |0102|0105|0108|0109|0201|0204|0305|0307,61,1,
-location_street_name,String,true,,103 N Beacon St,INTERSECTION Verdun St & Gallivan Blvd,,Unsorted,0,45,1800,18,,1,,0.01,,,,,,,,,,,97,20 Washington St|563 Columbus Ave|INTERSECTION Gallivan Blvd & Washington St,3,2,*PREVIEW: NULL|103 N Beacon St|11 Aberdeen St|1148 Hyde Park Ave|119 L St|12 Derne St|126 Elm St|127...,94,1,
-location_zipcode,String,true,,02109,02215,,Unsorted,0,5,415,4.15,,17,,0.17,,,,,,,,,,,24,,1,17,02126|02134|02210|02215,4,1,
-latitude,Float,,4233.6674,42.2553,42.3806,0.1253,Unsorted,,,,,42.3367,0,4,0,0.0163,42.2034,42.2619,42.3204,42.3432,42.3594,0.039,42.4179,42.4764,-0.1667,78,42.3594,1,20,*PREVIEW: 42.2553|42.2601|42.2609|42.2645|42.2674|42.2789|42.2797|42.2804|42.2821|42.2878,74,1,
-longitude,Float,,-7107.2688,-71.1626,-71.0298,0.1328,Unsorted,,,,,-71.0727,0,4,0,0.0121,-71.1741,-71.1294,-71.0848,-71.0609,-71.055,0.0298,-71.0104,-70.9658,-0.6101,77,-71.0587,1,19,*PREVIEW: -71.0298|-71.0301|-71.0309|-71.0323|-71.0325|-71.0329|-71.0336|-71.0338|-71.034|-71.0355,72,1,
-source,String,true,,Citizens Connect App,Self Service,,Unsorted,12,20,1801,18.01,,0,,0,,,,,,,,,,,4,Citizens Connect App,1,56,Self Service,1,3,
-qsv__rowcount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,100
-qsv__columncount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,29
-qsv__filesize_bytes,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,45236
-qsv__fingerprint_hash,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,650cef751485638851b828e32c6970fb63f299f954d58863e93daa693fb687f3
+field,type,is_ascii,sum,min,max,range,sort_order,sortiness,min_length,max_length,sum_length,avg_length,mean,nullcount,max_precision,sparsity,mad,lower_outer_fence,lower_inner_fence,q1,q2_median,q3,iqr,upper_inner_fence,upper_outer_fence,skewness,cardinality,mode,mode_count,mode_occurrences,antimode,antimode_count,antimode_occurrences,qsv__value
+case_enquiry_id,Integer,,10100411645180,101004113298,101004155594,42296,Unsorted,0.798,,,,,101004116451.8,0,,0,673,101004109567,101004111646,101004113725,101004114353,101004115111,1386,101004117190,101004119269,0.0938,100,,0,0,*ALL,0,1,
+open_dt,Date,,,2022-01-01,2022-01-31,30,Unsorted,0.8586,,,,,2022-01-03,0,,0,1,2021-12-26,2021-12-29,2022-01-01,2022-01-03,2022-01-03,2,2022-01-06,2022-01-09,-1,10,2022-01-03,1,38,2022-01-06|2022-01-08|2022-01-19|2022-01-21,4,1,
+target_dt,Date,,,2022-01-03,2022-05-20,137,Unsorted,0.2273,,,,,2022-01-16,11,,0.11,1,2021-11-26,2021-12-15,2022-01-04,2022-01-05,2022-01-17,13,2022-02-05,2022-02-25,0.8462,22,2022-01-04,1,37,2022-01-11|2022-01-19|2022-02-04|2022-02-14|2022-02-17|2022-03-02|2022-03-10|2022-04-01|2022-05-20,9,1,
+closed_dt,Date,,,2022-01-01,2022-04-25,114,Unsorted,0.3571,,,,,2022-01-07,15,,0.15,1,2021-12-31,2022-01-01,2022-01-03,2022-01-03,2022-01-04,1,2022-01-05,2022-01-07,1,18,2022-01-03,1,35,2022-01-07|2022-01-09|2022-01-13|2022-01-19|2022-02-12|2022-02-28|2022-03-09|2022-04-25,8,1,
+ontime,String,true,,ONTIME,OVERDUE,,Unsorted,0.6768,6,7,617,6.17,,0,,0,,,,,,,,,,,2,ONTIME,1,83,OVERDUE,1,17,
+case_status,String,true,,Closed,Open,,Unsorted,0.7172,4,6,570,5.7,,0,,0,,,,,,,,,,,2,Closed,1,85,Open,1,15,
+closure_reason,String,true,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,-0.0909,1,284,8314,83.14,,0,,0,,,,,,,,,,,86, ,1,15,*PREVIEW: Case Closed Case Resolved  NEW CART#21026466 DELV ON 1/11/22  |Case Closed Case Resolved  ...,85,1,
+case_title,String,true,,Abandoned Vehicles,Traffic Signal Inspection,,Unsorted,0.0101,10,57,2386,23.86,,0,,0,,,,,,,,,,,42,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|BTDT: Complaint|City/State Snow Issues|DISPATCHED Short Term Rental...,24,1,
+subject,String,true,,Animal Control,Transportation - Traffic Division,,Unsorted,0.3737,14,33,2570,25.7,,0,,0,,,,,,,,,,,9,Public Works Department,1,51,Animal Control|Boston Police Department|Boston Water & Sewer Commission,3,1,
+reason,String,true,,Administrative & General Requests,Street Lights,,Unsorted,0.1717,7,33,1892,18.92,,0,,0,,,,,,,,,,,20,Enforcement & Abandoned Vehicles,1,23,Administrative & General Requests|Animal Issues|Building|Employee & General Comments|Noise Disturban...,7,1,
+type,String,true,,Abandoned Vehicles,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,0.0707,10,47,2266,22.66,,0,,0,,,,,,,,,,,36,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|City/State Snow Issues|Electrical|General Comments For a Program or...,15,1,
+queue,String,true,,BTDT_AVRS Interface Queue,PWDx_Street Light_General Lighting Request,,Unsorted,0.2121,13,55,2802,28.02,,0,,0,,,,,,,,,,,35,BTDT_Parking Enforcement,1,21,*PREVIEW: BTDT_BostonBikes|BTDT_Engineering_New Sign and Pavement Marking Requests|BTDT_Sign Shop_Si...,15,1,
+department,String,true,,BTDT,PWDx,,Unsorted,0.3737,3,4,392,3.92,,0,,0,,,,,,,,,,,7,PWDx,1,49,GEN_,1,2,
+submittedphoto,String,true,,https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg,https://311.boston.gov/media/boston/report/photos/61d75bba05bbcf180c2d41de/report.jpg,,Unsorted,0.8537,0,100,3633,36.33,,58,,0.58,,,,,,,,,,,43,,1,58,*PREVIEW: https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg|http...,42,1,
+closedphoto,NULL,,,,,,,,,,,,,100,,1,,,,,,,,,,,1,,1,100,,0,0,
+location,String,true,, ,INTERSECTION of Verdun St & Gallivan Blvd  Dorchester  MA  ,,Unsorted,-0.0303,1,63,3938,39.38,,0,,0,,,,,,,,,,,98,563 Columbus Ave  Roxbury  MA  02118|INTERSECTION of Gallivan Blvd & Washington St  Dorchester  MA  ,2,2,*PREVIEW:  |103 N Beacon St  Brighton  MA  02135|11 Aberdeen St  Boston  MA  02215|1148 Hyde Park Av...,96,1,
+fire_district,String,true,, ,9,,Unsorted,0.1515,1,2,113,1.13,,0,,0,,,,,,,,,,,10,3,1,19, ,1,1,
+pwd_district,String,true,, ,1C,,Unsorted,0.0707,1,3,209,2.09,,0,,0,,,,,,,,,,,14,1B,1,16, ,1,1,
+city_council_district,String,true,, ,9,,Unsorted,0.1313,1,1,100,1,,0,,0,,,,,,,,,,,10,1,1,22, ,1,1,
+police_district,String,true,, ,E5,,Unsorted,0.1717,1,3,223,2.23,,0,,0,,,,,,,,,,,13,A1,1,20, ,1,1,
+neighborhood,String,true,, ,West Roxbury,,Unsorted,0.0303,1,38,1486,14.86,,0,,0,,,,,,,,,,,19,Dorchester,1,15, |Brighton|Mission Hill,3,1,
+neighborhood_services_district,String,true,, ,9,,Unsorted,0.0303,1,2,139,1.39,,0,,0,,,,,,,,,,,16,3,1,15, |12,2,1,
+ward,String,true,, ,Ward 9,,Unsorted,0.0505,1,7,499,4.99,,0,,0,,,,,,,,,,,42,Ward 3,1,10,*PREVIEW:  |01|02|04|06|07|1|10|16|18,23,1,
+precinct,String,true,, ,2210,,Unsorted,0.0408,0,4,393,3.93,,1,,0.01,,,,,,,,,,,76,0306,1,5,*PREVIEW: NULL| |0102|0105|0108|0109|0201|0204|0305|0307,61,1,
+location_street_name,String,true,,103 N Beacon St,INTERSECTION Verdun St & Gallivan Blvd,,Unsorted,-0.0204,0,45,1800,18,,1,,0.01,,,,,,,,,,,97,20 Washington St|563 Columbus Ave|INTERSECTION Gallivan Blvd & Washington St,3,2,*PREVIEW: NULL|103 N Beacon St|11 Aberdeen St|1148 Hyde Park Ave|119 L St|12 Derne St|126 Elm St|127...,94,1,
+location_zipcode,String,true,,02109,02215,,Unsorted,0.0488,0,5,415,4.15,,17,,0.17,,,,,,,,,,,24,,1,17,02126|02134|02210|02215,4,1,
+latitude,Float,,4233.6674,42.2553,42.3806,0.1253,Unsorted,0.0505,,,,,42.3367,0,4,0,0.0163,42.2034,42.2619,42.3204,42.3432,42.3594,0.039,42.4179,42.4764,-0.1667,78,42.3594,1,20,*PREVIEW: 42.2553|42.2601|42.2609|42.2645|42.2674|42.2789|42.2797|42.2804|42.2821|42.2878,74,1,
+longitude,Float,,-7107.2688,-71.1626,-71.0298,0.1328,Unsorted,0.1515,,,,,-71.0727,0,4,0,0.0121,-71.1741,-71.1294,-71.0848,-71.0609,-71.055,0.0298,-71.0104,-70.9658,-0.6101,77,-71.0587,1,19,*PREVIEW: -71.0298|-71.0301|-71.0309|-71.0323|-71.0325|-71.0329|-71.0336|-71.0338|-71.034|-71.0355,72,1,
+source,String,true,,Citizens Connect App,Self Service,,Unsorted,0.5354,12,20,1801,18.01,,0,,0,,,,,,,,,,,4,Citizens Connect App,1,56,Self Service,1,3,
+qsv__rowcount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,100
+qsv__columncount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,29
+qsv__filesize_bytes,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,45236
+qsv__fingerprint_hash,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,da83e4d320741c82e33f5025ee4f096ff477d02b16fcf40163f7c17269cf933d
diff --git a/resources/test/boston311-100-everything-inferdates-defaultwhitelist-stats.csv b/resources/test/boston311-100-everything-inferdates-defaultwhitelist-stats.csv
index c087e6e2d..0a726b5f2 100644
--- a/resources/test/boston311-100-everything-inferdates-defaultwhitelist-stats.csv
+++ b/resources/test/boston311-100-everything-inferdates-defaultwhitelist-stats.csv
@@ -1,34 +1,34 @@
-field,type,is_ascii,sum,min,max,range,sort_order,min_length,max_length,sum_length,avg_length,mean,nullcount,max_precision,sparsity,mad,lower_outer_fence,lower_inner_fence,q1,q2_median,q3,iqr,upper_inner_fence,upper_outer_fence,skewness,cardinality,mode,mode_count,mode_occurrences,antimode,antimode_count,antimode_occurrences,qsv__value
-case_enquiry_id,Integer,,10100411645180,101004113298,101004155594,42296,Unsorted,,,,,101004116451.8,0,,0,673,101004109567,101004111646,101004113725,101004114353,101004115111,1386,101004117190,101004119269,0.0938,100,,0,0,*ALL,0,1,
-open_dt,DateTime,,,2022-01-01T00:16:00+00:00,2022-01-31T11:46:00+00:00,30.47917,Unsorted,,,,,2022-01-04T07:07:45.050+00:00,0,,0,0.76261,2021-12-27T14:16:49+00:00,2021-12-30T06:00:07+00:00,2022-01-01T21:43:25+00:00,2022-01-03T07:02:14+00:00,2022-01-03T16:12:17+00:00,1.77005,2022-01-06T07:55:35+00:00,2022-01-08T23:38:53+00:00,-0.5684,100,,0,0,*ALL,0,1,
-target_dt,String,true,,2022-01-03 10:32:34,2022-05-20 13:03:21,,Unsorted,0,19,1691,16.91,,11,,0.11,,,,,,,,,,,42,2022-01-04 08:30:00,1,25,*PREVIEW: 2022-01-03 10:32:34|2022-01-03 11:58:12|2022-01-04 09:58:36|2022-01-04 10:41:29|2022-01-04...,34,1,
-closed_dt,DateTime,,,2022-01-01T12:56:14+00:00,2022-04-25T14:30:31+00:00,114.06547,Unsorted,,,,,2022-01-08T01:10:44.411+00:00,15,,0.15,0.77213,2021-12-29T15:13:29+00:00,2021-12-31T19:50:08.750+00:00,2022-01-03T00:26:48.500+00:00,2022-01-03T12:15:23+00:00,2022-01-04T11:31:15+00:00,1.46142,2022-01-06T16:07:54.750+00:00,2022-01-08T20:44:34.500+00:00,0.3266,86,,1,15,*PREVIEW: 2022-01-01 12:56:14|2022-01-01 14:17:15|2022-01-01 14:59:41|2022-01-01 15:10:16|2022-01-01...,85,1,
-ontime,String,true,,ONTIME,OVERDUE,,Unsorted,6,7,617,6.17,,0,,0,,,,,,,,,,,2,ONTIME,1,83,OVERDUE,1,17,
-case_status,String,true,,Closed,Open,,Unsorted,4,6,570,5.7,,0,,0,,,,,,,,,,,2,Closed,1,85,Open,1,15,
-closure_reason,String,true,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,1,284,8314,83.14,,0,,0,,,,,,,,,,,86, ,1,15,*PREVIEW: Case Closed Case Resolved  NEW CART#21026466 DELV ON 1/11/22  |Case Closed Case Resolved  ...,85,1,
-case_title,String,true,,Abandoned Vehicles,Traffic Signal Inspection,,Unsorted,10,57,2386,23.86,,0,,0,,,,,,,,,,,42,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|BTDT: Complaint|City/State Snow Issues|DISPATCHED Short Term Rental...,24,1,
-subject,String,true,,Animal Control,Transportation - Traffic Division,,Unsorted,14,33,2570,25.7,,0,,0,,,,,,,,,,,9,Public Works Department,1,51,Animal Control|Boston Police Department|Boston Water & Sewer Commission,3,1,
-reason,String,true,,Administrative & General Requests,Street Lights,,Unsorted,7,33,1892,18.92,,0,,0,,,,,,,,,,,20,Enforcement & Abandoned Vehicles,1,23,Administrative & General Requests|Animal Issues|Building|Employee & General Comments|Noise Disturban...,7,1,
-type,String,true,,Abandoned Vehicles,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,10,47,2266,22.66,,0,,0,,,,,,,,,,,36,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|City/State Snow Issues|Electrical|General Comments For a Program or...,15,1,
-queue,String,true,,BTDT_AVRS Interface Queue,PWDx_Street Light_General Lighting Request,,Unsorted,13,55,2802,28.02,,0,,0,,,,,,,,,,,35,BTDT_Parking Enforcement,1,21,*PREVIEW: BTDT_BostonBikes|BTDT_Engineering_New Sign and Pavement Marking Requests|BTDT_Sign Shop_Si...,15,1,
-department,String,true,,BTDT,PWDx,,Unsorted,3,4,392,3.92,,0,,0,,,,,,,,,,,7,PWDx,1,49,GEN_,1,2,
-submittedphoto,String,true,,https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg,https://311.boston.gov/media/boston/report/photos/61d75bba05bbcf180c2d41de/report.jpg,,Unsorted,0,100,3633,36.33,,58,,0.58,,,,,,,,,,,43,,1,58,*PREVIEW: https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg|http...,42,1,
-closedphoto,NULL,,,,,,,,,,,,100,,1,,,,,,,,,,,1,,1,100,,0,0,
-location,String,true,, ,INTERSECTION of Verdun St & Gallivan Blvd  Dorchester  MA  ,,Unsorted,1,63,3938,39.38,,0,,0,,,,,,,,,,,98,563 Columbus Ave  Roxbury  MA  02118|INTERSECTION of Gallivan Blvd & Washington St  Dorchester  MA  ,2,2,*PREVIEW:  |103 N Beacon St  Brighton  MA  02135|11 Aberdeen St  Boston  MA  02215|1148 Hyde Park Av...,96,1,
-fire_district,String,true,, ,9,,Unsorted,1,2,113,1.13,,0,,0,,,,,,,,,,,10,3,1,19, ,1,1,
-pwd_district,String,true,, ,1C,,Unsorted,1,3,209,2.09,,0,,0,,,,,,,,,,,14,1B,1,16, ,1,1,
-city_council_district,String,true,, ,9,,Unsorted,1,1,100,1,,0,,0,,,,,,,,,,,10,1,1,22, ,1,1,
-police_district,String,true,, ,E5,,Unsorted,1,3,223,2.23,,0,,0,,,,,,,,,,,13,A1,1,20, ,1,1,
-neighborhood,String,true,, ,West Roxbury,,Unsorted,1,38,1486,14.86,,0,,0,,,,,,,,,,,19,Dorchester,1,15, |Brighton|Mission Hill,3,1,
-neighborhood_services_district,String,true,, ,9,,Unsorted,1,2,139,1.39,,0,,0,,,,,,,,,,,16,3,1,15, |12,2,1,
-ward,String,true,, ,Ward 9,,Unsorted,1,7,499,4.99,,0,,0,,,,,,,,,,,42,Ward 3,1,10,*PREVIEW:  |01|02|04|06|07|1|10|16|18,23,1,
-precinct,String,true,, ,2210,,Unsorted,0,4,393,3.93,,1,,0.01,,,,,,,,,,,76,0306,1,5,*PREVIEW: NULL| |0102|0105|0108|0109|0201|0204|0305|0307,61,1,
-location_street_name,String,true,,103 N Beacon St,INTERSECTION Verdun St & Gallivan Blvd,,Unsorted,0,45,1800,18,,1,,0.01,,,,,,,,,,,97,20 Washington St|563 Columbus Ave|INTERSECTION Gallivan Blvd & Washington St,3,2,*PREVIEW: NULL|103 N Beacon St|11 Aberdeen St|1148 Hyde Park Ave|119 L St|12 Derne St|126 Elm St|127...,94,1,
-location_zipcode,String,true,,02109,02215,,Unsorted,0,5,415,4.15,,17,,0.17,,,,,,,,,,,24,,1,17,02126|02134|02210|02215,4,1,
-latitude,Float,,4233.6674,42.2553,42.3806,0.1253,Unsorted,,,,,42.3367,0,4,0,0.0163,42.2034,42.2619,42.3204,42.3432,42.3594,0.039,42.4179,42.4764,-0.1667,78,42.3594,1,20,*PREVIEW: 42.2553|42.2601|42.2609|42.2645|42.2674|42.2789|42.2797|42.2804|42.2821|42.2878,74,1,
-longitude,Float,,-7107.2688,-71.1626,-71.0298,0.1328,Unsorted,,,,,-71.0727,0,4,0,0.0121,-71.1741,-71.1294,-71.0848,-71.0609,-71.055,0.0298,-71.0104,-70.9658,-0.6101,77,-71.0587,1,19,*PREVIEW: -71.0298|-71.0301|-71.0309|-71.0323|-71.0325|-71.0329|-71.0336|-71.0338|-71.034|-71.0355,72,1,
-source,String,true,,Citizens Connect App,Self Service,,Unsorted,12,20,1801,18.01,,0,,0,,,,,,,,,,,4,Citizens Connect App,1,56,Self Service,1,3,
-qsv__rowcount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,100
-qsv__columncount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,29
-qsv__filesize_bytes,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,47702
-qsv__fingerprint_hash,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0cd656e6c2b1bb94ce1ca62426cb2ec8318aa52205d9972de031d7190fc990ce
+field,type,is_ascii,sum,min,max,range,sort_order,sortiness,min_length,max_length,sum_length,avg_length,mean,nullcount,max_precision,sparsity,mad,lower_outer_fence,lower_inner_fence,q1,q2_median,q3,iqr,upper_inner_fence,upper_outer_fence,skewness,cardinality,mode,mode_count,mode_occurrences,antimode,antimode_count,antimode_occurrences,qsv__value
+case_enquiry_id,Integer,,10100411645180,101004113298,101004155594,42296,Unsorted,0.798,,,,,101004116451.8,0,,0,673,101004109567,101004111646,101004113725,101004114353,101004115111,1386,101004117190,101004119269,0.0938,100,,0,0,*ALL,0,1,
+open_dt,DateTime,,,2022-01-01T00:16:00+00:00,2022-01-31T11:46:00+00:00,30.47917,Unsorted,0.798,,,,,2022-01-04T07:07:45.050+00:00,0,,0,0.76261,2021-12-27T14:16:49+00:00,2021-12-30T06:00:07+00:00,2022-01-01T21:43:25+00:00,2022-01-03T07:02:14+00:00,2022-01-03T16:12:17+00:00,1.77005,2022-01-06T07:55:35+00:00,2022-01-08T23:38:53+00:00,-0.5684,100,,0,0,*ALL,0,1,
+target_dt,String,true,,2022-01-03 10:32:34,2022-05-20 13:03:21,,Unsorted,0.2273,0,19,1691,16.91,,11,,0.11,,,,,,,,,,,42,2022-01-04 08:30:00,1,25,*PREVIEW: 2022-01-03 10:32:34|2022-01-03 11:58:12|2022-01-04 09:58:36|2022-01-04 10:41:29|2022-01-04...,34,1,
+closed_dt,DateTime,,,2022-01-01T12:56:14+00:00,2022-04-25T14:30:31+00:00,114.06547,Unsorted,-0.0714,,,,,2022-01-08T01:10:44.411+00:00,15,,0.15,0.77213,2021-12-29T15:13:29+00:00,2021-12-31T19:50:08.750+00:00,2022-01-03T00:26:48.500+00:00,2022-01-03T12:15:23+00:00,2022-01-04T11:31:15+00:00,1.46142,2022-01-06T16:07:54.750+00:00,2022-01-08T20:44:34.500+00:00,0.3266,86,,1,15,*PREVIEW: 2022-01-01 12:56:14|2022-01-01 14:17:15|2022-01-01 14:59:41|2022-01-01 15:10:16|2022-01-01...,85,1,
+ontime,String,true,,ONTIME,OVERDUE,,Unsorted,0.6768,6,7,617,6.17,,0,,0,,,,,,,,,,,2,ONTIME,1,83,OVERDUE,1,17,
+case_status,String,true,,Closed,Open,,Unsorted,0.7172,4,6,570,5.7,,0,,0,,,,,,,,,,,2,Closed,1,85,Open,1,15,
+closure_reason,String,true,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,-0.0909,1,284,8314,83.14,,0,,0,,,,,,,,,,,86, ,1,15,*PREVIEW: Case Closed Case Resolved  NEW CART#21026466 DELV ON 1/11/22  |Case Closed Case Resolved  ...,85,1,
+case_title,String,true,,Abandoned Vehicles,Traffic Signal Inspection,,Unsorted,0.0101,10,57,2386,23.86,,0,,0,,,,,,,,,,,42,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|BTDT: Complaint|City/State Snow Issues|DISPATCHED Short Term Rental...,24,1,
+subject,String,true,,Animal Control,Transportation - Traffic Division,,Unsorted,0.3737,14,33,2570,25.7,,0,,0,,,,,,,,,,,9,Public Works Department,1,51,Animal Control|Boston Police Department|Boston Water & Sewer Commission,3,1,
+reason,String,true,,Administrative & General Requests,Street Lights,,Unsorted,0.1717,7,33,1892,18.92,,0,,0,,,,,,,,,,,20,Enforcement & Abandoned Vehicles,1,23,Administrative & General Requests|Animal Issues|Building|Employee & General Comments|Noise Disturban...,7,1,
+type,String,true,,Abandoned Vehicles,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,0.0707,10,47,2266,22.66,,0,,0,,,,,,,,,,,36,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|City/State Snow Issues|Electrical|General Comments For a Program or...,15,1,
+queue,String,true,,BTDT_AVRS Interface Queue,PWDx_Street Light_General Lighting Request,,Unsorted,0.2121,13,55,2802,28.02,,0,,0,,,,,,,,,,,35,BTDT_Parking Enforcement,1,21,*PREVIEW: BTDT_BostonBikes|BTDT_Engineering_New Sign and Pavement Marking Requests|BTDT_Sign Shop_Si...,15,1,
+department,String,true,,BTDT,PWDx,,Unsorted,0.3737,3,4,392,3.92,,0,,0,,,,,,,,,,,7,PWDx,1,49,GEN_,1,2,
+submittedphoto,String,true,,https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg,https://311.boston.gov/media/boston/report/photos/61d75bba05bbcf180c2d41de/report.jpg,,Unsorted,0.8537,0,100,3633,36.33,,58,,0.58,,,,,,,,,,,43,,1,58,*PREVIEW: https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg|http...,42,1,
+closedphoto,NULL,,,,,,,,,,,,,100,,1,,,,,,,,,,,1,,1,100,,0,0,
+location,String,true,, ,INTERSECTION of Verdun St & Gallivan Blvd  Dorchester  MA  ,,Unsorted,-0.0303,1,63,3938,39.38,,0,,0,,,,,,,,,,,98,563 Columbus Ave  Roxbury  MA  02118|INTERSECTION of Gallivan Blvd & Washington St  Dorchester  MA  ,2,2,*PREVIEW:  |103 N Beacon St  Brighton  MA  02135|11 Aberdeen St  Boston  MA  02215|1148 Hyde Park Av...,96,1,
+fire_district,String,true,, ,9,,Unsorted,0.1515,1,2,113,1.13,,0,,0,,,,,,,,,,,10,3,1,19, ,1,1,
+pwd_district,String,true,, ,1C,,Unsorted,0.0707,1,3,209,2.09,,0,,0,,,,,,,,,,,14,1B,1,16, ,1,1,
+city_council_district,String,true,, ,9,,Unsorted,0.1313,1,1,100,1,,0,,0,,,,,,,,,,,10,1,1,22, ,1,1,
+police_district,String,true,, ,E5,,Unsorted,0.1717,1,3,223,2.23,,0,,0,,,,,,,,,,,13,A1,1,20, ,1,1,
+neighborhood,String,true,, ,West Roxbury,,Unsorted,0.0303,1,38,1486,14.86,,0,,0,,,,,,,,,,,19,Dorchester,1,15, |Brighton|Mission Hill,3,1,
+neighborhood_services_district,String,true,, ,9,,Unsorted,0.0303,1,2,139,1.39,,0,,0,,,,,,,,,,,16,3,1,15, |12,2,1,
+ward,String,true,, ,Ward 9,,Unsorted,0.0505,1,7,499,4.99,,0,,0,,,,,,,,,,,42,Ward 3,1,10,*PREVIEW:  |01|02|04|06|07|1|10|16|18,23,1,
+precinct,String,true,, ,2210,,Unsorted,0.0408,0,4,393,3.93,,1,,0.01,,,,,,,,,,,76,0306,1,5,*PREVIEW: NULL| |0102|0105|0108|0109|0201|0204|0305|0307,61,1,
+location_street_name,String,true,,103 N Beacon St,INTERSECTION Verdun St & Gallivan Blvd,,Unsorted,-0.0204,0,45,1800,18,,1,,0.01,,,,,,,,,,,97,20 Washington St|563 Columbus Ave|INTERSECTION Gallivan Blvd & Washington St,3,2,*PREVIEW: NULL|103 N Beacon St|11 Aberdeen St|1148 Hyde Park Ave|119 L St|12 Derne St|126 Elm St|127...,94,1,
+location_zipcode,String,true,,02109,02215,,Unsorted,0.0488,0,5,415,4.15,,17,,0.17,,,,,,,,,,,24,,1,17,02126|02134|02210|02215,4,1,
+latitude,Float,,4233.6674,42.2553,42.3806,0.1253,Unsorted,0.0505,,,,,42.3367,0,4,0,0.0163,42.2034,42.2619,42.3204,42.3432,42.3594,0.039,42.4179,42.4764,-0.1667,78,42.3594,1,20,*PREVIEW: 42.2553|42.2601|42.2609|42.2645|42.2674|42.2789|42.2797|42.2804|42.2821|42.2878,74,1,
+longitude,Float,,-7107.2688,-71.1626,-71.0298,0.1328,Unsorted,0.1515,,,,,-71.0727,0,4,0,0.0121,-71.1741,-71.1294,-71.0848,-71.0609,-71.055,0.0298,-71.0104,-70.9658,-0.6101,77,-71.0587,1,19,*PREVIEW: -71.0298|-71.0301|-71.0309|-71.0323|-71.0325|-71.0329|-71.0336|-71.0338|-71.034|-71.0355,72,1,
+source,String,true,,Citizens Connect App,Self Service,,Unsorted,0.5354,12,20,1801,18.01,,0,,0,,,,,,,,,,,4,Citizens Connect App,1,56,Self Service,1,3,
+qsv__rowcount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,100
+qsv__columncount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,29
+qsv__filesize_bytes,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,47702
+qsv__fingerprint_hash,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0dfc92641f245a36f0e0eb5f6279925b8dc8cc36acc382af7d35974ba33f3bf9
diff --git a/resources/test/boston311-100-everything-nodate-stats.csv b/resources/test/boston311-100-everything-nodate-stats.csv
index e052932e8..907c6e4a1 100644
--- a/resources/test/boston311-100-everything-nodate-stats.csv
+++ b/resources/test/boston311-100-everything-nodate-stats.csv
@@ -1,34 +1,34 @@
-field,type,is_ascii,sum,min,max,range,sort_order,min_length,max_length,sum_length,avg_length,mean,nullcount,max_precision,sparsity,mad,lower_outer_fence,lower_inner_fence,q1,q2_median,q3,iqr,upper_inner_fence,upper_outer_fence,skewness,cardinality,mode,mode_count,mode_occurrences,antimode,antimode_count,antimode_occurrences,qsv__value
-case_enquiry_id,Integer,,10100411645180,101004113298,101004155594,42296,Unsorted,,,,,101004116451.8,0,,0,673,101004109567,101004111646,101004113725,101004114353,101004115111,1386,101004117190,101004119269,0.0938,100,,0,0,*ALL,0,1,
-open_dt,String,true,,2022-01-01 00:16:00,2022-01-31 11:46:00,,Unsorted,19,19,1900,19,,0,,0,,,,,,,,,,,100,,0,0,*ALL,0,1,
-target_dt,String,true,,2022-01-03 10:32:34,2022-05-20 13:03:21,,Unsorted,0,19,1691,16.91,,11,,0.11,,,,,,,,,,,42,2022-01-04 08:30:00,1,25,*PREVIEW: 2022-01-03 10:32:34|2022-01-03 11:58:12|2022-01-04 09:58:36|2022-01-04 10:41:29|2022-01-04...,34,1,
-closed_dt,String,true,,2022-01-01 12:56:14,2022-04-25 14:30:31,,Unsorted,0,19,1615,16.15,,15,,0.15,,,,,,,,,,,86,,1,15,*PREVIEW: 2022-01-01 12:56:14|2022-01-01 14:17:15|2022-01-01 14:59:41|2022-01-01 15:10:16|2022-01-01...,85,1,
-ontime,String,true,,ONTIME,OVERDUE,,Unsorted,6,7,617,6.17,,0,,0,,,,,,,,,,,2,ONTIME,1,83,OVERDUE,1,17,
-case_status,String,true,,Closed,Open,,Unsorted,4,6,570,5.7,,0,,0,,,,,,,,,,,2,Closed,1,85,Open,1,15,
-closure_reason,String,true,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,1,284,8314,83.14,,0,,0,,,,,,,,,,,86, ,1,15,*PREVIEW: Case Closed Case Resolved  NEW CART#21026466 DELV ON 1/11/22  |Case Closed Case Resolved  ...,85,1,
-case_title,String,true,,Abandoned Vehicles,Traffic Signal Inspection,,Unsorted,10,57,2386,23.86,,0,,0,,,,,,,,,,,42,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|BTDT: Complaint|City/State Snow Issues|DISPATCHED Short Term Rental...,24,1,
-subject,String,true,,Animal Control,Transportation - Traffic Division,,Unsorted,14,33,2570,25.7,,0,,0,,,,,,,,,,,9,Public Works Department,1,51,Animal Control|Boston Police Department|Boston Water & Sewer Commission,3,1,
-reason,String,true,,Administrative & General Requests,Street Lights,,Unsorted,7,33,1892,18.92,,0,,0,,,,,,,,,,,20,Enforcement & Abandoned Vehicles,1,23,Administrative & General Requests|Animal Issues|Building|Employee & General Comments|Noise Disturban...,7,1,
-type,String,true,,Abandoned Vehicles,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,10,47,2266,22.66,,0,,0,,,,,,,,,,,36,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|City/State Snow Issues|Electrical|General Comments For a Program or...,15,1,
-queue,String,true,,BTDT_AVRS Interface Queue,PWDx_Street Light_General Lighting Request,,Unsorted,13,55,2802,28.02,,0,,0,,,,,,,,,,,35,BTDT_Parking Enforcement,1,21,*PREVIEW: BTDT_BostonBikes|BTDT_Engineering_New Sign and Pavement Marking Requests|BTDT_Sign Shop_Si...,15,1,
-department,String,true,,BTDT,PWDx,,Unsorted,3,4,392,3.92,,0,,0,,,,,,,,,,,7,PWDx,1,49,GEN_,1,2,
-submittedphoto,String,true,,https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg,https://311.boston.gov/media/boston/report/photos/61d75bba05bbcf180c2d41de/report.jpg,,Unsorted,0,100,3633,36.33,,58,,0.58,,,,,,,,,,,43,,1,58,*PREVIEW: https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg|http...,42,1,
-closedphoto,NULL,,,,,,,,,,,,100,,1,,,,,,,,,,,1,,1,100,,0,0,
-location,String,true,, ,INTERSECTION of Verdun St & Gallivan Blvd  Dorchester  MA  ,,Unsorted,1,63,3938,39.38,,0,,0,,,,,,,,,,,98,563 Columbus Ave  Roxbury  MA  02118|INTERSECTION of Gallivan Blvd & Washington St  Dorchester  MA  ,2,2,*PREVIEW:  |103 N Beacon St  Brighton  MA  02135|11 Aberdeen St  Boston  MA  02215|1148 Hyde Park Av...,96,1,
-fire_district,String,true,, ,9,,Unsorted,1,2,113,1.13,,0,,0,,,,,,,,,,,10,3,1,19, ,1,1,
-pwd_district,String,true,, ,1C,,Unsorted,1,3,209,2.09,,0,,0,,,,,,,,,,,14,1B,1,16, ,1,1,
-city_council_district,String,true,, ,9,,Unsorted,1,1,100,1,,0,,0,,,,,,,,,,,10,1,1,22, ,1,1,
-police_district,String,true,, ,E5,,Unsorted,1,3,223,2.23,,0,,0,,,,,,,,,,,13,A1,1,20, ,1,1,
-neighborhood,String,true,, ,West Roxbury,,Unsorted,1,38,1486,14.86,,0,,0,,,,,,,,,,,19,Dorchester,1,15, |Brighton|Mission Hill,3,1,
-neighborhood_services_district,String,true,, ,9,,Unsorted,1,2,139,1.39,,0,,0,,,,,,,,,,,16,3,1,15, |12,2,1,
-ward,String,true,, ,Ward 9,,Unsorted,1,7,499,4.99,,0,,0,,,,,,,,,,,42,Ward 3,1,10,*PREVIEW:  |01|02|04|06|07|1|10|16|18,23,1,
-precinct,String,true,, ,2210,,Unsorted,0,4,393,3.93,,1,,0.01,,,,,,,,,,,76,0306,1,5,*PREVIEW: NULL| |0102|0105|0108|0109|0201|0204|0305|0307,61,1,
-location_street_name,String,true,,103 N Beacon St,INTERSECTION Verdun St & Gallivan Blvd,,Unsorted,0,45,1800,18,,1,,0.01,,,,,,,,,,,97,20 Washington St|563 Columbus Ave|INTERSECTION Gallivan Blvd & Washington St,3,2,*PREVIEW: NULL|103 N Beacon St|11 Aberdeen St|1148 Hyde Park Ave|119 L St|12 Derne St|126 Elm St|127...,94,1,
-location_zipcode,String,true,,02109,02215,,Unsorted,0,5,415,4.15,,17,,0.17,,,,,,,,,,,24,,1,17,02126|02134|02210|02215,4,1,
-latitude,Float,,4233.6674,42.2553,42.3806,0.1253,Unsorted,,,,,42.3367,0,4,0,0.0163,42.2034,42.2619,42.3204,42.3432,42.3594,0.039,42.4179,42.4764,-0.1667,78,42.3594,1,20,*PREVIEW: 42.2553|42.2601|42.2609|42.2645|42.2674|42.2789|42.2797|42.2804|42.2821|42.2878,74,1,
-longitude,Float,,-7107.2688,-71.1626,-71.0298,0.1328,Unsorted,,,,,-71.0727,0,4,0,0.0121,-71.1741,-71.1294,-71.0848,-71.0609,-71.055,0.0298,-71.0104,-70.9658,-0.6101,77,-71.0587,1,19,*PREVIEW: -71.0298|-71.0301|-71.0309|-71.0323|-71.0325|-71.0329|-71.0336|-71.0338|-71.034|-71.0355,72,1,
-source,String,true,,Citizens Connect App,Self Service,,Unsorted,12,20,1801,18.01,,0,,0,,,,,,,,,,,4,Citizens Connect App,1,56,Self Service,1,3,
-qsv__rowcount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,100
-qsv__columncount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,29
-qsv__filesize_bytes,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,47702
-qsv__fingerprint_hash,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,53018bee30100ff6bb30214ed7dbb6d7c1a40115a6c91b47b451fb76dd211e59
+field,type,is_ascii,sum,min,max,range,sort_order,sortiness,min_length,max_length,sum_length,avg_length,mean,nullcount,max_precision,sparsity,mad,lower_outer_fence,lower_inner_fence,q1,q2_median,q3,iqr,upper_inner_fence,upper_outer_fence,skewness,cardinality,mode,mode_count,mode_occurrences,antimode,antimode_count,antimode_occurrences,qsv__value
+case_enquiry_id,Integer,,10100411645180,101004113298,101004155594,42296,Unsorted,0.798,,,,,101004116451.8,0,,0,673,101004109567,101004111646,101004113725,101004114353,101004115111,1386,101004117190,101004119269,0.0938,100,,0,0,*ALL,0,1,
+open_dt,String,true,,2022-01-01 00:16:00,2022-01-31 11:46:00,,Unsorted,0.798,19,19,1900,19,,0,,0,,,,,,,,,,,100,,0,0,*ALL,0,1,
+target_dt,String,true,,2022-01-03 10:32:34,2022-05-20 13:03:21,,Unsorted,0.2273,0,19,1691,16.91,,11,,0.11,,,,,,,,,,,42,2022-01-04 08:30:00,1,25,*PREVIEW: 2022-01-03 10:32:34|2022-01-03 11:58:12|2022-01-04 09:58:36|2022-01-04 10:41:29|2022-01-04...,34,1,
+closed_dt,String,true,,2022-01-01 12:56:14,2022-04-25 14:30:31,,Unsorted,-0.0714,0,19,1615,16.15,,15,,0.15,,,,,,,,,,,86,,1,15,*PREVIEW: 2022-01-01 12:56:14|2022-01-01 14:17:15|2022-01-01 14:59:41|2022-01-01 15:10:16|2022-01-01...,85,1,
+ontime,String,true,,ONTIME,OVERDUE,,Unsorted,0.6768,6,7,617,6.17,,0,,0,,,,,,,,,,,2,ONTIME,1,83,OVERDUE,1,17,
+case_status,String,true,,Closed,Open,,Unsorted,0.7172,4,6,570,5.7,,0,,0,,,,,,,,,,,2,Closed,1,85,Open,1,15,
+closure_reason,String,true,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,-0.0909,1,284,8314,83.14,,0,,0,,,,,,,,,,,86, ,1,15,*PREVIEW: Case Closed Case Resolved  NEW CART#21026466 DELV ON 1/11/22  |Case Closed Case Resolved  ...,85,1,
+case_title,String,true,,Abandoned Vehicles,Traffic Signal Inspection,,Unsorted,0.0101,10,57,2386,23.86,,0,,0,,,,,,,,,,,42,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|BTDT: Complaint|City/State Snow Issues|DISPATCHED Short Term Rental...,24,1,
+subject,String,true,,Animal Control,Transportation - Traffic Division,,Unsorted,0.3737,14,33,2570,25.7,,0,,0,,,,,,,,,,,9,Public Works Department,1,51,Animal Control|Boston Police Department|Boston Water & Sewer Commission,3,1,
+reason,String,true,,Administrative & General Requests,Street Lights,,Unsorted,0.1717,7,33,1892,18.92,,0,,0,,,,,,,,,,,20,Enforcement & Abandoned Vehicles,1,23,Administrative & General Requests|Animal Issues|Building|Employee & General Comments|Noise Disturban...,7,1,
+type,String,true,,Abandoned Vehicles,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,0.0707,10,47,2266,22.66,,0,,0,,,,,,,,,,,36,Parking Enforcement,1,20,*PREVIEW: Animal Generic Request|City/State Snow Issues|Electrical|General Comments For a Program or...,15,1,
+queue,String,true,,BTDT_AVRS Interface Queue,PWDx_Street Light_General Lighting Request,,Unsorted,0.2121,13,55,2802,28.02,,0,,0,,,,,,,,,,,35,BTDT_Parking Enforcement,1,21,*PREVIEW: BTDT_BostonBikes|BTDT_Engineering_New Sign and Pavement Marking Requests|BTDT_Sign Shop_Si...,15,1,
+department,String,true,,BTDT,PWDx,,Unsorted,0.3737,3,4,392,3.92,,0,,0,,,,,,,,,,,7,PWDx,1,49,GEN_,1,2,
+submittedphoto,String,true,,https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg,https://311.boston.gov/media/boston/report/photos/61d75bba05bbcf180c2d41de/report.jpg,,Unsorted,0.8537,0,100,3633,36.33,,58,,0.58,,,,,,,,,,,43,,1,58,*PREVIEW: https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg|http...,42,1,
+closedphoto,NULL,,,,,,,,,,,,,100,,1,,,,,,,,,,,1,,1,100,,0,0,
+location,String,true,, ,INTERSECTION of Verdun St & Gallivan Blvd  Dorchester  MA  ,,Unsorted,-0.0303,1,63,3938,39.38,,0,,0,,,,,,,,,,,98,563 Columbus Ave  Roxbury  MA  02118|INTERSECTION of Gallivan Blvd & Washington St  Dorchester  MA  ,2,2,*PREVIEW:  |103 N Beacon St  Brighton  MA  02135|11 Aberdeen St  Boston  MA  02215|1148 Hyde Park Av...,96,1,
+fire_district,String,true,, ,9,,Unsorted,0.1515,1,2,113,1.13,,0,,0,,,,,,,,,,,10,3,1,19, ,1,1,
+pwd_district,String,true,, ,1C,,Unsorted,0.0707,1,3,209,2.09,,0,,0,,,,,,,,,,,14,1B,1,16, ,1,1,
+city_council_district,String,true,, ,9,,Unsorted,0.1313,1,1,100,1,,0,,0,,,,,,,,,,,10,1,1,22, ,1,1,
+police_district,String,true,, ,E5,,Unsorted,0.1717,1,3,223,2.23,,0,,0,,,,,,,,,,,13,A1,1,20, ,1,1,
+neighborhood,String,true,, ,West Roxbury,,Unsorted,0.0303,1,38,1486,14.86,,0,,0,,,,,,,,,,,19,Dorchester,1,15, |Brighton|Mission Hill,3,1,
+neighborhood_services_district,String,true,, ,9,,Unsorted,0.0303,1,2,139,1.39,,0,,0,,,,,,,,,,,16,3,1,15, |12,2,1,
+ward,String,true,, ,Ward 9,,Unsorted,0.0505,1,7,499,4.99,,0,,0,,,,,,,,,,,42,Ward 3,1,10,*PREVIEW:  |01|02|04|06|07|1|10|16|18,23,1,
+precinct,String,true,, ,2210,,Unsorted,0.0408,0,4,393,3.93,,1,,0.01,,,,,,,,,,,76,0306,1,5,*PREVIEW: NULL| |0102|0105|0108|0109|0201|0204|0305|0307,61,1,
+location_street_name,String,true,,103 N Beacon St,INTERSECTION Verdun St & Gallivan Blvd,,Unsorted,-0.0204,0,45,1800,18,,1,,0.01,,,,,,,,,,,97,20 Washington St|563 Columbus Ave|INTERSECTION Gallivan Blvd & Washington St,3,2,*PREVIEW: NULL|103 N Beacon St|11 Aberdeen St|1148 Hyde Park Ave|119 L St|12 Derne St|126 Elm St|127...,94,1,
+location_zipcode,String,true,,02109,02215,,Unsorted,0.0488,0,5,415,4.15,,17,,0.17,,,,,,,,,,,24,,1,17,02126|02134|02210|02215,4,1,
+latitude,Float,,4233.6674,42.2553,42.3806,0.1253,Unsorted,0.0505,,,,,42.3367,0,4,0,0.0163,42.2034,42.2619,42.3204,42.3432,42.3594,0.039,42.4179,42.4764,-0.1667,78,42.3594,1,20,*PREVIEW: 42.2553|42.2601|42.2609|42.2645|42.2674|42.2789|42.2797|42.2804|42.2821|42.2878,74,1,
+longitude,Float,,-7107.2688,-71.1626,-71.0298,0.1328,Unsorted,0.1515,,,,,-71.0727,0,4,0,0.0121,-71.1741,-71.1294,-71.0848,-71.0609,-71.055,0.0298,-71.0104,-70.9658,-0.6101,77,-71.0587,1,19,*PREVIEW: -71.0298|-71.0301|-71.0309|-71.0323|-71.0325|-71.0329|-71.0336|-71.0338|-71.034|-71.0355,72,1,
+source,String,true,,Citizens Connect App,Self Service,,Unsorted,0.5354,12,20,1801,18.01,,0,,0,,,,,,,,,,,4,Citizens Connect App,1,56,Self Service,1,3,
+qsv__rowcount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,100
+qsv__columncount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,29
+qsv__filesize_bytes,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,47702
+qsv__fingerprint_hash,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2427f44545835ae925b85f6c5fc713f9aaf81756146c770e19369ab90387e062
diff --git a/resources/test/boston311-100-stats.csv b/resources/test/boston311-100-stats.csv
index 571de8a27..6047a3cf3 100644
--- a/resources/test/boston311-100-stats.csv
+++ b/resources/test/boston311-100-stats.csv
@@ -1,34 +1,34 @@
-field,type,is_ascii,sum,min,max,range,sort_order,min_length,max_length,sum_length,avg_length,mean,nullcount,max_precision,sparsity,qsv__value
-case_enquiry_id,Integer,,10100411645180,101004113298,101004155594,42296,Unsorted,,,,,101004116451.8,0,,0,
-open_dt,DateTime,,,2022-01-01T00:16:00+00:00,2022-01-31T11:46:00+00:00,30.47917,Unsorted,,,,,2022-01-04T07:07:45.050+00:00,0,,0,
-target_dt,DateTime,,,2022-01-03T10:32:34+00:00,2022-05-20T13:03:21+00:00,137.10471,Unsorted,,,,,2022-01-17T03:14:16.404+00:00,11,,0.11,
-closed_dt,DateTime,,,2022-01-01T12:56:14+00:00,2022-04-25T14:30:31+00:00,114.06547,Unsorted,,,,,2022-01-08T01:10:44.411+00:00,15,,0.15,
-ontime,String,true,,ONTIME,OVERDUE,,Unsorted,6,7,617,6.17,,0,,0,
-case_status,String,true,,Closed,Open,,Unsorted,4,6,570,5.7,,0,,0,
-closure_reason,String,true,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,1,284,8314,83.14,,0,,0,
-case_title,String,true,,Abandoned Vehicles,Traffic Signal Inspection,,Unsorted,10,57,2386,23.86,,0,,0,
-subject,String,true,,Animal Control,Transportation - Traffic Division,,Unsorted,14,33,2570,25.7,,0,,0,
-reason,String,true,,Administrative & General Requests,Street Lights,,Unsorted,7,33,1892,18.92,,0,,0,
-type,String,true,,Abandoned Vehicles,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,10,47,2266,22.66,,0,,0,
-queue,String,true,,BTDT_AVRS Interface Queue,PWDx_Street Light_General Lighting Request,,Unsorted,13,55,2802,28.02,,0,,0,
-department,String,true,,BTDT,PWDx,,Unsorted,3,4,392,3.92,,0,,0,
-submittedphoto,String,true,,https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg,https://311.boston.gov/media/boston/report/photos/61d75bba05bbcf180c2d41de/report.jpg,,Unsorted,0,100,3633,36.33,,58,,0.58,
-closedphoto,NULL,,,,,,,,,,,,100,,1,
-location,String,true,, ,INTERSECTION of Verdun St & Gallivan Blvd  Dorchester  MA  ,,Unsorted,1,63,3938,39.38,,0,,0,
-fire_district,String,true,, ,9,,Unsorted,1,2,113,1.13,,0,,0,
-pwd_district,String,true,, ,1C,,Unsorted,1,3,209,2.09,,0,,0,
-city_council_district,String,true,, ,9,,Unsorted,1,1,100,1,,0,,0,
-police_district,String,true,, ,E5,,Unsorted,1,3,223,2.23,,0,,0,
-neighborhood,String,true,, ,West Roxbury,,Unsorted,1,38,1486,14.86,,0,,0,
-neighborhood_services_district,String,true,, ,9,,Unsorted,1,2,139,1.39,,0,,0,
-ward,String,true,, ,Ward 9,,Unsorted,1,7,499,4.99,,0,,0,
-precinct,String,true,, ,2210,,Unsorted,0,4,393,3.93,,1,,0.01,
-location_street_name,String,true,,103 N Beacon St,INTERSECTION Verdun St & Gallivan Blvd,,Unsorted,0,45,1800,18,,1,,0.01,
-location_zipcode,String,true,,02109,02215,,Unsorted,0,5,415,4.15,,17,,0.17,
-latitude,Float,,4233.6674,42.2553,42.3806,0.1253,Unsorted,,,,,42.3367,0,4,0,
-longitude,Float,,-7107.2688,-71.1626,-71.0298,0.1328,Unsorted,,,,,-71.0727,0,4,0,
-source,String,true,,Citizens Connect App,Self Service,,Unsorted,12,20,1801,18.01,,0,,0,
-qsv__rowcount,,,,,,,,,,,,,,,,100
-qsv__columncount,,,,,,,,,,,,,,,,29
-qsv__filesize_bytes,,,,,,,,,,,,,,,,47702
-qsv__fingerprint_hash,,,,,,,,,,,,,,,,85611c6f33acdc78001619e131cbf15cd0e2cc41abb8865ec879a0b529ce2642
+field,type,is_ascii,sum,min,max,range,sort_order,sortiness,min_length,max_length,sum_length,avg_length,mean,nullcount,max_precision,sparsity,qsv__value
+case_enquiry_id,Integer,,10100411645180,101004113298,101004155594,42296,Unsorted,0.798,,,,,101004116451.8,0,,0,
+open_dt,DateTime,,,2022-01-01T00:16:00+00:00,2022-01-31T11:46:00+00:00,30.47917,Unsorted,0.798,,,,,2022-01-04T07:07:45.050+00:00,0,,0,
+target_dt,DateTime,,,2022-01-03T10:32:34+00:00,2022-05-20T13:03:21+00:00,137.10471,Unsorted,0.2273,,,,,2022-01-17T03:14:16.404+00:00,11,,0.11,
+closed_dt,DateTime,,,2022-01-01T12:56:14+00:00,2022-04-25T14:30:31+00:00,114.06547,Unsorted,-0.0714,,,,,2022-01-08T01:10:44.411+00:00,15,,0.15,
+ontime,String,true,,ONTIME,OVERDUE,,Unsorted,0.6768,6,7,617,6.17,,0,,0,
+case_status,String,true,,Closed,Open,,Unsorted,0.7172,4,6,570,5.7,,0,,0,
+closure_reason,String,true,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,-0.0909,1,284,8314,83.14,,0,,0,
+case_title,String,true,,Abandoned Vehicles,Traffic Signal Inspection,,Unsorted,0.0101,10,57,2386,23.86,,0,,0,
+subject,String,true,,Animal Control,Transportation - Traffic Division,,Unsorted,0.3737,14,33,2570,25.7,,0,,0,
+reason,String,true,,Administrative & General Requests,Street Lights,,Unsorted,0.1717,7,33,1892,18.92,,0,,0,
+type,String,true,,Abandoned Vehicles,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,0.0707,10,47,2266,22.66,,0,,0,
+queue,String,true,,BTDT_AVRS Interface Queue,PWDx_Street Light_General Lighting Request,,Unsorted,0.2121,13,55,2802,28.02,,0,,0,
+department,String,true,,BTDT,PWDx,,Unsorted,0.3737,3,4,392,3.92,,0,,0,
+submittedphoto,String,true,,https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg,https://311.boston.gov/media/boston/report/photos/61d75bba05bbcf180c2d41de/report.jpg,,Unsorted,0.8537,0,100,3633,36.33,,58,,0.58,
+closedphoto,NULL,,,,,,,,,,,,,100,,1,
+location,String,true,, ,INTERSECTION of Verdun St & Gallivan Blvd  Dorchester  MA  ,,Unsorted,-0.0303,1,63,3938,39.38,,0,,0,
+fire_district,String,true,, ,9,,Unsorted,0.1515,1,2,113,1.13,,0,,0,
+pwd_district,String,true,, ,1C,,Unsorted,0.0707,1,3,209,2.09,,0,,0,
+city_council_district,String,true,, ,9,,Unsorted,0.1313,1,1,100,1,,0,,0,
+police_district,String,true,, ,E5,,Unsorted,0.1717,1,3,223,2.23,,0,,0,
+neighborhood,String,true,, ,West Roxbury,,Unsorted,0.0303,1,38,1486,14.86,,0,,0,
+neighborhood_services_district,String,true,, ,9,,Unsorted,0.0303,1,2,139,1.39,,0,,0,
+ward,String,true,, ,Ward 9,,Unsorted,0.0505,1,7,499,4.99,,0,,0,
+precinct,String,true,, ,2210,,Unsorted,0.0408,0,4,393,3.93,,1,,0.01,
+location_street_name,String,true,,103 N Beacon St,INTERSECTION Verdun St & Gallivan Blvd,,Unsorted,-0.0204,0,45,1800,18,,1,,0.01,
+location_zipcode,String,true,,02109,02215,,Unsorted,0.0488,0,5,415,4.15,,17,,0.17,
+latitude,Float,,4233.6674,42.2553,42.3806,0.1253,Unsorted,0.0505,,,,,42.3367,0,4,0,
+longitude,Float,,-7107.2688,-71.1626,-71.0298,0.1328,Unsorted,0.1515,,,,,-71.0727,0,4,0,
+source,String,true,,Citizens Connect App,Self Service,,Unsorted,0.5354,12,20,1801,18.01,,0,,0,
+qsv__rowcount,,,,,,,,,,,,,,,,,100
+qsv__columncount,,,,,,,,,,,,,,,,,29
+qsv__filesize_bytes,,,,,,,,,,,,,,,,,47702
+qsv__fingerprint_hash,,,,,,,,,,,,,,,,,6baac425a955df5a52aa16e21a329a68bae1ebeca05226468cd32f859bd7bd7a
diff --git a/resources/test/boston311-100-with-nonascii-stats.csv b/resources/test/boston311-100-with-nonascii-stats.csv
index e7207048c..d9d1c4926 100644
--- a/resources/test/boston311-100-with-nonascii-stats.csv
+++ b/resources/test/boston311-100-with-nonascii-stats.csv
@@ -1,34 +1,34 @@
-field,type,is_ascii,sum,min,max,range,sort_order,min_length,max_length,sum_length,avg_length,mean,nullcount,max_precision,sparsity,qsv__value
-case_enquiry_id,Integer,,10100411645180,101004113298,101004155594,42296,Unsorted,,,,,101004116451.8,0,,0,
-open_dt,String,true,,2022-01-01 00:16:00,2022-01-31 11:46:00,,Unsorted,19,19,1900,19,,0,,0,
-target_dt,String,true,,2022-01-03 10:32:34,2022-05-20 13:03:21,,Unsorted,0,19,1691,16.91,,11,,0.11,
-closed_dt,String,true,,2022-01-01 12:56:14,2022-04-25 14:30:31,,Unsorted,0,19,1615,16.15,,15,,0.15,
-ontime,String,true,,ONTIME,OVERDUE,,Unsorted,6,7,617,6.17,,0,,0,
-case_status,String,true,,Closed,Open,,Unsorted,4,6,570,5.7,,0,,0,
-closure_reason,String,false,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,1,284,8315,83.15,,0,,0,
-case_title,String,true,,Abandoned Vehicles,Traffic Signal Inspection,,Unsorted,10,57,2386,23.86,,0,,0,
-subject,String,true,,Animal Control,Transportation - Traffic Division,,Unsorted,14,33,2570,25.7,,0,,0,
-reason,String,true,,Administrative & General Requests,Street Lights,,Unsorted,7,33,1892,18.92,,0,,0,
-type,String,true,,Abandoned Vehicles,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,10,47,2266,22.66,,0,,0,
-queue,String,true,,BTDT_AVRS Interface Queue,PWDx_Street Light_General Lighting Request,,Unsorted,13,55,2802,28.02,,0,,0,
-department,String,true,,BTDT,PWDx,,Unsorted,3,4,392,3.92,,0,,0,
-submittedphoto,String,true,,https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg,https://311.boston.gov/media/boston/report/photos/61d75bba05bbcf180c2d41de/report.jpg,,Unsorted,0,100,3633,36.33,,58,,0.58,
-closedphoto,NULL,,,,,,,,,,,,100,,1,
-location,String,false,, ,INTERSECTION of Verdun St & Gallivan Blvd  Dorchester  MA  ,,Unsorted,1,63,3939,39.39,,0,,0,
-fire_district,String,true,, ,9,,Unsorted,1,2,113,1.13,,0,,0,
-pwd_district,String,true,, ,1C,,Unsorted,1,3,209,2.09,,0,,0,
-city_council_district,String,true,, ,9,,Unsorted,1,1,100,1,,0,,0,
-police_district,String,true,, ,E5,,Unsorted,1,3,223,2.23,,0,,0,
-neighborhood,String,true,, ,West Roxbury,,Unsorted,1,38,1486,14.86,,0,,0,
-neighborhood_services_district,String,true,, ,9,,Unsorted,1,2,139,1.39,,0,,0,
-ward,String,true,, ,Ward 9,,Unsorted,1,7,499,4.99,,0,,0,
-precinct,String,true,, ,2210,,Unsorted,0,4,393,3.93,,1,,0.01,
-location_street_name,String,true,,103 N Beacon St,INTERSECTION Verdun St & Gallivan Blvd,,Unsorted,0,45,1800,18,,1,,0.01,
-location_zipcode,String,true,,02109,02215,,Unsorted,0,5,415,4.15,,17,,0.17,
-latitude,Float,,4233.6674,42.2553,42.3806,0.1253,Unsorted,,,,,42.3367,0,4,0,
-longitude,Float,,-7107.2688,-71.1626,-71.0298,0.1328,Unsorted,,,,,-71.0727,0,4,0,
-source,String,true,,Citizens Connect App,Self Service,,Unsorted,12,20,1801,18.01,,0,,0,
-qsv__rowcount,,,,,,,,,,,,,,,,100
-qsv__columncount,,,,,,,,,,,,,,,,29
-qsv__filesize_bytes,,,,,,,,,,,,,,,,47704
-qsv__fingerprint_hash,,,,,,,,,,,,,,,,980cb8155b8cf89f84c25b6cc61f734d6b7eb0542150e93bc359abc811ba7e84
+field,type,is_ascii,sum,min,max,range,sort_order,sortiness,min_length,max_length,sum_length,avg_length,mean,nullcount,max_precision,sparsity,qsv__value
+case_enquiry_id,Integer,,10100411645180,101004113298,101004155594,42296,Unsorted,0.798,,,,,101004116451.8,0,,0,
+open_dt,String,true,,2022-01-01 00:16:00,2022-01-31 11:46:00,,Unsorted,0.798,19,19,1900,19,,0,,0,
+target_dt,String,true,,2022-01-03 10:32:34,2022-05-20 13:03:21,,Unsorted,0.2273,0,19,1691,16.91,,11,,0.11,
+closed_dt,String,true,,2022-01-01 12:56:14,2022-04-25 14:30:31,,Unsorted,-0.0714,0,19,1615,16.15,,15,,0.15,
+ontime,String,true,,ONTIME,OVERDUE,,Unsorted,0.6768,6,7,617,6.17,,0,,0,
+case_status,String,true,,Closed,Open,,Unsorted,0.7172,4,6,570,5.7,,0,,0,
+closure_reason,String,false,, ,Case Closed. Closed date : Wed Jan 19 11:42:16 EST 2022 Resolved Removed df  ,,Unsorted,-0.0909,1,284,8315,83.15,,0,,0,
+case_title,String,true,,Abandoned Vehicles,Traffic Signal Inspection,,Unsorted,0.0101,10,57,2386,23.86,,0,,0,
+subject,String,true,,Animal Control,Transportation - Traffic Division,,Unsorted,0.3737,14,33,2570,25.7,,0,,0,
+reason,String,true,,Administrative & General Requests,Street Lights,,Unsorted,0.1717,7,33,1892,18.92,,0,,0,
+type,String,true,,Abandoned Vehicles,Unsatisfactory Utilities - Electrical  Plumbing,,Unsorted,0.0707,10,47,2266,22.66,,0,,0,
+queue,String,true,,BTDT_AVRS Interface Queue,PWDx_Street Light_General Lighting Request,,Unsorted,0.2121,13,55,2802,28.02,,0,,0,
+department,String,true,,BTDT,PWDx,,Unsorted,0.3737,3,4,392,3.92,,0,,0,
+submittedphoto,String,true,,https://311.boston.gov/media/boston/report/photos/61d03f0d05bbcf180c2965fd/report.jpg,https://311.boston.gov/media/boston/report/photos/61d75bba05bbcf180c2d41de/report.jpg,,Unsorted,0.8537,0,100,3633,36.33,,58,,0.58,
+closedphoto,NULL,,,,,,,,,,,,,100,,1,
+location,String,false,, ,INTERSECTION of Verdun St & Gallivan Blvd  Dorchester  MA  ,,Unsorted,-0.0303,1,63,3939,39.39,,0,,0,
+fire_district,String,true,, ,9,,Unsorted,0.1515,1,2,113,1.13,,0,,0,
+pwd_district,String,true,, ,1C,,Unsorted,0.0707,1,3,209,2.09,,0,,0,
+city_council_district,String,true,, ,9,,Unsorted,0.1313,1,1,100,1,,0,,0,
+police_district,String,true,, ,E5,,Unsorted,0.1717,1,3,223,2.23,,0,,0,
+neighborhood,String,true,, ,West Roxbury,,Unsorted,0.0303,1,38,1486,14.86,,0,,0,
+neighborhood_services_district,String,true,, ,9,,Unsorted,0.0303,1,2,139,1.39,,0,,0,
+ward,String,true,, ,Ward 9,,Unsorted,0.0505,1,7,499,4.99,,0,,0,
+precinct,String,true,, ,2210,,Unsorted,0.0408,0,4,393,3.93,,1,,0.01,
+location_street_name,String,true,,103 N Beacon St,INTERSECTION Verdun St & Gallivan Blvd,,Unsorted,-0.0204,0,45,1800,18,,1,,0.01,
+location_zipcode,String,true,,02109,02215,,Unsorted,0.0488,0,5,415,4.15,,17,,0.17,
+latitude,Float,,4233.6674,42.2553,42.3806,0.1253,Unsorted,0.0505,,,,,42.3367,0,4,0,
+longitude,Float,,-7107.2688,-71.1626,-71.0298,0.1328,Unsorted,0.1515,,,,,-71.0727,0,4,0,
+source,String,true,,Citizens Connect App,Self Service,,Unsorted,0.5354,12,20,1801,18.01,,0,,0,
+qsv__rowcount,,,,,,,,,,,,,,,,,100
+qsv__columncount,,,,,,,,,,,,,,,,,29
+qsv__filesize_bytes,,,,,,,,,,,,,,,,,47704
+qsv__fingerprint_hash,,,,,,,,,,,,,,,,,fdd60e727dc06419e103ce97894e78911c6f00beae0e4d6104e52eecbb21a1af
diff --git a/tests/test_index.rs b/tests/test_index.rs
index 48483c37b..c1a0ea4d9 100644
--- a/tests/test_index.rs
+++ b/tests/test_index.rs
@@ -73,6 +73,7 @@ fn index_outdated_stats() {
             "max",
             "range",
             "sort_order",
+            "sortiness",
             "min_length",
             "max_length",
             "sum_length",
@@ -103,6 +104,7 @@ fn index_outdated_stats() {
             "Ascending",
             "1",
             "1",
+            "1",
             "3",
             "1",
             "0",
@@ -129,6 +131,7 @@ fn index_outdated_stats() {
             "3",
             "2",
             "Descending",
+            "-1",
             "",
             "",
             "",
@@ -174,6 +177,7 @@ fn index_outdated_stats() {
             "",
             "",
             "",
+            "",
             "3"
         ],
         svec![
@@ -202,6 +206,7 @@ fn index_outdated_stats() {
             "",
             "",
             "",
+            "",
             "2"
         ],
         svec![
@@ -230,6 +235,7 @@ fn index_outdated_stats() {
             "",
             "",
             "",
+            "",
             "26"
         ],
         svec![
@@ -258,7 +264,8 @@ fn index_outdated_stats() {
             "",
             "",
             "",
-            "dcbb915759b3be9b3e8918f8dfaa3c84472c93cded28ab42b20d6092b5667cbe"
+            "",
+            "544903bb86bbdfdbbd119d5db5316048887046c24d7b6a4e757a2cf88543006f"
         ],
     ];
 
diff --git a/tests/test_stats.rs b/tests/test_stats.rs
index 02b472512..fd87c8901 100644
--- a/tests/test_stats.rs
+++ b/tests/test_stats.rs
@@ -1247,6 +1247,7 @@ fn stats_zero_cv() {
             "max",
             "range",
             "sort_order",
+            "sortiness",
             "min_length",
             "max_length",
             "sum_length",
@@ -1275,6 +1276,7 @@ fn stats_zero_cv() {
             "5",
             "4",
             "Ascending",
+            "1",
             "",
             "",
             "",
@@ -1303,6 +1305,7 @@ fn stats_zero_cv() {
             "10",
             "20",
             "Ascending",
+            "1",
             "",
             "",
             "",
@@ -1331,6 +1334,7 @@ fn stats_zero_cv() {
             "100.0",
             "200",
             "Ascending",
+            "1",
             "",
             "",
             "",
@@ -1351,8 +1355,9 @@ fn stats_zero_cv() {
             ""
         ],
         svec![
-            "col4", "Integer", "", "935", "-900", "1000", "1900", "Unsorted", "", "", "", "", "",
-            "", "", "187", "304.3603", "0", "", "680.5703", "463176", "363.9414", "0", "", "0", ""
+            "col4", "Integer", "", "935", "-900", "1000", "1900", "Unsorted", "-0.5", "", "", "",
+            "", "", "", "", "187", "304.3603", "0", "", "680.5703", "463176", "363.9414", "0", "",
+            "0", ""
         ],
         svec![
             "qsv__rowcount",
@@ -1380,6 +1385,7 @@ fn stats_zero_cv() {
             "",
             "",
             "",
+            "",
             "5"
         ],
         svec![
@@ -1408,6 +1414,7 @@ fn stats_zero_cv() {
             "",
             "",
             "",
+            "",
             "4"
         ],
         svec![
@@ -1436,6 +1443,7 @@ fn stats_zero_cv() {
             "",
             "",
             "",
+            "",
             "93"
         ],
         svec![
@@ -1464,7 +1472,8 @@ fn stats_zero_cv() {
             "",
             "",
             "",
-            "ce57f80725771c5a46e0299cdb0ae3ae3a97ede6721f02274bcf040aac21012f"
+            "",
+            "6e3c1035ebfe08ef286abbc7fd528717ff0ee129fbb9aacca5f27c529e39eff8"
         ],
     ];
     assert_eq!(got, expected);
@@ -1494,14 +1503,14 @@ fn stats_output_tab_delimited() {
     wrk.assert_success(&mut cmd);
 
     let got = std::fs::read_to_string(out_file).unwrap();
-    let expected = r#"field	type	is_ascii	sum	min	max	range	sort_order	min_length	max_length	sum_length	avg_length	stddev_length	variance_length	cv_length	mean	sem	geometric_mean	harmonic_mean	stddev	variance	cv	nullcount	max_precision	sparsity	qsv__value
-col1	Integer		15	1	5	4	Ascending								3	0.6325	2.6052	2.1898	1.4142	2	47.1405	0		0	
-col2	Integer		10644	0	4321	4321	Descending								2128.8	685.6979	0		1533.267	2350907.76	72.0249	0		0	
-col3	String	true		01	10		Ascending	2	2	10	2	0	0	0								0		0	
-qsv__rowcount																									5
-qsv__columncount																									3
-qsv__filesize_bytes																									62
-qsv__fingerprint_hash																									09e5437cc0b15ac327d13792b139d7fb2277e1e1b7851a2cb6ba92274e5261c4
+    let expected = r#"field	type	is_ascii	sum	min	max	range	sort_order	sortiness	min_length	max_length	sum_length	avg_length	stddev_length	variance_length	cv_length	mean	sem	geometric_mean	harmonic_mean	stddev	variance	cv	nullcount	max_precision	sparsity	qsv__value
+col1	Integer		15	1	5	4	Ascending	1								3	0.6325	2.6052	2.1898	1.4142	2	47.1405	0		0	
+col2	Integer		10644	0	4321	4321	Descending	-1								2128.8	685.6979	0		1533.267	2350907.76	72.0249	0		0	
+col3	String	true		01	10		Ascending	1	2	2	10	2	0	0	0								0		0	
+qsv__rowcount																										5
+qsv__columncount																										3
+qsv__filesize_bytes																										62
+qsv__fingerprint_hash																										cf6ba47ee7251a17b8d7789636b2c5d2302ea6e0fb69572f1eaf25aead083cef
 "#;
     assert_eq!(got, expected);
 }
@@ -1530,14 +1539,14 @@ fn stats_output_ssv_delimited() {
     wrk.assert_success(&mut cmd);
 
     let got = std::fs::read_to_string(out_file).unwrap();
-    let expected = r#"field;type;is_ascii;sum;min;max;range;sort_order;min_length;max_length;sum_length;avg_length;stddev_length;variance_length;cv_length;mean;sem;geometric_mean;harmonic_mean;stddev;variance;cv;nullcount;max_precision;sparsity;qsv__value
-col1;Integer;;15;1;5;4;Ascending;;;;;;;;3;0.6325;2.6052;2.1898;1.4142;2;47.1405;0;;0;
-col2;Integer;;10644;0;4321;4321;Descending;;;;;;;;2128.8;685.6979;0;;1533.267;2350907.76;72.0249;0;;0;
-col3;String;true;;01;10;;Ascending;2;2;10;2;0;0;0;;;;;;;;0;;0;
-qsv__rowcount;;;;;;;;;;;;;;;;;;;;;;;;;5
-qsv__columncount;;;;;;;;;;;;;;;;;;;;;;;;;3
-qsv__filesize_bytes;;;;;;;;;;;;;;;;;;;;;;;;;62
-qsv__fingerprint_hash;;;;;;;;;;;;;;;;;;;;;;;;;09e5437cc0b15ac327d13792b139d7fb2277e1e1b7851a2cb6ba92274e5261c4
+    let expected = r#"field;type;is_ascii;sum;min;max;range;sort_order;sortiness;min_length;max_length;sum_length;avg_length;stddev_length;variance_length;cv_length;mean;sem;geometric_mean;harmonic_mean;stddev;variance;cv;nullcount;max_precision;sparsity;qsv__value
+col1;Integer;;15;1;5;4;Ascending;1;;;;;;;;3;0.6325;2.6052;2.1898;1.4142;2;47.1405;0;;0;
+col2;Integer;;10644;0;4321;4321;Descending;-1;;;;;;;;2128.8;685.6979;0;;1533.267;2350907.76;72.0249;0;;0;
+col3;String;true;;01;10;;Ascending;1;2;2;10;2;0;0;0;;;;;;;;0;;0;
+qsv__rowcount;;;;;;;;;;;;;;;;;;;;;;;;;;5
+qsv__columncount;;;;;;;;;;;;;;;;;;;;;;;;;;3
+qsv__filesize_bytes;;;;;;;;;;;;;;;;;;;;;;;;;;62
+qsv__fingerprint_hash;;;;;;;;;;;;;;;;;;;;;;;;;;cf6ba47ee7251a17b8d7789636b2c5d2302ea6e0fb69572f1eaf25aead083cef
 "#;
     assert_eq!(got, expected);
 }
@@ -1569,14 +1578,14 @@ fn stats_output_csvsz_delimited() {
     cmd.arg("decompress").arg(out_file.clone());
 
     let got: String = wrk.stdout(&mut cmd);
-    let expected = r#"field,type,is_ascii,sum,min,max,range,sort_order,min_length,max_length,sum_length,avg_length,stddev_length,variance_length,cv_length,mean,sem,geometric_mean,harmonic_mean,stddev,variance,cv,nullcount,max_precision,sparsity,qsv__value
-col1,Integer,,15,1,5,4,Ascending,,,,,,,,3,0.6325,2.6052,2.1898,1.4142,2,47.1405,0,,0,
-col2,Integer,,10644,0,4321,4321,Descending,,,,,,,,2128.8,685.6979,0,,1533.267,2350907.76,72.0249,0,,0,
-col3,String,true,,01,10,,Ascending,2,2,10,2,0,0,0,,,,,,,,0,,0,
-qsv__rowcount,,,,,,,,,,,,,,,,,,,,,,,,,5
-qsv__columncount,,,,,,,,,,,,,,,,,,,,,,,,,3
-qsv__filesize_bytes,,,,,,,,,,,,,,,,,,,,,,,,,62
-qsv__fingerprint_hash,,,,,,,,,,,,,,,,,,,,,,,,,09e5437cc0b15ac327d13792b139d7fb2277e1e1b7851a2cb6ba92274e5261c4"#;
+    let expected = r#"field,type,is_ascii,sum,min,max,range,sort_order,sortiness,min_length,max_length,sum_length,avg_length,stddev_length,variance_length,cv_length,mean,sem,geometric_mean,harmonic_mean,stddev,variance,cv,nullcount,max_precision,sparsity,qsv__value
+col1,Integer,,15,1,5,4,Ascending,1,,,,,,,,3,0.6325,2.6052,2.1898,1.4142,2,47.1405,0,,0,
+col2,Integer,,10644,0,4321,4321,Descending,-1,,,,,,,,2128.8,685.6979,0,,1533.267,2350907.76,72.0249,0,,0,
+col3,String,true,,01,10,,Ascending,1,2,2,10,2,0,0,0,,,,,,,,0,,0,
+qsv__rowcount,,,,,,,,,,,,,,,,,,,,,,,,,,5
+qsv__columncount,,,,,,,,,,,,,,,,,,,,,,,,,,3
+qsv__filesize_bytes,,,,,,,,,,,,,,,,,,,,,,,,,,62
+qsv__fingerprint_hash,,,,,,,,,,,,,,,,,,,,,,,,,,cf6ba47ee7251a17b8d7789636b2c5d2302ea6e0fb69572f1eaf25aead083cef"#;
     assert_eq!(got, expected);
 }