Repository: roapi/roapi (language: python)

Issue:
Unsupported datatype: Date32
I had an error when querying the data that had the `Date32` type. I checked DataFusion library but it works on Date32.

thread 'actix-rt|system:0|arbiter:5' panicked at 'Unsupported datatype: Date32', /Users/runner/.cargo/git/checkouts/arrow-7d34e669e95701bb/f7cf157/rust/arrow/src/json/writer.rs:326:13

Reproduce:
Config:
```yml
addr: 0.0.0.0:8080
tables:
- name: han_shop
  option:
    format: csv
  uri: ../examples.csv
```
SQL query:
```sql
SELECT name FROM example
```

CSV data:
```csv
period,name,
2021-03-01,Linh,
2021-03-01,Han,
```

Schema:
```json
{
    "name": "period",
    "data_type": "Date32",
    "nullable": false,
    "dict_id": 0,
    "dict_is_ordered": false
}
```

Command:
```bash
roapi-http -c ./config-1615885382668.yml
```


Discussion / hints:
I was about to ask the same question, I'm trying to load a Parquet file with a Date field, same error
Ha, good catch, this should be an easy fix. I didn't implement date32 in the json serializer because I wasn't sure what's the best json value to serialize into. For example, we can serialize it into a number or a string. Returning a number (count of days) is the more efficient and robust behavior, but would make it a little bit less readable for human, which I guess is not too big of a problem :)
I think it should return a number so that it would be easy to query :")

Fix patch:
diff --git a/Cargo.lock b/Cargo.lock
index 04e25d6a1..f54fcdd16 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -42,7 +42,7 @@ dependencies = [
  "actix-service",
  "actix-tls",
  "actix-utils",
- "ahash 0.7.2",
+ "ahash",
  "base64 0.13.0",
  "bitflags",
  "brotli2",
@@ -185,7 +185,7 @@ dependencies = [
  "actix-service",
  "actix-utils",
  "actix-web-codegen",
- "ahash 0.7.2",
+ "ahash",
  "awc",
  "bytes",
  "derive_more",
@@ -223,12 +223,6 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
-[[package]]
-name = "ahash"
-version = "0.4.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e"
-
 [[package]]
 name = "ahash"
 version = "0.7.2"
@@ -273,6 +267,15 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "ansi_term"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.38"
@@ -294,7 +297,6 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
 [[package]]
 name = "arrow"
 version = "4.0.0-SNAPSHOT"
-source = "git+https://github.com/houqp/arrow.git?rev=cfec7bc607b9b564eceb352e54b21fa005a3d606#cfec7bc607b9b564eceb352e54b21fa005a3d606"
 dependencies = [
  "cfg_aliases",
  "chrono",
@@ -545,7 +547,7 @@ version = "2.33.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002"
 dependencies = [
- "ansi_term",
+ "ansi_term 0.11.0",
  "atty",
  "bitflags",
  "strsim 0.8.0",
@@ -741,18 +743,27 @@ dependencies = [
  "sct",
 ]
 
+[[package]]
+name = "ctor"
+version = "0.1.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e98e2ad1a782e33928b96fc3948e7c355e5af34ba4de7670fe8bac2a3b2006d"
+dependencies = [
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "datafusion"
 version = "4.0.0-SNAPSHOT"
-source = "git+https://github.com/houqp/arrow.git?rev=cfec7bc607b9b564eceb352e54b21fa005a3d606#cfec7bc607b9b564eceb352e54b21fa005a3d606"
 dependencies = [
- "ahash 0.7.2",
+ "ahash",
  "arrow",
  "async-trait",
  "chrono",
  "clap 2.33.3",
  "futures",
- "hashbrown",
+ "hashbrown 0.11.2",
  "lazy_static",
  "log",
  "md-5",
@@ -764,7 +775,8 @@ dependencies = [
  "regex",
  "rustyline 7.1.0",
  "sha2",
- "sqlparser 0.8.0",
+ "smallvec",
+ "sqlparser 0.9.0",
  "tokio",
  "tokio-stream",
  "unicode-segmentation",
@@ -814,6 +826,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "diff"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499"
+
 [[package]]
 name = "digest"
 version = "0.9.0"
@@ -1170,8 +1188,14 @@ name = "hashbrown"
 version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
+
+[[package]]
+name = "hashbrown"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
 dependencies = [
- "ahash 0.4.7",
+ "ahash",
 ]
 
 [[package]]
@@ -1319,7 +1343,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3"
 dependencies = [
  "autocfg",
- "hashbrown",
+ "hashbrown 0.9.1",
 ]
 
 [[package]]
@@ -1731,6 +1755,15 @@ version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "afb2e1c3ee07430c2cf76151675e583e0f19985fa6efae47d6848a3e2c824f85"
 
+[[package]]
+name = "output_vt100"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53cdc5b785b7a58c5aad8216b3dfa114df64b0b06ae6e1501cef91df2fbdf8f9"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "packed_simd_2"
 version = "0.3.4"
@@ -1769,7 +1802,6 @@ dependencies = [
 [[package]]
 name = "parquet"
 version = "4.0.0-SNAPSHOT"
-source = "git+https://github.com/houqp/arrow.git?rev=cfec7bc607b9b564eceb352e54b21fa005a3d606#cfec7bc607b9b564eceb352e54b21fa005a3d606"
 dependencies = [
  "arrow",
  "base64 0.12.3",
@@ -1850,6 +1882,18 @@ version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
 
+[[package]]
+name = "pretty_assertions"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f297542c27a7df8d45de2b0e620308ab883ad232d06c14b76ac3e144bda50184"
+dependencies = [
+ "ansi_term 0.12.1",
+ "ctor",
+ "diff",
+ "output_vt100",
+]
+
 [[package]]
 name = "prettytable-rs"
 version = "0.8.0"
@@ -2182,6 +2226,7 @@ dependencies = [
  "columnq",
  "env_logger",
  "log",
+ "pretty_assertions",
  "serde",
  "serde_derive",
  "serde_json",
@@ -2612,9 +2657,9 @@ dependencies = [
 
 [[package]]
 name = "sqlparser"
-version = "0.8.0"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b929a9577d01ee7cddb820696ed536868d81557d3c14363a7514d71add1d058"
+checksum = "4fa863a2dfc4879a35647c51dadf495a2ad53745eaf3723fda27006e745fb0ba"
 dependencies = [
  "log",
 ]
@@ -3302,18 +3347,18 @@ checksum = "81a974bcdd357f0dca4d41677db03436324d45a4c9ed2d0b873a5a360ce41c36"
 
 [[package]]
 name = "zstd"
-version = "0.6.1+zstd.1.4.9"
+version = "0.7.0+zstd.1.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5de55e77f798f205d8561b8fe2ef57abfb6e0ff2abe7fd3c089e119cdb5631a3"
+checksum = "9428752481d8372e15b1bf779ea518a179ad6c771cca2d2c60e4fbff3cc2cd52"
 dependencies = [
  "zstd-safe",
 ]
 
 [[package]]
 name = "zstd-safe"
-version = "3.0.1+zstd.1.4.9"
+version = "3.1.0+zstd.1.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1387cabcd938127b30ce78c4bf00b30387dddf704e3f0881dbc4ff62b5566f8c"
+checksum = "5aa1926623ad7fe406e090555387daf73db555b948134b4d73eac5eb08fb666d"
 dependencies = [
  "libc",
  "zstd-sys",
@@ -3321,9 +3366,9 @@ dependencies = [
 
 [[package]]
 name = "zstd-sys"
-version = "1.4.20+zstd.1.4.9"
+version = "1.5.0+zstd.1.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebd5b733d7cf2d9447e2c3e76a5589b4f5e5ae065c22a2bc0b023cbc331b6c8e"
+checksum = "4e6c094340240369025fc6b731b054ee2a834328fa584310ac96aa4baebdc465"
 dependencies = [
  "cc",
  "libc",
diff --git a/Cargo.toml b/Cargo.toml
index 09ee1fe21..455f16375 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -6,14 +6,14 @@ members = [
 ]
 
 [patch."https://github.com/apache/arrow.git"]
-arrow = { git = "https://github.com/houqp/arrow.git", rev = "cfec7bc607b9b564eceb352e54b21fa005a3d606" }
-parquet = { git = "https://github.com/houqp/arrow.git", rev = "cfec7bc607b9b564eceb352e54b21fa005a3d606" }
-datafusion = { git = "https://github.com/houqp/arrow.git", rev = "cfec7bc607b9b564eceb352e54b21fa005a3d606" }
+arrow = { git = "https://github.com/houqp/arrow.git", rev = "88394d00b91365770e7b791a34b20bd849d678ee" }
+parquet = { git = "https://github.com/houqp/arrow.git", rev = "88394d00b91365770e7b791a34b20bd849d678ee" }
+datafusion = { git = "https://github.com/houqp/arrow.git", rev = "88394d00b91365770e7b791a34b20bd849d678ee" }
 
 [patch.crates-io]
-arrow = { git = "https://github.com/houqp/arrow.git", rev = "cfec7bc607b9b564eceb352e54b21fa005a3d606" }
-parquet = { git = "https://github.com/houqp/arrow.git", rev = "cfec7bc607b9b564eceb352e54b21fa005a3d606" }
-datafusion = { git = "https://github.com/houqp/arrow.git", rev = "cfec7bc607b9b564eceb352e54b21fa005a3d606" }
+arrow = { git = "https://github.com/houqp/arrow.git", rev = "88394d00b91365770e7b791a34b20bd849d678ee" }
+parquet = { git = "https://github.com/houqp/arrow.git", rev = "88394d00b91365770e7b791a34b20bd849d678ee" }
+datafusion = { git = "https://github.com/houqp/arrow.git", rev = "88394d00b91365770e7b791a34b20bd849d678ee" }
 
 actix-cors = { git = "https://github.com/houqp/actix-extras.git", rev = "ab3bdb6a5924b6d881d204856199e7539e273d2f" }
 deltalake = { git = "https://github.com/delta-io/delta-rs.git", rev = "38be44cdc8e330a21d9ba31017c29429c2082411" }
diff --git a/columnq/src/columnq.rs b/columnq/src/columnq.rs
index 855b379b2..14265844f 100644
--- a/columnq/src/columnq.rs
+++ b/columnq/src/columnq.rs
@@ -22,7 +22,8 @@ impl ColumnQ {
     pub async fn load_table(&mut self, t: &TableSource) -> Result<(), ColumnQError> {
         let table = table::load(&t).await?;
         self.schema_map.insert(t.name.clone(), table.schema());
-        self.dfctx.register_table(&t.name, Arc::new(table));
+        self.dfctx
+            .register_table(t.name.as_str(), Arc::new(table))?;
 
         Ok(())
     }
diff --git a/columnq/src/query/graphql.rs b/columnq/src/query/graphql.rs
index 45b83f308..25475acad 100644
--- a/columnq/src/query/graphql.rs
+++ b/columnq/src/query/graphql.rs
@@ -257,7 +257,7 @@ pub fn query_to_df(
     let field = field.ok_or_else(|| invalid_query("field not found in selection".to_string()))?;
 
     let mut df = dfctx
-        .table(&field.name)
+        .table(field.name)
         .map_err(|e| QueryError::invalid_table(e, &field.name))?;
 
     // apply projection
@@ -297,7 +297,7 @@ pub fn query_to_df(
             "sort" => match value {
                 Value::List(sort_options) => {
                     df = df
-                        .sort(&to_datafusion_sort_columns(sort_options)?)
+                        .sort(to_datafusion_sort_columns(sort_options)?)
                         .map_err(QueryError::invalid_sort)?;
                 }
                 other => {
@@ -379,7 +379,7 @@ mod tests {
 
         let expected_df = dfctx
             .table("properties")?
-            .select(&[col("Address"), col("Bed"), col("Bath")])?
+            .select(vec![col("Address"), col("Bed"), col("Bath")])?
             .filter(col("Bath").gt_eq(lit(2i64)))?
             .filter(col("Bed").gt(lit(3i64)))?;
 
diff --git a/columnq/src/query/rest.rs b/columnq/src/query/rest.rs
index da25035d4..efbfd22b1 100644
--- a/columnq/src/query/rest.rs
+++ b/columnq/src/query/rest.rs
@@ -87,7 +87,7 @@ pub fn table_query_to_df(
                         _ => column_sort_expr_asc(val.to_string()),
                     })
                     .collect::<Vec<_>>();
-                df = df.sort(&sort_exprs).map_err(QueryError::invalid_sort)?;
+                df = df.sort(sort_exprs).map_err(QueryError::invalid_sort)?;
             }
             // limit=100
             "limit" => {
diff --git a/roapi-http/Cargo.toml b/roapi-http/Cargo.toml
index 99a585a3e..efa9c9460 100644
--- a/roapi-http/Cargo.toml
+++ b/roapi-http/Cargo.toml
@@ -38,3 +38,4 @@ simd = ["columnq/simd"]
 
 [dev-dependencies]
 actix-rt = "*"
+pretty_assertions = "*"
diff --git a/roapi-http/src/encoding/json.rs b/roapi-http/src/encoding/json.rs
index 238311678..3c645ae77 100644
--- a/roapi-http/src/encoding/json.rs
+++ b/roapi-http/src/encoding/json.rs
@@ -4,3 +4,153 @@ pub fn record_batches_to_bytes(
     let json_rows = arrow::json::writer::record_batches_to_json_rows(&batches);
     serde_json::to_vec(&json_rows)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::sync::Arc;
+
+    use arrow::array::*;
+    use arrow::datatypes::*;
+    use arrow::record_batch::*;
+    use pretty_assertions::assert_eq;
+
+    #[test]
+    fn serialize_date_columns() {
+        let schema = Schema::new(vec![
+            Field::new("d32", DataType::Date32, false),
+            Field::new("d64", DataType::Date64, false),
+        ]);
+
+        let record_batch = RecordBatch::try_new(
+            Arc::new(schema),
+            vec![
+                Arc::new(Date32Array::from(vec![1, 18729])),
+                Arc::new(Date64Array::from(vec![1, 1618200268000])),
+            ],
+        )
+        .unwrap();
+
+        let data = record_batches_to_bytes(&[record_batch]).unwrap();
+
+        assert_eq!(
+            std::str::from_utf8(&data).unwrap(),
+            serde_json::json!([
+                {
+                    "d32": "1970-01-02",
+                    "d64": "1970-01-01",
+                },
+                {
+                    "d32": "2021-04-12",
+                    "d64": "2021-04-12",
+                },
+            ])
+            .to_string(),
+        )
+    }
+
+    #[test]
+    fn serialize_timestamp_columns() {
+        let schema = Schema::new(vec![
+            Field::new("sec", DataType::Timestamp(TimeUnit::Second, None), false),
+            Field::new(
+                "msec",
+                DataType::Timestamp(TimeUnit::Millisecond, None),
+                false,
+            ),
+            Field::new(
+                "usec",
+                DataType::Timestamp(TimeUnit::Microsecond, None),
+                false,
+            ),
+            Field::new(
+                "nsec",
+                DataType::Timestamp(TimeUnit::Nanosecond, None),
+                false,
+            ),
+        ]);
+
+        let record_batch = RecordBatch::try_new(
+            Arc::new(schema),
+            vec![
+                Arc::new(TimestampSecondArray::from(vec![1618200268, 1620792268])),
+                Arc::new(TimestampMillisecondArray::from(vec![
+                    1618200268000,
+                    1620792268001,
+                ])),
+                Arc::new(TimestampMicrosecondArray::from(vec![
+                    1618200268000000,
+                    1620792268000002,
+                ])),
+                Arc::new(TimestampNanosecondArray::from(vec![
+                    1618200268000000000,
+                    1620792268000000003,
+                ])),
+            ],
+        )
+        .unwrap();
+
+        let data = record_batches_to_bytes(&[record_batch]).unwrap();
+
+        assert_eq!(
+            std::str::from_utf8(&data).unwrap(),
+            serde_json::json!([
+                {
+                    "sec": "2021-04-12 04:04:28",
+                    "msec": "2021-04-12 04:04:28",
+                    "usec": "2021-04-12 04:04:28",
+                    "nsec": "2021-04-12 04:04:28",
+                },
+                {
+                    "sec": "2021-05-12 04:04:28",
+                    "msec": "2021-05-12 04:04:28.001",
+                    "usec": "2021-05-12 04:04:28.000002",
+                    "nsec": "2021-05-12 04:04:28.000000003",
+                }
+            ])
+            .to_string(),
+        )
+    }
+
+    #[test]
+    fn serialize_time_columns() {
+        let schema = Schema::new(vec![
+            Field::new("t32sec", DataType::Time32(TimeUnit::Second), false),
+            Field::new("t32msec", DataType::Time32(TimeUnit::Millisecond), false),
+            Field::new("t64usec", DataType::Time64(TimeUnit::Microsecond), false),
+            Field::new("t64nsec", DataType::Time64(TimeUnit::Nanosecond), false),
+        ]);
+
+        let record_batch = RecordBatch::try_new(
+            Arc::new(schema),
+            vec![
+                Arc::new(Time32SecondArray::from(vec![1, 120])),
+                Arc::new(Time32MillisecondArray::from(vec![1, 120])),
+                Arc::new(Time64MicrosecondArray::from(vec![1, 120])),
+                Arc::new(Time64NanosecondArray::from(vec![1, 120])),
+            ],
+        )
+        .unwrap();
+
+        let data = record_batches_to_bytes(&[record_batch]).unwrap();
+
+        assert_eq!(
+            std::str::from_utf8(&data).unwrap(),
+            serde_json::json!([
+                {
+                    "t32sec": "00:00:01",
+                    "t32msec": "00:00:00.001",
+                    "t64usec": "00:00:00.000001",
+                    "t64nsec": "00:00:00.000000001",
+                },
+                {
+                    "t32sec": "00:02:00",
+                    "t32msec": "00:00:00.120",
+                    "t64usec": "00:00:00.000120",
+                    "t64nsec": "00:00:00.000000120",
+                },
+            ])
+            .to_string()
+        )
+    }
+}


Test patch:
diff --git a/columnq/src/test_util.rs b/columnq/src/test_util.rs
index cdd6daa2e..7cd16e19a 100644
--- a/columnq/src/test_util.rs
+++ b/columnq/src/test_util.rs
@@ -125,12 +125,12 @@ schema:
 }
 
 pub fn register_table_properties(dfctx: &mut ExecutionContext) -> anyhow::Result<()> {
-    dfctx.register_table("properties", Arc::new(properties_table()?));
+    dfctx.register_table("properties", Arc::new(properties_table()?))?;
     Ok(())
 }
 
 pub async fn register_table_ubuntu_ami(dfctx: &mut ExecutionContext) -> anyhow::Result<()> {
-    dfctx.register_table("ubuntu_ami", Arc::new(ubuntu_ami_table().await?));
+    dfctx.register_table("ubuntu_ami", Arc::new(ubuntu_ami_table().await?))?;
     Ok(())
 }