Repository: tobymao/sqlglot

Issue:
Support additional duckdb data types
I've started work to move our type parser in ibis from parsy to sqlglot for performance reasons: sqlglot is much much faster.

sqlglot is missing a few types that we previously supported (parsing metadata that come back from DuckDB's `DESCRIBE` directive) and while these aren't blocks because they are simple to intercept, it would be nice if sqlglot supported some or all of these.

**Is your feature request related to a problem? Please describe.**

Not related to a problem.

Here's the list of types that are currently supported by ibis and potentially returned from duckdb's `DESCRIBE` that aren't supported in sqlglot:

- `INT128`
- `HUGEINT`
- `NUMERIC` with no precision or scale arguments
- `TIMESTAMP_SEC`
- `TIMESTAMP_S`
- `TIMESTAMP_MS`
- `TIMESTAMP_US`
- `TIMESTAMP_NS`
- `TIMESTAMP_TZ`

**Describe the solution you'd like**

- `INT128`, `HUGEINT` -> `DataType.Type.INT128`
- `NUMERIC` -> `DECIMAL(18, 3)` DuckDB's default precision and scale
- `TIMESTAMP_*` -> ? not sure if these have an exact sqlglot equivalent

**Describe alternatives you've considered**

I've implemented the only viable alternative I can think of, which is to handle these strings before passing the text to sqlglot.

**Additional context**

NA


Discussion:
> I've started work to move our type parser in ibis from parsy to sqlglot for performance reasons: sqlglot is much much faster.

That's great! :-)

I'll have a look and let you know once there's an update.

Resolving patch:
diff --git a/sqlglot/dialects/clickhouse.py b/sqlglot/dialects/clickhouse.py
--- a/sqlglot/dialects/clickhouse.py
+++ b/sqlglot/dialects/clickhouse.py
@@ -41,7 +41,6 @@ class Tokenizer(tokens.Tokenizer):
             "FLOAT32": TokenType.FLOAT,
             "FLOAT64": TokenType.DOUBLE,
             "GLOBAL": TokenType.GLOBAL,
-            "INT128": TokenType.INT128,
             "INT16": TokenType.SMALLINT,
             "INT256": TokenType.INT256,
             "INT32": TokenType.INT,
diff --git a/sqlglot/dialects/duckdb.py b/sqlglot/dialects/duckdb.py
--- a/sqlglot/dialects/duckdb.py
+++ b/sqlglot/dialects/duckdb.py
@@ -110,14 +110,14 @@ class Tokenizer(tokens.Tokenizer):
             "//": TokenType.DIV,
             "ATTACH": TokenType.COMMAND,
             "BINARY": TokenType.VARBINARY,
-            "BPCHAR": TokenType.TEXT,
             "BITSTRING": TokenType.BIT,
+            "BPCHAR": TokenType.TEXT,
             "CHAR": TokenType.TEXT,
             "CHARACTER VARYING": TokenType.TEXT,
             "EXCLUDE": TokenType.EXCEPT,
+            "HUGEINT": TokenType.INT128,
             "INT1": TokenType.TINYINT,
             "LOGICAL": TokenType.BOOLEAN,
-            "NUMERIC": TokenType.DOUBLE,
             "PIVOT_WIDER": TokenType.PIVOT,
             "SIGNED": TokenType.INT,
             "STRING": TokenType.VARCHAR,
@@ -186,6 +186,22 @@ class Parser(parser.Parser):
             TokenType.UTINYINT,
         }
 
+        def _parse_types(
+            self, check_func: bool = False, schema: bool = False
+        ) -> t.Optional[exp.Expression]:
+            this = super()._parse_types(check_func=check_func, schema=schema)
+
+            # DuckDB treats NUMERIC and DECIMAL without precision as DECIMAL(18, 3)
+            # See: https://duckdb.org/docs/sql/data_types/numeric
+            if (
+                isinstance(this, exp.DataType)
+                and this.is_type("numeric", "decimal")
+                and not this.expressions
+            ):
+                return exp.DataType.build("DECIMAL(18, 3)")
+
+            return this
+
         def _pivot_column_names(self, aggregations: t.List[exp.Expression]) -> t.List[str]:
             if len(aggregations) == 1:
                 return super()._pivot_column_names(aggregations)
diff --git a/sqlglot/tokens.py b/sqlglot/tokens.py
--- a/sqlglot/tokens.py
+++ b/sqlglot/tokens.py
@@ -659,6 +659,7 @@ class Tokenizer(metaclass=_Tokenizer):
         "TINYINT": TokenType.TINYINT,
         "SHORT": TokenType.SMALLINT,
         "SMALLINT": TokenType.SMALLINT,
+        "INT128": TokenType.INT128,
         "INT2": TokenType.SMALLINT,
         "INTEGER": TokenType.INT,
         "INT": TokenType.INT,


Test patch:
diff --git a/tests/dialects/test_duckdb.py b/tests/dialects/test_duckdb.py
--- a/tests/dialects/test_duckdb.py
+++ b/tests/dialects/test_duckdb.py
@@ -497,8 +497,12 @@ def test_cast(self):
         self.validate_identity("CAST(x AS USMALLINT)")
         self.validate_identity("CAST(x AS UTINYINT)")
         self.validate_identity("CAST(x AS TEXT)")
+        self.validate_identity("CAST(x AS INT128)")
+        self.validate_identity("CAST(x AS DOUBLE)")
+        self.validate_identity("CAST(x AS DECIMAL(15, 4))")
 
-        self.validate_all("CAST(x AS NUMERIC)", write={"duckdb": "CAST(x AS DOUBLE)"})
+        self.validate_all("CAST(x AS NUMERIC(1, 2))", write={"duckdb": "CAST(x AS DECIMAL(1, 2))"})
+        self.validate_all("CAST(x AS HUGEINT)", write={"duckdb": "CAST(x AS INT128)"})
         self.validate_all("CAST(x AS CHAR)", write={"duckdb": "CAST(x AS TEXT)"})
         self.validate_all("CAST(x AS BPCHAR)", write={"duckdb": "CAST(x AS TEXT)"})
         self.validate_all("CAST(x AS STRING)", write={"duckdb": "CAST(x AS TEXT)"})
@@ -513,6 +517,20 @@ def test_cast(self):
         self.validate_all("CAST(x AS BINARY)", write={"duckdb": "CAST(x AS BLOB)"})
         self.validate_all("CAST(x AS VARBINARY)", write={"duckdb": "CAST(x AS BLOB)"})
         self.validate_all("CAST(x AS LOGICAL)", write={"duckdb": "CAST(x AS BOOLEAN)"})
+        self.validate_all(
+            "CAST(x AS NUMERIC)",
+            write={
+                "duckdb": "CAST(x AS DECIMAL(18, 3))",
+                "postgres": "CAST(x AS DECIMAL(18, 3))",
+            },
+        )
+        self.validate_all(
+            "CAST(x AS DECIMAL)",
+            write={
+                "duckdb": "CAST(x AS DECIMAL(18, 3))",
+                "postgres": "CAST(x AS DECIMAL(18, 3))",
+            },
+        )
         self.validate_all(
             "CAST(x AS BIT)",
             read={