Repository: tobymao/sqlglot (2024-09-16 ~ 2024-09-23)

problem_statement: Optimizer Misinterprets Union Identifiers as Columns in DuckDB Queries
When testing the creation of a union with DuckDB, it appears that sqlglot mistakenly treats the union identifier as a column rather than recognizing it as a general identifier when the statement is qualified in the optimizer.

DuckDB uses identifiers to tag unions, as described in [official documentation](https://duckdb.org/docs/sql/data_types/union)

Here is how to reproduce the exception:

```python
import sqlglot
import sqlglot.optimizer

statement = sqlglot.parse_one("select union_value(k1 := 1)", dialect="duckdb")
sqlglot.optimizer.optimize(statement).sql(dialect='duckdb')
```

In this case, `k1` is simply an identifier—just like a key in a struct. It would be helpful if it could be handled in a case-insensitive manner, similar to other struct keys.

The exception raised is as follows:

```
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/Users/rusty/Development/flight-cloud/.venv/lib/python3.12/site-packages/sqlglot/optimizer/optimizer.py", line 92, in optimize
    optimized = rule(optimized, **rule_kwargs)
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/rusty/Development/flight-cloud/.venv/lib/python3.12/site-packages/sqlglot/optimizer/qualify.py", line 99, in qualify
    validate_qualify_columns_func(expression)
  File "/Users/rusty/Development/flight-cloud/.venv/lib/python3.12/site-packages/sqlglot/optimizer/qualify_columns.py", line 106, in validate_qualify_columns
    raise OptimizeError(f"Column '{column}' could not be resolved{for_table}")
sqlglot.errors.OptimizeError: Column 'k1' could not be resolved
```

In contrast, DuckDB handles this case correctly, as shown below:

```sql
v1.1.0 fa5c2fe15f
Enter ".help" for usage hints.
Connected to a transient in-memory database.
Use ".open FILENAME" to reopen on a persistent database.
D select union_value(k1 := 1);
┌──────────────────────┐
│ union_value(k1 := 1) │
│  union(k1 integer)   │
├──────────────────────┤
│ 1                    │
└──────────────────────┘
```

Thank you for looking into this.

This behavior in DuckDB demonstrates that identifiers like `k1` should be recognized as keys in a struct rather than treated as columns. Handling this more gracefully in `sqlglot` would align with DuckDB’s behavior.
date: 2024-09-16
patch: diff --git a/sqlglot/parser.py b/sqlglot/parser.py
index a47ca8ebb..a21803573 100644
--- a/sqlglot/parser.py
+++ b/sqlglot/parser.py
@@ -4343,6 +4343,9 @@ class Parser(metaclass=_Parser):
             )
 
         while self._match_set(self.ASSIGNMENT):
+            if isinstance(this, exp.Column) and len(this.parts) == 1:
+                this = this.this
+
             this = self.expression(
                 self.ASSIGNMENT[self._prev.token_type],
                 this=this,

test_patch: diff --git a/tests/dialects/test_bigquery.py b/tests/dialects/test_bigquery.py
index b8838962c0..e2adfeabce 100644
--- a/tests/dialects/test_bigquery.py
+++ b/tests/dialects/test_bigquery.py
@@ -22,6 +22,8 @@ class TestBigQuery(Validator):
     maxDiff = None
 
     def test_bigquery(self):
+        self.validate_identity("REGEXP_EXTRACT(x, '(?<)')")
+
         self.validate_all(
             "EXTRACT(HOUR FROM DATETIME(2008, 12, 25, 15, 30, 00))",
             write={
@@ -1502,6 +1504,8 @@ WHERE
             },
         )
 
+        self.validate_identity("SELECT * FROM a-b c", "SELECT * FROM a-b AS c")
+
     def test_errors(self):
         with self.assertRaises(TokenError):
             transpile("'\\'", read="bigquery")
@@ -1958,3 +1962,26 @@ OPTIONS (
                 "duckdb": "WITH Races AS (SELECT '800M' AS race) SELECT race, participant FROM Races AS r CROSS JOIN (SELECT UNNEST([{'name': 'Rudisha', 'laps': [23.4, 26.3, 26.4, 26.1]}], max_depth => 2)) AS participant",
             },
         )
+
+    def test_range_type(self):
+        for type, value in (
+            ("RANGE<DATE>", "'[2020-01-01, 2020-12-31)'"),
+            ("RANGE<DATE>", "'[UNBOUNDED, 2020-12-31)'"),
+            ("RANGE<DATETIME>", "'[2020-01-01 12:00:00, 2020-12-31 12:00:00)'"),
+            ("RANGE<TIMESTAMP>", "'[2020-10-01 12:00:00+08, 2020-12-31 12:00:00+08)'"),
+        ):
+            with self.subTest(f"Testing BigQuery's RANGE<T> type: {type} {value}"):
+                self.validate_identity(f"SELECT {type} {value}", f"SELECT CAST({value} AS {type})")
+
+                self.assertEqual(self.parse_one(type), exp.DataType.build(type, dialect="bigquery"))
+
+        self.validate_identity(
+            "SELECT RANGE(CAST('2022-12-01' AS DATE), CAST('2022-12-31' AS DATE))"
+        )
+        self.validate_identity("SELECT RANGE(NULL, CAST('2022-12-31' AS DATE))")
+        self.validate_identity(
+            "SELECT RANGE(CAST('2022-10-01 14:53:27' AS DATETIME), CAST('2022-10-01 16:00:00' AS DATETIME))"
+        )
+        self.validate_identity(
+            "SELECT RANGE(CAST('2022-10-01 14:53:27 America/Los_Angeles' AS TIMESTAMP), CAST('2022-10-01 16:00:00 America/Los_Angeles' AS TIMESTAMP))"
+        )
diff --git a/tests/dialects/test_clickhouse.py b/tests/dialects/test_clickhouse.py
index d203141dfa..aa6c4f1b65 100644
--- a/tests/dialects/test_clickhouse.py
+++ b/tests/dialects/test_clickhouse.py
@@ -519,6 +519,14 @@ class TestClickhouse(Validator):
         self.validate_identity("SELECT TRIM(LEADING '(' FROM '(   Hello, world!   )')")
         self.validate_identity("current_timestamp").assert_is(exp.Column)
 
+        self.validate_identity("SELECT * APPLY(sum) FROM columns_transformers")
+        self.validate_identity("SELECT COLUMNS('[jk]') APPLY(toString) FROM columns_transformers")
+        self.validate_identity(
+            "SELECT COLUMNS('[jk]') APPLY(toString) APPLY(length) APPLY(max) FROM columns_transformers"
+        )
+        self.validate_identity("SELECT * APPLY(sum), COLUMNS('col') APPLY(sum) APPLY(avg) FROM t")
+        self.validate_identity("SELECT col apply", "SELECT col AS apply")
+
     def test_clickhouse_values(self):
         values = exp.select("*").from_(
             exp.values([exp.tuple_(1, 2, 3)], alias="subq", columns=["a", "b", "c"])
diff --git a/tests/dialects/test_duckdb.py b/tests/dialects/test_duckdb.py
index 2887dd0f1d..ad503ae955 100644
--- a/tests/dialects/test_duckdb.py
+++ b/tests/dialects/test_duckdb.py
@@ -295,6 +295,9 @@ class TestDuckDB(Validator):
         self.validate_identity("SUMMARIZE tbl").assert_is(exp.Summarize)
         self.validate_identity("SUMMARIZE SELECT * FROM tbl").assert_is(exp.Summarize)
         self.validate_identity("CREATE TABLE tbl_summary AS SELECT * FROM (SUMMARIZE tbl)")
+        self.validate_identity("UNION_VALUE(k1 := 1)").find(exp.PropertyEQ).this.assert_is(
+            exp.Identifier
+        )
         self.validate_identity(
             "SELECT species, island, COUNT(*) FROM t GROUP BY GROUPING SETS (species), GROUPING SETS (island)"
         )


----

problem_statement: Clickhouse COLUMNS APPLY parsing error
I was trying to use dynamic columns selection with APPLY, but sqlglot fails to parse it

**Fully reproducible code snippet**

`select COLUMNS('.*field') APPLY(toUInt64) FROM ABC;`

Leads to sqlglot.errors.ParseError: Invalid expression / Unexpected token.

**Official Documentation**
https://clickhouse.com/docs/en/sql-reference/statements/select#apply


date: 2024-09-20
patch: diff --git a/sqlglot/dialects/clickhouse.py b/sqlglot/dialects/clickhouse.py
index ec1984e94..c8671f690 100644
--- a/sqlglot/dialects/clickhouse.py
+++ b/sqlglot/dialects/clickhouse.py
@@ -756,6 +756,26 @@ class ClickHouse(Dialect):
         def _parse_constraint(self) -> t.Optional[exp.Expression]:
             return super()._parse_constraint() or self._parse_projection_def()
 
+        def _parse_alias(
+            self, this: t.Optional[exp.Expression], explicit: bool = False
+        ) -> t.Optional[exp.Expression]:
+            # In clickhouse "SELECT <expr> APPLY(...)" is a query modifier,
+            # so "APPLY" shouldn't be parsed as <expr>'s alias. However, "SELECT <expr> apply" is a valid alias
+            if self._match_pair(TokenType.APPLY, TokenType.L_PAREN, advance=False):
+                return this
+
+            return super()._parse_alias(this=this, explicit=explicit)
+
+        def _parse_expression(self) -> t.Optional[exp.Expression]:
+            this = super()._parse_expression()
+
+            # Clickhouse allows "SELECT <expr> [APPLY(func)] [...]]" modifier
+            while self._match_pair(TokenType.APPLY, TokenType.L_PAREN):
+                this = exp.Apply(this=this, expression=self._parse_var(any_token=True))
+                self._match(TokenType.R_PAREN)
+
+            return this
+
     class Generator(generator.Generator):
         QUERY_HINTS = False
         STRUCT_DELIMITER = ("(", ")")
diff --git a/sqlglot/expressions.py b/sqlglot/expressions.py
index 2831e306a..95f3c01cc 100644
--- a/sqlglot/expressions.py
+++ b/sqlglot/expressions.py
@@ -11,7 +11,6 @@ SQL expressions, such as `sqlglot.expressions.select`.
 """
 
 from __future__ import annotations
-
 import datetime
 import math
 import numbers
@@ -36,6 +35,7 @@ from sqlglot.helper import (
 from sqlglot.tokens import Token, TokenError
 
 if t.TYPE_CHECKING:
+    from typing_extensions import Self
     from sqlglot._typing import E, Lit
     from sqlglot.dialects.dialect import DialectType
 
@@ -1368,7 +1368,7 @@ class DML(Expression):
         dialect: DialectType = None,
         copy: bool = True,
         **opts,
-    ) -> DML:
+    ) -> "Self":
         """
         Set the RETURNING expression. Not supported by all dialects.
 
@@ -4398,6 +4398,15 @@ class Alter(Expression):
         "not_valid": False,
     }
 
+    @property
+    def kind(self) -> t.Optional[str]:
+        kind = self.args.get("kind")
+        return kind and kind.upper()
+
+    @property
+    def actions(self) -> t.List[Expression]:
+        return self.args.get("actions") or []
+
 
 class AddConstraint(Expression):
     arg_types = {"expressions": True}
@@ -4913,6 +4922,10 @@ class ApproxDistinct(AggFunc):
     _sql_names = ["APPROX_DISTINCT", "APPROX_COUNT_DISTINCT"]
 
 
+class Apply(Func):
+    arg_types = {"this": True, "expression": True}
+
+
 class Array(Func):
     arg_types = {"expressions": False, "bracket_notation": False}
     is_var_len_args = True
@@ -6276,7 +6289,7 @@ class Use(Expression):
     arg_types = {"this": True, "kind": False}
 
 
-class Merge(Expression):
+class Merge(DML):
     arg_types = {
         "this": True,
         "using": True,
@@ -6840,9 +6853,7 @@ def delete(
     if where:
         delete_expr = delete_expr.where(where, dialect=dialect, copy=False, **opts)
     if returning:
-        delete_expr = t.cast(
-            Delete, delete_expr.returning(returning, dialect=dialect, copy=False, **opts)
-        )
+        delete_expr = delete_expr.returning(returning, dialect=dialect, copy=False, **opts)
     return delete_expr
 
 
@@ -6885,7 +6896,7 @@ def insert(
     insert = Insert(this=this, expression=expr, overwrite=overwrite)
 
     if returning:
-        insert = t.cast(Insert, insert.returning(returning, dialect=dialect, copy=False, **opts))
+        insert = insert.returning(returning, dialect=dialect, copy=False, **opts)
 
     return insert
 
@@ -6895,6 +6906,7 @@ def merge(
     into: ExpOrStr,
     using: ExpOrStr,
     on: ExpOrStr,
+    returning: t.Optional[ExpOrStr] = None,
     dialect: DialectType = None,
     copy: bool = True,
     **opts,
@@ -6915,6 +6927,7 @@ def merge(
         into: The target table to merge data into.
         using: The source table to merge data from.
         on: The join condition for the merge.
+        returning: The columns to return from the merge.
         dialect: The dialect used to parse the input expressions.
         copy: Whether to copy the expression.
         **opts: Other options to use to parse the input expressions.
@@ -6922,7 +6935,7 @@ def merge(
     Returns:
         Merge: The syntax tree for the MERGE statement.
     """
-    return Merge(
+    merge = Merge(
         this=maybe_parse(into, dialect=dialect, copy=copy, **opts),
         using=maybe_parse(using, dialect=dialect, copy=copy, **opts),
         on=maybe_parse(on, dialect=dialect, copy=copy, **opts),
@@ -6931,6 +6944,10 @@ def merge(
             for when_expr in when_exprs
         ],
     )
+    if returning:
+        merge = merge.returning(returning, dialect=dialect, copy=False, **opts)
+
+    return merge
 
 
 def condition(
diff --git a/sqlglot/generator.py b/sqlglot/generator.py
index 23b19b489..e20293135 100644
--- a/sqlglot/generator.py
+++ b/sqlglot/generator.py
@@ -2002,7 +2002,8 @@ class Generator(metaclass=_Generator):
             values = f"VALUES{self.seg('')}{args}"
             values = (
                 f"({values})"
-                if self.WRAP_DERIVED_VALUES and (alias or isinstance(expression.parent, exp.From))
+                if self.WRAP_DERIVED_VALUES
+                and (alias or isinstance(expression.parent, (exp.From, exp.Table)))
                 else values
             )
             return f"{values} AS {alias}" if alias else values
@@ -3625,13 +3626,15 @@ class Generator(metaclass=_Generator):
         using = f"USING {self.sql(expression, 'using')}"
         on = f"ON {self.sql(expression, 'on')}"
         expressions = self.expressions(expression, sep=" ", indent=False)
+        returning = self.sql(expression, "returning")
+        if returning:
+            expressions = f"{expressions}{returning}"
+
         sep = self.sep()
-        returning = self.expressions(expression, key="returning", indent=False)
-        returning = f"RETURNING {returning}" if returning else ""
 
         return self.prepend_ctes(
             expression,
-            f"MERGE INTO {this}{table_alias}{sep}{using}{sep}{on}{sep}{expressions}{sep}{returning}",
+            f"MERGE INTO {this}{table_alias}{sep}{using}{sep}{on}{sep}{expressions}",
         )
 
     @unsupported_args("format")
@@ -4323,6 +4326,19 @@ class Generator(metaclass=_Generator):
                 parent_cond = parent.expression.this
                 parent_cond.replace(parent_cond.and_(expression.this.is_(exp.null()).not_()))
             else:
-                array_agg = f"{array_agg} FILTER(WHERE {self.sql(expression, 'this')} IS NOT NULL)"
+                # DISTINCT is already present in the agg function, do not propagate it to FILTER as well
+                this = expression.this
+                this_sql = (
+                    self.expressions(this)
+                    if isinstance(this, exp.Distinct)
+                    else self.sql(expression, "this")
+                )
+                array_agg = f"{array_agg} FILTER(WHERE {this_sql} IS NOT NULL)"
 
         return array_agg
+
+    def apply_sql(self, expression: exp.Apply) -> str:
+        this = self.sql(expression, "this")
+        expr = self.sql(expression, "expression")
+
+        return f"{this} APPLY({expr})"


----

problem_statement: Support BigQuery `RANGE` type
**Is your feature request related to a problem? Please describe.**

BigQuery added a [`RANGE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#range_type) type which takes a type parameter, one of `DATE`, `DATETIME` or `TIMESTAMP`

**Describe the solution you'd like**

Add support for this new type.

**Describe alternatives you've considered**

N/A

**Additional context**

Would like to use this type in sqlmesh models.

date: 2024-09-23
patch: diff --git a/sqlglot/expressions.py b/sqlglot/expressions.py
index 95f3c01cc..de48a8a43 100644
--- a/sqlglot/expressions.py
+++ b/sqlglot/expressions.py
@@ -4123,6 +4123,7 @@ class DataType(Expression):
         NUMRANGE = auto()
         NVARCHAR = auto()
         OBJECT = auto()
+        RANGE = auto()
         ROWVERSION = auto()
         SERIAL = auto()
         SET = auto()
diff --git a/sqlglot/parser.py b/sqlglot/parser.py
index a21803573..a9fc38e51 100644
--- a/sqlglot/parser.py
+++ b/sqlglot/parser.py
@@ -269,6 +269,7 @@ class Parser(metaclass=_Parser):
         TokenType.LOWCARDINALITY,
         TokenType.MAP,
         TokenType.NULLABLE,
+        TokenType.RANGE,
         *STRUCT_TYPE_TOKENS,
     }
 
@@ -2984,6 +2985,13 @@ class Parser(metaclass=_Parser):
                     if table
                     else self._parse_select(nested=True, parse_set_operation=False)
                 )
+
+                # Transform exp.Values into a exp.Table to pass through parse_query_modifiers
+                # in case a modifier (e.g. join) is following
+                if table and isinstance(this, exp.Values) and this.alias:
+                    alias = this.args["alias"].pop()
+                    this = exp.Table(this=this, alias=alias)
+
                 this = self._parse_query_modifiers(self._parse_set_operations(this))
 
             self._match_r_paren()
@@ -5039,7 +5047,7 @@ class Parser(metaclass=_Parser):
                 if not field:
                     self.raise_error("Expected type")
             elif op and self._curr:
-                field = self._parse_column_reference()
+                field = self._parse_column_reference() or self._parse_bracket()
             else:
                 field = self._parse_field(any_token=True, anonymous_func=True)
 
@@ -6846,7 +6854,7 @@ class Parser(metaclass=_Parser):
             using=using,
             on=on,
             expressions=self._parse_when_matched(),
-            returning=self._match(TokenType.RETURNING) and self._parse_csv(self._parse_bitwise),
+            returning=self._parse_returning(),
         )
 
     def _parse_when_matched(self) -> t.List[exp.When]: