Repository: tobymao/sqlglot (2024-09-26 ~ 2024-10-01)

problem_statement: Postgresql overlay string function causes parser error
When parsing the following PostgreSQL command with dialect="postgres";

`select overlay('Txxxxas' placing 'hom' from 2 for 4)`

SqlGlot throws ParseError:

![Screenshot 2024-09-25 170438](https://github.com/user-attachments/assets/0e6cd1d2-dc5f-4a28-9157-fca57d398e99)


https://w3resource.com/PostgreSQL/overlay-function.php

date: 2024-09-26
patch: diff --git a/sqlglot/expressions.py b/sqlglot/expressions.py
index 35de71158..c5e40bed0 100644
--- a/sqlglot/expressions.py
+++ b/sqlglot/expressions.py
@@ -5913,6 +5913,10 @@ class Normalize(Func):
     arg_types = {"this": True, "form": False}
 
 
+class Overlay(Func):
+    arg_types = {"this": True, "expression": True, "from": True, "for": False}
+
+
 # https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-predict#mlpredict_function
 class Predict(Func):
     arg_types = {"this": True, "expression": True, "params_struct": False}
diff --git a/sqlglot/generator.py b/sqlglot/generator.py
index c7473b275..1f85126b7 100644
--- a/sqlglot/generator.py
+++ b/sqlglot/generator.py
@@ -4378,3 +4378,12 @@ class Generator(metaclass=_Generator):
             func = f"*{func}"
 
         return func
+
+    def overlay_sql(self, expression: exp.Overlay):
+        this = self.sql(expression, "this")
+        expr = self.sql(expression, "expression")
+        from_sql = self.sql(expression, "from")
+        for_sql = self.sql(expression, "for")
+        for_sql = f" FOR {for_sql}" if for_sql else ""
+
+        return f"OVERLAY({this} PLACING {expr} FROM {from_sql}{for_sql})"
diff --git a/sqlglot/parser.py b/sqlglot/parser.py
index 125a1c5ec..5f7349673 100644
--- a/sqlglot/parser.py
+++ b/sqlglot/parser.py
@@ -1107,6 +1107,7 @@ class Parser(metaclass=_Parser):
         "MATCH": lambda self: self._parse_match_against(),
         "NORMALIZE": lambda self: self._parse_normalize(),
         "OPENJSON": lambda self: self._parse_open_json(),
+        "OVERLAY": lambda self: self._parse_overlay(),
         "POSITION": lambda self: self._parse_position(),
         "PREDICT": lambda self: self._parse_predict(),
         "SAFE_CAST": lambda self: self._parse_cast(False, safe=True),
@@ -7473,3 +7474,14 @@ class Parser(metaclass=_Parser):
             principals=principals,
             grant_option=grant_option,
         )
+
+    def _parse_overlay(self) -> exp.Overlay:
+        return self.expression(
+            exp.Overlay,
+            **{  # type: ignore
+                "this": self._parse_bitwise(),
+                "expression": self._match_text_seq("PLACING") and self._parse_bitwise(),
+                "from": self._match_text_seq("FROM") and self._parse_bitwise(),
+                "for": self._match_text_seq("FOR") and self._parse_bitwise(),
+            },
+        )

test_patch: diff --git a/tests/dialects/test_duckdb.py b/tests/dialects/test_duckdb.py
index e4788ec116..c9f4d62a4c 100644
--- a/tests/dialects/test_duckdb.py
+++ b/tests/dialects/test_duckdb.py
@@ -858,6 +858,15 @@ class TestDuckDB(Validator):
         self.validate_identity(
             "SELECT COALESCE(*COLUMNS(['a', 'b', 'c'])) AS result FROM (SELECT NULL AS a, 42 AS b, TRUE AS c)"
         )
+        self.validate_all(
+            "SELECT UNNEST(foo) AS x",
+            write={
+                "redshift": UnsupportedError,
+            },
+        )
+        self.validate_identity("a ^ b", "POWER(a, b)")
+        self.validate_identity("a ** b", "POWER(a, b)")
+        self.validate_identity("a ~~~ b", "a GLOB b")
 
     def test_array_index(self):
         with self.assertLogs(helper_logger) as cm:
@@ -967,6 +976,15 @@ class TestDuckDB(Validator):
                 "spark": "DATE_FORMAT(x, 'yy-M-ss')",
             },
         )
+
+        self.validate_all(
+            "SHA1(x)",
+            write={
+                "duckdb": "SHA1(x)",
+                "": "SHA(x)",
+            },
+        )
+
         self.validate_all(
             "STRFTIME(x, '%Y-%m-%d %H:%M:%S')",
             write={
@@ -1086,6 +1104,7 @@ class TestDuckDB(Validator):
         self.validate_identity("CAST(x AS INT16)", "CAST(x AS SMALLINT)")
         self.validate_identity("CAST(x AS NUMERIC(1, 2))", "CAST(x AS DECIMAL(1, 2))")
         self.validate_identity("CAST(x AS HUGEINT)", "CAST(x AS INT128)")
+        self.validate_identity("CAST(x AS UHUGEINT)", "CAST(x AS UINT128)")
         self.validate_identity("CAST(x AS CHAR)", "CAST(x AS TEXT)")
         self.validate_identity("CAST(x AS BPCHAR)", "CAST(x AS TEXT)")
         self.validate_identity("CAST(x AS STRING)", "CAST(x AS TEXT)")
diff --git a/tests/dialects/test_postgres.py b/tests/dialects/test_postgres.py
index e4d239e5ae..63266a50e6 100644
--- a/tests/dialects/test_postgres.py
+++ b/tests/dialects/test_postgres.py
@@ -800,6 +800,8 @@ class TestPostgres(Validator):
         self.validate_identity(
             "SELECT 1 FROM ((VALUES (1)) AS vals(id) LEFT OUTER JOIN tbl ON vals.id = tbl.id)"
         )
+        self.validate_identity("SELECT OVERLAY(a PLACING b FROM 1)")
+        self.validate_identity("SELECT OVERLAY(a PLACING b FROM 1 FOR 1)")
 
     def test_ddl(self):
         # Checks that user-defined types are parsed into DataType instead of Identifier
diff --git a/tests/test_diff.py b/tests/test_diff.py
index fa012a890a..a83452329f 100644
--- a/tests/test_diff.py
+++ b/tests/test_diff.py
@@ -1,14 +1,18 @@
 import unittest
 
 from sqlglot import exp, parse_one
-from sqlglot.diff import Insert, Keep, Move, Remove, Update, diff
+from sqlglot.diff import Insert, Move, Remove, Update, diff
 from sqlglot.expressions import Join, to_table
 
 
+def diff_delta_only(source, target, matchings=None, **kwargs):
+    return diff(source, target, matchings=matchings, delta_only=True, **kwargs)
+
+
 class TestDiff(unittest.TestCase):
     def test_simple(self):
         self._validate_delta_only(
-            diff(parse_one("SELECT a + b"), parse_one("SELECT a - b")),
+            diff_delta_only(parse_one("SELECT a + b"), parse_one("SELECT a - b")),
             [
                 Remove(parse_one("a + b")),  # the Add node
                 Insert(parse_one("a - b")),  # the Sub node
@@ -16,21 +20,21 @@ class TestDiff(unittest.TestCase):
         )
 
         self._validate_delta_only(
-            diff(parse_one("SELECT a, b, c"), parse_one("SELECT a, c")),
+            diff_delta_only(parse_one("SELECT a, b, c"), parse_one("SELECT a, c")),
             [
                 Remove(parse_one("b")),  # the Column node
             ],
         )
 
         self._validate_delta_only(
-            diff(parse_one("SELECT a, b"), parse_one("SELECT a, b, c")),
+            diff_delta_only(parse_one("SELECT a, b"), parse_one("SELECT a, b, c")),
             [
                 Insert(parse_one("c")),  # the Column node
             ],
         )
 
         self._validate_delta_only(
-            diff(
+            diff_delta_only(
                 parse_one("SELECT a FROM table_one"),
                 parse_one("SELECT a FROM table_two"),
             ),
@@ -44,7 +48,9 @@ class TestDiff(unittest.TestCase):
 
     def test_lambda(self):
         self._validate_delta_only(
-            diff(parse_one("SELECT a, b, c, x(a -> a)"), parse_one("SELECT a, b, c, x(b -> b)")),
+            diff_delta_only(
+                parse_one("SELECT a, b, c, x(a -> a)"), parse_one("SELECT a, b, c, x(b -> b)")
+            ),
             [
                 Update(
                     exp.Lambda(this=exp.to_identifier("a"), expressions=[exp.to_identifier("a")]),
@@ -55,14 +61,16 @@ class TestDiff(unittest.TestCase):
 
     def test_udf(self):
         self._validate_delta_only(
-            diff(parse_one('SELECT a, b, "my.udf1"()'), parse_one('SELECT a, b, "my.udf2"()')),
+            diff_delta_only(
+                parse_one('SELECT a, b, "my.udf1"()'), parse_one('SELECT a, b, "my.udf2"()')
+            ),
             [
                 Insert(parse_one('"my.udf2"()')),
                 Remove(parse_one('"my.udf1"()')),
             ],
         )
         self._validate_delta_only(
-            diff(
+            diff_delta_only(
                 parse_one('SELECT a, b, "my.udf"(x, y, z)'),
                 parse_one('SELECT a, b, "my.udf"(x, y, w)'),
             ),
@@ -74,28 +82,28 @@ class TestDiff(unittest.TestCase):
 
     def test_node_position_changed(self):
         self._validate_delta_only(
-            diff(parse_one("SELECT a, b, c"), parse_one("SELECT c, a, b")),
+            diff_delta_only(parse_one("SELECT a, b, c"), parse_one("SELECT c, a, b")),
             [
                 Move(parse_one("c")),  # the Column node
             ],
         )
 
         self._validate_delta_only(
-            diff(parse_one("SELECT a + b"), parse_one("SELECT b + a")),
+            diff_delta_only(parse_one("SELECT a + b"), parse_one("SELECT b + a")),
             [
                 Move(parse_one("a")),  # the Column node
             ],
         )
 
         self._validate_delta_only(
-            diff(parse_one("SELECT aaaa AND bbbb"), parse_one("SELECT bbbb AND aaaa")),
+            diff_delta_only(parse_one("SELECT aaaa AND bbbb"), parse_one("SELECT bbbb AND aaaa")),
             [
                 Move(parse_one("aaaa")),  # the Column node
             ],
         )
 
         self._validate_delta_only(
-            diff(
+            diff_delta_only(
                 parse_one("SELECT aaaa OR bbbb OR cccc"),
                 parse_one("SELECT cccc OR bbbb OR aaaa"),
             ),
@@ -120,7 +128,7 @@ class TestDiff(unittest.TestCase):
         """
 
         self._validate_delta_only(
-            diff(parse_one(expr_src), parse_one(expr_tgt)),
+            diff_delta_only(parse_one(expr_src), parse_one(expr_tgt)),
             [
                 Remove(parse_one("LOWER(c) AS c")),  # the Alias node
                 Remove(parse_one("LOWER(c)")),  # the Lower node
@@ -133,8 +141,7 @@ class TestDiff(unittest.TestCase):
         expr_src = "SELECT a, b FROM t1 LEFT JOIN t2 ON t1.key = t2.key"
         expr_tgt = "SELECT a, b FROM t1 RIGHT JOIN t2 ON t1.key = t2.key"
 
-        changes = diff(parse_one(expr_src), parse_one(expr_tgt))
-        changes = _delta_only(changes)
+        changes = diff_delta_only(parse_one(expr_src), parse_one(expr_tgt))
 
         self.assertEqual(len(changes), 2)
         self.assertTrue(isinstance(changes[0], Remove))
@@ -145,22 +152,31 @@ class TestDiff(unittest.TestCase):
         expr_src = parse_one("SELECT ROW_NUMBER() OVER (PARTITION BY a ORDER BY b)")
         expr_tgt = parse_one("SELECT RANK() OVER (PARTITION BY a ORDER BY b)")
 
-        self._validate_delta_only(diff(expr_src, expr_src), [])
+        self._validate_delta_only(diff_delta_only(expr_src, expr_src), [])
 
         self._validate_delta_only(
-            diff(expr_src, expr_tgt),
+            diff_delta_only(expr_src, expr_tgt),
             [
-                Remove(parse_one("ROW_NUMBER()")),  # the Anonymous node
-                Insert(parse_one("RANK()")),  # the Anonymous node
+                Remove(parse_one("ROW_NUMBER()")),
+                Insert(parse_one("RANK()")),
+                Update(source=expr_src.selects[0], target=expr_tgt.selects[0]),
             ],
         )
 
+        expr_src = parse_one("SELECT MAX(x) OVER (ORDER BY y) FROM z", "oracle")
+        expr_tgt = parse_one("SELECT MAX(x) KEEP (DENSE_RANK LAST ORDER BY y) FROM z", "oracle")
+
+        self._validate_delta_only(
+            diff_delta_only(expr_src, expr_tgt),
+            [Update(source=expr_src.selects[0], target=expr_tgt.selects[0])],
+        )
+
     def test_pre_matchings(self):
         expr_src = parse_one("SELECT 1")
         expr_tgt = parse_one("SELECT 1, 2, 3, 4")
 
         self._validate_delta_only(
-            diff(expr_src, expr_tgt),
+            diff_delta_only(expr_src, expr_tgt),
             [
                 Remove(expr_src),
                 Insert(expr_tgt),
@@ -171,7 +187,7 @@ class TestDiff(unittest.TestCase):
         )
 
         self._validate_delta_only(
-            diff(expr_src, expr_tgt, matchings=[(expr_src, expr_tgt)]),
+            diff_delta_only(expr_src, expr_tgt, matchings=[(expr_src, expr_tgt)]),
             [
                 Insert(exp.Literal.number(2)),
                 Insert(exp.Literal.number(3)),
@@ -180,23 +196,34 @@ class TestDiff(unittest.TestCase):
         )
 
         with self.assertRaises(ValueError):
-            diff(expr_src, expr_tgt, matchings=[(expr_src, expr_tgt), (expr_src, expr_tgt)])
+            diff_delta_only(
+                expr_src, expr_tgt, matchings=[(expr_src, expr_tgt), (expr_src, expr_tgt)]
+            )
 
     def test_identifier(self):
         expr_src = parse_one("SELECT a FROM tbl")
         expr_tgt = parse_one("SELECT a, tbl.b from tbl")
 
         self._validate_delta_only(
-            diff(expr_src, expr_tgt),
+            diff_delta_only(expr_src, expr_tgt),
             [
                 Insert(expression=exp.to_column("tbl.b")),
             ],
         )
 
-    def _validate_delta_only(self, actual_diff, expected_delta):
-        actual_delta = _delta_only(actual_diff)
-        self.assertEqual(set(actual_delta), set(expected_delta))
+        expr_src = parse_one("SELECT 1 AS c1, 2 AS c2")
+        expr_tgt = parse_one("SELECT 2 AS c1, 3 AS c2")
 
+        self._validate_delta_only(
+            diff_delta_only(expr_src, expr_tgt),
+            [
+                Remove(expression=exp.alias_(1, "c1")),
+                Remove(expression=exp.Literal.number(1)),
+                Insert(expression=exp.alias_(3, "c2")),
+                Insert(expression=exp.Literal.number(3)),
+                Update(source=exp.alias_(2, "c2"), target=exp.alias_(2, "c1")),
+            ],
+        )
 
-def _delta_only(changes):
-    return [d for d in changes if not isinstance(d, Keep)]
+    def _validate_delta_only(self, actual_delta, expected_delta):
+        self.assertEqual(set(actual_delta), set(expected_delta))


----

problem_statement: Incorrect Handling of the `~~~` Operator in DuckDB
When testing SQLGlot with DuckDB, I discovered that the `~~~` operator is incorrectly handled. This operator is equivalent to the `GLOB` keyword in DuckDB, but SQLGlot incorrectly translates it.

#### Original Query:

```sql
SELECT ?::VARCHAR ~~~ ?::VARCHAR LIMIT 1
```

#### SQLGlot Output:

```sql
SELECT REGEXP_MATCHES(CAST(? AS TEXT), ~~CAST(? AS TEXT)) LIMIT 1
```

#### Expected Behavior:
SQLGlot should produce either the `~~~` operator or the `GLOB` function, as defined by DuckDB for pattern matching. The correct translation would be:

```sql
SELECT ?::VARCHAR ~~~ ?::VARCHAR LIMIT 1
```

#### Reference:
The `~~~` operator is documented to function the same as `GLOB` in DuckDB:
- [DuckDB Pattern Matching Documentation](https://duckdb.org/docs/sql/functions/pattern_matching.html)


date: 2024-10-01
patch: diff --git a/sqlglot/tokens.py b/sqlglot/tokens.py
index b2143bcb0..fd084d869 100644
--- a/sqlglot/tokens.py
+++ b/sqlglot/tokens.py
@@ -649,6 +649,7 @@ class Tokenizer(metaclass=_Tokenizer):
         "<->": TokenType.LR_ARROW,
         "&&": TokenType.DAMP,
         "??": TokenType.DQMARK,
+        "~~~": TokenType.GLOB,
         "ALL": TokenType.ALL,
         "ALWAYS": TokenType.ALWAYS,
         "AND": TokenType.AND,
@@ -818,6 +819,7 @@ class Tokenizer(metaclass=_Tokenizer):
         "SMALLINT": TokenType.SMALLINT,
         "INT128": TokenType.INT128,
         "HUGEINT": TokenType.INT128,
+        "UHUGEINT": TokenType.UINT128,
         "INT2": TokenType.SMALLINT,
         "INTEGER": TokenType.INT,
         "INT": TokenType.INT,


----

problem_statement: sqlglot.diff Bug for exp.Window Differences
The following statements are considered equal by `sqlglot.diff`.

However, one is `OVER`, while the other is `KEEP DENSE_RANK`, and thus these should not be considered semantically identical.

```
import sqlglot


parsed = sqlglot.parse_one(
    '''
        SELECT MAX(foo) OVER (
            ORDER BY bar
        )
        FROM baz
    ''',
    dialect='oracle'
)

parsed2 = sqlglot.parse_one(
    '''
        SELECT MAX(foo) KEEP (
            DENSE_RANK LAST
            ORDER BY bar
        )
        FROM baz
    ''',
    dialect='oracle'
)

diffs = sqlglot.diff(parsed, parsed2)

from sqlglot.diff import Keep

# This should fail but does not.
assert all(isinstance(diff, Keep) for diff in diffs)
```
date: 2024-10-01
patch: diff --git a/sqlglot/diff.py b/sqlglot/diff.py
index 22c506a9b..eaca8b8a7 100644
--- a/sqlglot/diff.py
+++ b/sqlglot/diff.py
@@ -62,6 +62,7 @@ def diff(
     source: exp.Expression,
     target: exp.Expression,
     matchings: t.List[t.Tuple[exp.Expression, exp.Expression]] | None = None,
+    delta_only: bool = False,
     **kwargs: t.Any,
 ) -> t.List[Edit]:
     """
@@ -89,6 +90,8 @@ def diff(
             heuristics produce better results for subtrees that are known by a caller to be matching.
             Note: expression references in this list must refer to the same node objects that are
             referenced in source / target trees.
+        delta_only: excludes all `Keep` nodes from the diff.
+        kwargs: additional arguments to pass to the ChangeDistiller instance.
 
     Returns:
         the list of Insert, Remove, Move, Update and Keep objects for each node in the source and the
@@ -116,17 +119,24 @@ def diff(
     }
     matchings_copy = [(node_mappings[id(s)], node_mappings[id(t)]) for s, t in matchings]
 
-    return ChangeDistiller(**kwargs).diff(source_copy, target_copy, matchings=matchings_copy)
+    return ChangeDistiller(**kwargs).diff(
+        source_copy,
+        target_copy,
+        matchings=matchings_copy,
+        delta_only=delta_only,
+    )
 
 
 # The expression types for which Update edits are allowed.
 UPDATABLE_EXPRESSION_TYPES = (
+    exp.Alias,
     exp.Boolean,
+    exp.Column,
     exp.DataType,
+    exp.Lambda,
     exp.Literal,
     exp.Table,
-    exp.Column,
-    exp.Lambda,
+    exp.Window,
 )
 
 IGNORED_LEAF_EXPRESSION_TYPES = (exp.Identifier,)
@@ -149,6 +159,7 @@ class ChangeDistiller:
         source: exp.Expression,
         target: exp.Expression,
         matchings: t.List[t.Tuple[exp.Expression, exp.Expression]] | None = None,
+        delta_only: bool = False,
     ) -> t.List[Edit]:
         matchings = matchings or []
         pre_matched_nodes = {id(s): id(t) for s, t in matchings}
@@ -168,9 +179,13 @@ class ChangeDistiller:
         self._bigram_histo_cache: t.Dict[int, t.DefaultDict[str, int]] = {}
 
         matching_set = self._compute_matching_set() | {(s, t) for s, t in pre_matched_nodes.items()}
-        return self._generate_edit_script(matching_set)
+        return self._generate_edit_script(matching_set, delta_only)
 
-    def _generate_edit_script(self, matching_set: t.Set[t.Tuple[int, int]]) -> t.List[Edit]:
+    def _generate_edit_script(
+        self,
+        matching_set: t.Set[t.Tuple[int, int]],
+        delta_only: bool,
+    ) -> t.List[Edit]:
         edit_script: t.List[Edit] = []
         for removed_node_id in self._unmatched_source_nodes:
             edit_script.append(Remove(self._source_index[removed_node_id]))
@@ -186,7 +201,8 @@ class ChangeDistiller:
                 edit_script.extend(
                     self._generate_move_edits(source_node, target_node, matching_set)
                 )
-                edit_script.append(Keep(source_node, target_node))
+                if not delta_only:
+                    edit_script.append(Keep(source_node, target_node))
             else:
                 edit_script.append(Update(source_node, target_node))