Repository: tobymao/sqlglot (2024-06-05 ~ 2024-06-11)

problem_statement: Parsing issue for Snowflake COPY INTO with subqueries and FILE_FORMAT
Hi!

This issue sounds like an extension of previously reported: https://github.com/tobymao/sqlglot/issues/3434 where support for COPY INTO with subqueries was added.

Looks like the library is facing an issue when FILE_FORMAT is being added to the query:
```
from sqlglot import parse_one

query = "COPY INTO @my_stage/result/data FROM (SELECT * FROM orderstiny) file_format=(type='csv');"
parse_one(query, read='snowflake')
```
above is producing following error:

```
sqlglot.errors.ParseError: Required keyword: 'this' missing for <class 'sqlglot.expressions.CopyParameter'>. Line 1, Col: 77
```

Please let me know in case of any additional information is needed.
date: 2024-06-05
patch: diff --git a/sqlglot/dialects/snowflake.py b/sqlglot/dialects/snowflake.py
index ec32a92c..8c3c471c 100644
--- a/sqlglot/dialects/snowflake.py
+++ b/sqlglot/dialects/snowflake.py
@@ -714,7 +714,7 @@ class Snowflake(Dialect):
         def _parse_file_location(self) -> t.Optional[exp.Expression]:
             # Parse either a subquery or a staged file
             return (
-                self._parse_select(table=True)
+                self._parse_select(table=True, parse_subquery_alias=False)
                 if self._match(TokenType.L_PAREN, advance=False)
                 else self._parse_table_parts()
             )
diff --git a/sqlglot/dialects/tsql.py b/sqlglot/dialects/tsql.py
index ee403a71..ab49d08c 100644
--- a/sqlglot/dialects/tsql.py
+++ b/sqlglot/dialects/tsql.py
@@ -994,9 +994,13 @@ class TSQL(Dialect):
                 elif kind == "TABLE":
                     assert table
                     where = exp.and_(
-                        exp.column("table_name").eq(table.name),
-                        exp.column("table_schema").eq(table.db) if table.db else None,
-                        exp.column("table_catalog").eq(table.catalog) if table.catalog else None,
+                        exp.column("table_name").eq(exp.Literal.string(table.name)),
+                        exp.column("table_schema").eq(exp.Literal.string(table.db))
+                        if table.db
+                        else None,
+                        exp.column("table_catalog").eq(exp.Literal.string(table.catalog))
+                        if table.catalog
+                        else None,
                     )
                     sql = f"""IF NOT EXISTS (SELECT * FROM information_schema.tables WHERE {where}) EXEC({sql})"""
                 elif kind == "INDEX":
diff --git a/sqlglot/expressions.py b/sqlglot/expressions.py
index a74b7521..84133a3f 100644
--- a/sqlglot/expressions.py
+++ b/sqlglot/expressions.py
@@ -832,7 +832,7 @@ class Expression(metaclass=_Expression):
 
     def _binop(self, klass: t.Type[E], other: t.Any, reverse: bool = False) -> E:
         this = self.copy()
-        other = convert(other, copy=True)
+        other = maybe_parse(other, copy=True)
         if not isinstance(this, klass) and not isinstance(other, klass):
             this = _wrap(this, Binary)
             other = _wrap(other, Binary)

test_patch: diff --git a/tests/dialects/test_snowflake.py b/tests/dialects/test_snowflake.py
index ba857191..9d9371db 100644
--- a/tests/dialects/test_snowflake.py
+++ b/tests/dialects/test_snowflake.py
@@ -1934,6 +1934,9 @@ STORAGE_ALLOWED_LOCATIONS=('s3://mybucket1/path1/', 's3://mybucket2/path2/')""",
         self.validate_identity(
             """COPY INTO mytable (col1, col2) FROM 's3://mybucket/data/files' STORAGE_INTEGRATION = "storage" ENCRYPTION = (TYPE='NONE' MASTER_KEY='key') FILES = ('file1', 'file2') PATTERN = 'pattern' FILE_FORMAT = (FORMAT_NAME=my_csv_format NULL_IF=('')) PARSE_HEADER = TRUE"""
         )
+        self.validate_identity(
+            """COPY INTO @my_stage/result/data FROM (SELECT * FROM orderstiny) FILE_FORMAT = (TYPE='csv')"""
+        )
         self.validate_all(
             """COPY INTO 's3://example/data.csv'
     FROM EXTRA.EXAMPLE.TABLE
diff --git a/tests/test_build.py b/tests/test_build.py
index 150bb42e..523dcc0a 100644
--- a/tests/test_build.py
+++ b/tests/test_build.py
@@ -67,16 +67,17 @@ class TestBuild(unittest.TestCase):
             (lambda: 1 + x * 2 + 3, "1 + (x * 2) + 3"),
             (lambda: x * 1 * 2 + 3, "(x * 1 * 2) + 3"),
             (lambda: 1 + (x * 2) / 3, "1 + ((x * 2) / 3)"),
-            (lambda: x & "y", "x AND 'y'"),
-            (lambda: x | "y", "x OR 'y'"),
+            (lambda: x & "'y'", "x AND 'y'"),
+            (lambda: x | "'y'", "x OR 'y'"),
             (lambda: -x, "-x"),
             (lambda: ~x, "NOT x"),
             (lambda: x[1], "x[1]"),
             (lambda: x[1, 2], "x[1, 2]"),
             (lambda: x["y"] + 1, "x['y'] + 1"),
-            (lambda: x.like("y"), "x LIKE 'y'"),
-            (lambda: x.ilike("y"), "x ILIKE 'y'"),
-            (lambda: x.rlike("y"), "REGEXP_LIKE(x, 'y')"),
+            (lambda: x.like("y"), "x LIKE y"),
+            (lambda: x.like("'y'"), "x LIKE 'y'"),
+            (lambda: x.ilike("'y'"), "x ILIKE 'y'"),
+            (lambda: x.rlike("'y'"), "REGEXP_LIKE(x, 'y')"),
             (
                 lambda: case().when("x = 1", "x").else_("bar"),
                 "CASE WHEN x = 1 THEN x ELSE bar END",


----

problem_statement: Parsing fails when struct / json field is called `select`
**Fully reproducible code snippet**
```
    query = """
select
    invocations.invocation_args:select::varchar as invocation_select
from table
"""
sqlglot.parse(query, sqlglot.Dialects.SNOWFLAKE)
```

This snippet raises an exception: `sqlglot.errors.ParseError: Invalid expression / Unexpected token. Line 3, Col: 19.`

date: 2024-06-10
patch: diff --git a/sqlglot/parser.py b/sqlglot/parser.py
index b99d34c0..854f5e7d 100644
--- a/sqlglot/parser.py
+++ b/sqlglot/parser.py
@@ -4534,7 +4534,11 @@ class Parser(metaclass=_Parser):
 
         while self._match(TokenType.COLON):
             start_index = self._index
-            path = self._parse_column_ops(self._parse_field(any_token=True))
+
+            # Snowflake allows reserved keywords as json keys but advance_any() excludes TokenType.SELECT from any_tokens=True
+            path = self._parse_column_ops(
+                self._parse_field(any_token=True, tokens=(TokenType.SELECT,))
+            )
 
             # The cast :: operator has a lower precedence than the extraction operator :, so
             # we rearrange the AST appropriately to avoid casting the JSON path

test_patch: diff --git a/tests/dialects/test_snowflake.py b/tests/dialects/test_snowflake.py
index e5bf4f27..12864369 100644
--- a/tests/dialects/test_snowflake.py
+++ b/tests/dialects/test_snowflake.py
@@ -125,6 +125,10 @@ WHERE
             "SELECT a:from::STRING, a:from || ' test' ",
             "SELECT CAST(GET_PATH(a, 'from') AS TEXT), GET_PATH(a, 'from') || ' test'",
         )
+        self.validate_identity(
+            "SELECT a:select",
+            "SELECT GET_PATH(a, 'select')",
+        )
         self.validate_identity("x:from", "GET_PATH(x, 'from')")
         self.validate_identity(
             "value:values::string::int",


----

problem_statement: Duckdb - Error tokenizing multiple named parameters 
**Before you file an issue**
- [x] Make sure you specify the "read" dialect eg. `parse_one(sql, read="spark")`
- [x] Make sure you specify the "write" dialect eg. `ast.sql(dialect="duckdb")`
- [x] Check if the issue still exists on main

**Fully reproducible code snippet**
I'm writing a transformer that remap duckdb named parameters into a valid type based on the user's inputs.

```python
def resolve_user_params(query: str, params: dict[str, Any], dialect='duckdb') -> str:
    errors: set[str] = set()
    
    def _transform(node: sqlglot.expressions.Expression) -> sqlglot.expressions.Expression:
        if isinstance(node, sqlglot.expressions.Placeholder):
            key = node.name
            if key not in params:
                errors.add(f'Variable ${key} is an unknown user_param')
                return node
            value = params[key]
            match value:
                case bool():
                    return sqlglot.expressions.Boolean(this=value)
                case int():
                    return sqlglot.expressions.Literal(this=value, is_string=False)
                case float():
                    return sqlglot.expressions.Literal(this=value, is_string=False)
                case str():
                    return sqlglot.expressions.Literal(this=value, is_string=True)
                case _:
                    errors.add(f'Variable ${key} is not a valid type, got {type(value)}')
            return node
        return node

    
    parsed_query = sqlglot.parse_one(query, read=dialect)
    # print(repr(parsed_query))
    
    transformed_query = parsed_query.transform(_transform)
    final_query = transformed_query.sql(dialect=dialect, comments=False)

    print('query', query)
    print(' - params', params)
    print(' - errors', errors)
    print(' - final_query', final_query)

    return final_query
```

The function works with a basic query like:

```python
resolve_user_params(
    query="SELECT * FROM taxi_zones WHERE LocationID > $location_id_min AND LocationID IS NOT NULL",
    params={
        "location_id_min": 124,
    },
)

# query SELECT * FROM taxi_zones WHERE LocationID > $location_id_min AND LocationID IS NOT NULL
# - params {'location_id_min': 124}
# - errors set()
# - final_query SELECT * FROM taxi_zones WHERE LocationID > 124 AND NOT LocationID IS NULL
```

but it fails when I pass multiple named parameters:

```python
resolve_user_params(
    query="SELECT * FROM taxi_zones WHERE LocationID > $location_id_min AND Borough = $borough",
    params={
        "location_id_min": 124,
        "borough": 12,
    },
)

Traceback (most recent call last):
  File ".venv/lib/python3.12/site-packages/sqlglot/tokens.py", line 970, in tokenize
    self._scan()
  File ".venv/lib/python3.12/site-packages/sqlglot/tokens.py", line 1003, in _scan
    self._scan_keywords()
  File ".venv/lib/python3.12/site-packages/sqlglot/tokens.py", line 1134, in _scan_keywords
    if self._scan_string(word):
       ^^^^^^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/sqlglot/tokens.py", line 1306, in _scan_string
    text = self._extract_string(end, unescape_sequences=token_type != TokenType.RAW_STRING)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/sqlglot/tokens.py", line 1385, in _extract_string
    raise TokenError(f"Missing {delimiter} from {self._line}:{self._start}")
sqlglot.errors.TokenError: Missing $location_id_min AND Borough = $ from 1:44

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File ".venv/lib/python3.12/site-packages/marimo/_runtime/runner/cell_runner.py", line 302, in run
    return_value = execute_cell(cell, self.glbls)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/marimo/_ast/cell.py", line 460, in execute_cell
    return eval(cell.last_expr, glbls)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/var/folders/pg/0dm9b2p54yn_wqd89jd7j_wm0000gn/T/marimo_4911/__marimo__cell_lEQa__output.py", line 31, in <module>
  File "/var/folders/pg/0dm9b2p54yn_wqd89jd7j_wm0000gn/T/marimo_4911/__marimo__cell_bkHC_.py", line 26, in resolve_user_params
    parsed_query = sqlglot.parse_one(query, read='duckdb')
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/sqlglot/__init__.py", line 136, in parse_one
    result = dialect.parse(sql, **opts)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/sqlglot/dialects/dialect.py", line 518, in parse
    return self.parser(**opts).parse(self.tokenize(sql), sql)
                                     ^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/sqlglot/dialects/dialect.py", line 535, in tokenize
    return self.tokenizer.tokenize(sql)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/sqlglot/tokens.py", line 975, in tokenize
    raise TokenError(f"Error tokenizing '{context}'") from e
sqlglot.errors.TokenError: Error tokenizing 'cationID > $location_id_min AND Borough = $boroug'
```

**Official Documentation**
- Python SDK: https://duckdb.org/docs/api/python/dbapi#named-parameters
- Query Syntax: https://duckdb.org/docs/sql/query_syntax/prepared_statements.html#named-parameters-parameter

date: 2024-06-11
patch: diff --git a/sqlglot/tokens.py b/sqlglot/tokens.py
index 3b6fc428..1282e150 100644
--- a/sqlglot/tokens.py
+++ b/sqlglot/tokens.py
@@ -1273,18 +1273,6 @@ class Tokenizer(metaclass=_Tokenizer):
             elif token_type == TokenType.BIT_STRING:
                 base = 2
             elif token_type == TokenType.HEREDOC_STRING:
-                if (
-                    self.HEREDOC_TAG_IS_IDENTIFIER
-                    and not self._peek.isidentifier()
-                    and not self._peek == end
-                ):
-                    if self.HEREDOC_STRING_ALTERNATIVE != token_type.VAR:
-                        self._add(self.HEREDOC_STRING_ALTERNATIVE)
-                    else:
-                        self._scan_var()
-
-                    return True
-
                 self._advance()
 
                 if self._char == end:
@@ -1296,7 +1284,10 @@ class Tokenizer(metaclass=_Tokenizer):
                         raise_unmatched=not self.HEREDOC_TAG_IS_IDENTIFIER,
                     )
 
-                if self._end and tag and self.HEREDOC_TAG_IS_IDENTIFIER:
+                if tag and self.HEREDOC_TAG_IS_IDENTIFIER and (self._end or not tag.isidentifier()):
+                    if not self._end:
+                        self._advance(-1)
+
                     self._advance(-len(tag))
                     self._add(self.HEREDOC_STRING_ALTERNATIVE)
                     return True
diff --git a/sqlglotrs/src/tokenizer.rs b/sqlglotrs/src/tokenizer.rs
index e79d0e90..45bbe778 100644
--- a/sqlglotrs/src/tokenizer.rs
+++ b/sqlglotrs/src/tokenizer.rs
@@ -405,19 +405,6 @@ impl<'a> TokenizerState<'a> {
             } else if *token_type == self.token_types.bit_string {
                 (Some(2), *token_type, end.clone())
             } else if *token_type == self.token_types.heredoc_string {
-                if self.settings.heredoc_tag_is_identifier
-                    && !self.is_identifier(self.peek_char)
-                    && self.peek_char.to_string() != *end
-                {
-                    if self.token_types.heredoc_string_alternative != self.token_types.var {
-                        self.add(self.token_types.heredoc_string_alternative, None)?
-                    } else {
-                        self.scan_var()?
-                    };
-
-                    return Ok(true)
-                };
-
                 self.advance(1)?;
 
                 let tag = if self.current_char.to_string() == *end {
@@ -426,7 +413,14 @@ impl<'a> TokenizerState<'a> {
                     self.extract_string(end, false, false, !self.settings.heredoc_tag_is_identifier)?
                 };
 
-                if self.is_end && !tag.is_empty() && self.settings.heredoc_tag_is_identifier {
+                if !tag.is_empty()
+                    && self.settings.heredoc_tag_is_identifier
+                    && (self.is_end || !self.is_identifier(&tag))
+                {
+                    if !self.is_end {
+                        self.advance(-1)?;
+                    }
+
                     self.advance(-(tag.len() as isize))?;
                     self.add(self.token_types.heredoc_string_alternative, None)?;
                     return Ok(true)
@@ -494,7 +488,7 @@ impl<'a> TokenizerState<'a> {
             } else if self.peek_char.to_ascii_uppercase() == 'E' && scientific == 0 {
                 scientific += 1;
                 self.advance(1)?;
-            } else if self.is_identifier(self.peek_char) {
+            } else if self.is_alphabetic_or_underscore(self.peek_char) {
                 let number_text = self.text();
                 let mut literal = String::from("");
 
@@ -676,10 +670,18 @@ impl<'a> TokenizerState<'a> {
         Ok(text)
     }
 
-    fn is_identifier(&mut self, name: char) -> bool {
+    fn is_alphabetic_or_underscore(&mut self, name: char) -> bool {
         name.is_alphabetic() || name == '_'
     }
 
+    fn is_identifier(&mut self, s: &str) -> bool {
+        s.chars().enumerate().all(
+            |(i, c)|
+            if i == 0 { self.is_alphabetic_or_underscore(c) }
+            else { self.is_alphabetic_or_underscore(c) || c.is_digit(10) }
+        )
+    }
+
     fn extract_value(&mut self) -> Result<String, TokenizerError> {
         loop {
             if !self.peek_char.is_whitespace()

test_patch: diff --git a/tests/dialects/test_duckdb.py b/tests/dialects/test_duckdb.py
index 7bc493ac..08a65091 100644
--- a/tests/dialects/test_duckdb.py
+++ b/tests/dialects/test_duckdb.py
@@ -285,6 +285,7 @@ class TestDuckDB(Validator):
         self.validate_identity("FROM tbl", "SELECT * FROM tbl")
         self.validate_identity("x -> '$.family'")
         self.validate_identity("CREATE TABLE color (name ENUM('RED', 'GREEN', 'BLUE'))")
+        self.validate_identity("SELECT * FROM foo WHERE bar > $baz AND bla = $bob")
         self.validate_identity(
             "SELECT * FROM x LEFT JOIN UNNEST(y)", "SELECT * FROM x LEFT JOIN UNNEST(y) ON TRUE"
         )