Repository: tobymao/sqlglot

Issue:
Incorrectly parses number literals with `_`s (ClickHouse dialect)
ClickHouse [supports](https://clickhouse.com/docs/en/sql-reference/syntax#numeric) using `_` in numbers for better human-readability of big numbers.
`sqlglot` does not correctly parse them:

```
>>> sqlglot.parse_one("select 1_000", dialect="clickhouse")
Select(
  expressions=[
    Alias(
      this=Literal(this=1, is_string=False),
      alias=Identifier(this=_000, quoted=False))])
```

I would expect output to look like:
```
>>> sqlglot.parse_one("select 1_000", dialect="clickhouse")
Select(
  expressions=[
    Literal(this=1000, is_string=False)
])
```

Thanks for your awesome library!

Discussion:

Solution patch:
diff --git a/sqlglot/dialects/clickhouse.py b/sqlglot/dialects/clickhouse.py
index 9e7086f153..911e8c51d2 100644
--- a/sqlglot/dialects/clickhouse.py
+++ b/sqlglot/dialects/clickhouse.py
@@ -166,6 +166,8 @@ class ClickHouse(Dialect):
     LOG_BASE_FIRST: t.Optional[bool] = None
     FORCE_EARLY_ALIAS_REF_EXPANSION = True
     PRESERVE_ORIGINAL_NAMES = True
+    NUMBERS_CAN_BE_UNDERSCORE_SEPARATED = True
+    IDENTIFIERS_CAN_START_WITH_DIGIT = True
 
     # https://github.com/ClickHouse/ClickHouse/issues/33935#issue-1112165779
     NORMALIZATION_STRATEGY = NormalizationStrategy.CASE_SENSITIVE
diff --git a/sqlglot/dialects/dialect.py b/sqlglot/dialects/dialect.py
index 58172ef3a6..392a0d86fc 100644
--- a/sqlglot/dialects/dialect.py
+++ b/sqlglot/dialects/dialect.py
@@ -420,6 +420,9 @@ class Dialect(metaclass=_Dialect):
     SUPPORTS_VALUES_DEFAULT = True
     """Whether the DEFAULT keyword is supported in the VALUES clause."""
 
+    NUMBERS_CAN_BE_UNDERSCORE_SEPARATED = False
+    """Whether number literals can include underscores for better readability"""
+
     REGEXP_EXTRACT_DEFAULT_GROUP = 0
     """The default value for the capturing group."""
 
diff --git a/sqlglot/expressions.py b/sqlglot/expressions.py
index 19a87235de..a9f6a091f6 100644
--- a/sqlglot/expressions.py
+++ b/sqlglot/expressions.py
@@ -59,6 +59,7 @@ def __new__(cls, clsname, bases, attrs):
 
 
 SQLGLOT_META = "sqlglot.meta"
+SQLGLOT_ANONYMOUS = "sqlglot.anonymous"
 TABLE_PARTS = ("this", "db", "catalog")
 COLUMN_PARTS = ("this", "table", "db", "catalog")
 
diff --git a/sqlglot/parser.py b/sqlglot/parser.py
index 6e70fa1c1a..4cbaf89a91 100644
--- a/sqlglot/parser.py
+++ b/sqlglot/parser.py
@@ -5361,6 +5361,16 @@ def _parse_function_call(
             alias = not known_function or upper in self.FUNCTIONS_WITH_ALIASED_ARGS
             args = self._parse_csv(lambda: self._parse_lambda(alias=alias))
 
+            post_func_comments = self._curr and self._curr.comments
+            if known_function and post_func_comments:
+                # If the user-inputted comment "/* sqlglot.anonymous */" is following the function
+                # call we'll construct it as exp.Anonymous, even if it's "known"
+                if any(
+                    comment.lstrip().startswith(exp.SQLGLOT_ANONYMOUS)
+                    for comment in post_func_comments
+                ):
+                    known_function = False
+
             if alias and known_function:
                 args = self._kv_to_prop_eq(args)
 
diff --git a/sqlglot/tokens.py b/sqlglot/tokens.py
index ef7abd818f..192c293c97 100644
--- a/sqlglot/tokens.py
+++ b/sqlglot/tokens.py
@@ -997,6 +997,7 @@ def __init__(self, dialect: DialectType = None) -> None:
             self._rs_dialect_settings = RsTokenizerDialectSettings(
                 unescaped_sequences=self.dialect.UNESCAPED_SEQUENCES,
                 identifiers_can_start_with_digit=self.dialect.IDENTIFIERS_CAN_START_WITH_DIGIT,
+                numbers_can_be_underscore_separated=self.dialect.NUMBERS_CAN_BE_UNDERSCORE_SEPARATED,
             )
 
         self.reset()
@@ -1300,8 +1301,12 @@ def _scan_number(self) -> None:
                     self._add(TokenType.NUMBER, number_text)
                     self._add(TokenType.DCOLON, "::")
                     return self._add(token_type, literal)
-                elif self.dialect.IDENTIFIERS_CAN_START_WITH_DIGIT:
-                    return self._add(TokenType.VAR)
+                else:
+                    replaced = literal.replace("_", "")
+                    if self.dialect.NUMBERS_CAN_BE_UNDERSCORE_SEPARATED and replaced.isdigit():
+                        return self._add(TokenType.NUMBER, number_text + replaced)
+                    if self.dialect.IDENTIFIERS_CAN_START_WITH_DIGIT:
+                        return self._add(TokenType.VAR)
 
                 self._advance(-len(literal))
                 return self._add(TokenType.NUMBER, number_text)
diff --git a/sqlglotrs/src/settings.rs b/sqlglotrs/src/settings.rs
index 7bc4882e3b..d49776d5db 100644
--- a/sqlglotrs/src/settings.rs
+++ b/sqlglotrs/src/settings.rs
@@ -171,6 +171,7 @@ impl TokenizerSettings {
 pub struct TokenizerDialectSettings {
     pub unescaped_sequences: HashMap<String, String>,
     pub identifiers_can_start_with_digit: bool,
+    pub numbers_can_be_underscore_separated: bool,
 }
 
 #[pymethods]
@@ -179,10 +180,12 @@ impl TokenizerDialectSettings {
     pub fn new(
         unescaped_sequences: HashMap<String, String>,
         identifiers_can_start_with_digit: bool,
+        numbers_can_be_underscore_separated: bool,
     ) -> Self {
         TokenizerDialectSettings {
             unescaped_sequences,
             identifiers_can_start_with_digit,
+            numbers_can_be_underscore_separated,
         }
     }
 }
diff --git a/sqlglotrs/src/tokenizer.rs b/sqlglotrs/src/tokenizer.rs
index 8228b5a8c5..9aec50f677 100644
--- a/sqlglotrs/src/tokenizer.rs
+++ b/sqlglotrs/src/tokenizer.rs
@@ -531,10 +531,14 @@ impl<'a> TokenizerState<'a> {
                     )
                     .map(|x| *x);
 
+                let replaced = literal.replace("_", "");
+
                 if let Some(unwrapped_token_type) = token_type {
                     self.add(self.token_types.number, Some(number_text))?;
                     self.add(self.token_types.dcolon, Some("::".to_string()))?;
                     self.add(unwrapped_token_type, Some(literal))?;
+                } else if self.dialect_settings.numbers_can_be_underscore_separated && self.is_numeric(&replaced) {
+                    self.add(self.token_types.number, Some(number_text + &replaced))?;
                 } else if self.dialect_settings.identifiers_can_start_with_digit {
                     self.add(self.token_types.var, None)?;
                 } else {
@@ -706,6 +710,10 @@ impl<'a> TokenizerState<'a> {
         )
     }
 
+    fn is_numeric(&mut self, s: &str) -> bool {
+        s.chars().all(|c| c.is_digit(10))
+    }
+
     fn extract_value(&mut self) -> Result<String, TokenizerError> {
         loop {
             if !self.peek_char.is_whitespace()


Test patch:
diff --git a/tests/dialects/test_clickhouse.py b/tests/dialects/test_clickhouse.py
index d3d363eac6..49705d51fa 100644
--- a/tests/dialects/test_clickhouse.py
+++ b/tests/dialects/test_clickhouse.py
@@ -549,6 +549,9 @@ def test_clickhouse(self):
             "SELECT name FROM data WHERE NOT ((SELECT DISTINCT name FROM data) IS NULL)",
         )
 
+        self.validate_identity("SELECT 1_2_3_4_5", "SELECT 12345")
+        self.validate_identity("SELECT 1_b", "SELECT 1_b")
+
     def test_clickhouse_values(self):
         values = exp.select("*").from_(
             exp.values([exp.tuple_(1, 2, 3)], alias="subq", columns=["a", "b", "c"])
diff --git a/tests/dialects/test_hive.py b/tests/dialects/test_hive.py
index c569d9657f..d0b881e412 100644
--- a/tests/dialects/test_hive.py
+++ b/tests/dialects/test_hive.py
@@ -806,6 +806,8 @@ def test_hive(self):
             },
         )
 
+        self.validate_identity("SELECT 1_2")
+
     def test_escapes(self) -> None:
         self.validate_identity("'\n'", "'\\n'")
         self.validate_identity("'\\n'")
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 1bf951a65b..20967b9de0 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -888,3 +888,15 @@ def test_drop_column(self):
         ast = parse_one("ALTER TABLE tbl DROP COLUMN col")
         self.assertEqual(len(list(ast.find_all(exp.Table))), 1)
         self.assertEqual(len(list(ast.find_all(exp.Column))), 1)
+
+    def test_udf_meta(self):
+        ast = parse_one("YEAR(a) /* sqlglot.anonymous */")
+        self.assertIsInstance(ast, exp.Anonymous)
+
+        # Meta flag is case sensitive
+        ast = parse_one("YEAR(a) /* sqlglot.anONymous */")
+        self.assertIsInstance(ast, exp.Year)
+
+        # Incomplete or incorrect anonymous meta comments are not registered
+        ast = parse_one("YEAR(a) /* sqlglot.anon */")
+        self.assertIsInstance(ast, exp.Year)