Repository: reata/sqllineage

Issue:
Clickhouse SQL 'GLOBAL IN' not support
Clickhouse SQL 'GLOBAL IN' not support

sqllineage -f ./sql-global-ck.sql --dialect=clickhouse

```sql
SELECT id, name FROM distributed_table WHERE id GLOBAL IN (SELECT id FROM local_table);
```

Line 1, Position 49: Found unparsable section: 'GLOBAL IN (SELECT id FROM local_table)'


Python 3.8
Sqllineage 1.5.0



Discussion:
This is not supported by the underlying sqlfluff, not by sqllineage.
Upstream issue https://github.com/sqlfluff/sqlfluff/issues/5256 as reference.

Once they fix the issue and release a new version, we can upgrade the dependency and get this done.
Upstream issue is fixed in sqlfluff==3.0.0a5. Since 3.0.0a5 is an alpha release, we will not upgrade sqllineage dependency and will wait for 3.0.0 instead. But you can manually upgrade your sqlfluff to get unblocked.



Patch:
diff --git a/setup.py b/setup.py
index 12bc4acd..ba421ddb 100644
--- a/setup.py
+++ b/setup.py
@@ -68,7 +68,7 @@ def run(self) -> None:
     install_requires=[
         "sqlparse==0.4.4",
         "networkx>=2.4",
-        "sqlfluff==2.3.5",
+        "sqlfluff==3.0.3",
         "sqlalchemy>=2.0.0",
     ],
     entry_points={"console_scripts": ["sqllineage = sqllineage.cli:main"]},
diff --git a/sqllineage/core/parser/sqlfluff/utils.py b/sqllineage/core/parser/sqlfluff/utils.py
index 09a7559e..bdeaa06c 100644
--- a/sqllineage/core/parser/sqlfluff/utils.py
+++ b/sqllineage/core/parser/sqlfluff/utils.py
@@ -59,20 +59,12 @@ def find_from_expression_element(segment: BaseSegment) -> Optional[BaseSegment]:
         from_clause as grandparent
         from_expression/join_clause as parent
     """
-    from_expression_element = None
-    if segment.type in ["from_clause", "update_statement"]:
-        if from_expression := segment.get_child("from_expression"):
-            non_bracket = from_expression
-            while bracketed := non_bracket.get_child("bracketed"):
-                non_bracket = bracketed
-            if seg := non_bracket.get_child("from_expression_element"):
-                from_expression_element = seg
-            elif seg := non_bracket.get_child("from_expression"):
-                if sub_seg := seg.get_child("from_expression_element"):
-                    from_expression_element = sub_seg
-    elif segment.type in ("from_expression", "join_clause"):
-        if seg := segment.get_child("from_expression_element"):
-            from_expression_element = seg
+    try:
+        from_expression_element = next(
+            segment.recursive_crawl("from_expression_element")
+        )
+    except StopIteration:
+        from_expression_element = None
     return from_expression_element
 
 
@@ -94,14 +86,8 @@ def list_join_clause(segment: BaseSegment) -> List[BaseSegment]:
     """
     traverse from_clause, recursively goes into bracket by default
     """
-    if from_expression := segment.get_child("from_expression"):
-        if bracketed := from_expression.get_child("bracketed"):
-            join_clauses = bracketed.get_children("join_clause")
-            if inner_bracket := bracketed.get_child("bracketed"):
-                join_clauses = list_join_clause(inner_bracket) + join_clauses
-            return join_clauses
-        else:
-            return from_expression.get_children("join_clause")
+    if segment.type in ["from_clause", "update_statement"]:
+        return list(segment.recursive_crawl("join_clause"))
     return []
 
 


Test patch:
diff --git a/tests/core/test_parser.py b/tests/core/test_parser.py
index 777317ca..b24b9c15 100644
--- a/tests/core/test_parser.py
+++ b/tests/core/test_parser.py
@@ -1,9 +1,17 @@
 from unittest.mock import Mock
 
+from sqlfluff.core import Linter
+
 from sqllineage.core.parser.sqlfluff.models import SqlFluffColumn
+from sqllineage.core.parser.sqlfluff.utils import find_from_expression_element
 
 
 def test_column_extract_source_columns():
     segment_mock = Mock()
     segment_mock.type = ""
     assert [] == SqlFluffColumn._extract_source_columns(segment_mock)
+
+
+def test_return_none_find_from_expression_element():
+    file_segment = Linter(dialect="ansi").parse_string("TRUNCATE TABLE tab").tree
+    assert find_from_expression_element(file_segment) is None
diff --git a/tests/sql/table/test_select.py b/tests/sql/table/test_select.py
index ae70a270..fe143dcc 100644
--- a/tests/sql/table/test_select.py
+++ b/tests/sql/table/test_select.py
@@ -107,8 +107,33 @@ def test_select_parenthesized_from_table_join_recursive():
     sql = """SELECT tab1.id
 FROM ((tab1
     LEFT JOIN tab2 ON tab1.id = tab2.id)
-    LEFT JOIN tab3 ON tab1.id = tab3.id)"""
+    LEFT JOIN tab3 ON tab1.id = tab3.id)
+    """
     assert_table_lineage_equal(sql, {"tab1", "tab2", "tab3"})
+    sql = """SELECT tab1.id
+FROM (((tab1
+    LEFT JOIN tab2 ON tab1.id = tab2.id)
+    LEFT JOIN tab3 ON tab1.id = tab3.id)
+    LEFT JOIN tab4 ON tab1.id = tab4.id)
+    """
+    assert_table_lineage_equal(sql, {"tab1", "tab2", "tab3", "tab4"})
+    sql = """SELECT tab1.id
+FROM ((((tab1
+    LEFT JOIN tab2 ON tab1.id = tab2.id)
+    LEFT JOIN tab3 ON tab1.id = tab3.id)
+    LEFT JOIN tab4 ON tab1.id = tab4.id)
+    LEFT JOIN tab5 ON tab1.id = tab5.id)
+    """
+    assert_table_lineage_equal(sql, {"tab1", "tab2", "tab3", "tab4", "tab5"})
+    sql = """SELECT tab1.id
+FROM (((((tab1
+    LEFT JOIN tab2 ON tab1.id = tab2.id)
+    LEFT JOIN tab3 ON tab1.id = tab3.id)
+    LEFT JOIN tab4 ON tab1.id = tab4.id)
+    LEFT JOIN tab5 ON tab1.id = tab5.id)
+    LEFT JOIN tab6 ON tab1.id = tab6.id)
+        """
+    assert_table_lineage_equal(sql, {"tab1", "tab2", "tab3", "tab4", "tab5", "tab6"})
 
 
 def test_select_subquery():