Repository: reata/sqllineage

Issue:
SQLfluff Not read nested configs.
**Describe the bug**

SQLfluff can read nested configs and override values from them. But sqllineage use wrong read config function [here](https://github.com/reata/sqllineage/blob/master/sqllineage/core/parser/sqlfluff/analyzer.py#L83).

`FluffConfig.from_root` can read only local config from cwd. Intead of that, I think, we shoud use `FluffConfig.from_path` and throw the path of validating file. In that case, sqlfluff will read all configs from cwd to file location.


**To Reproduce**

Create some directories like `folder1/folder2/folder3` 
In `folder1` and `folder3` create `.sqlfluff` configs.
Add `foo.sql` to `folder3`


Use sql query with mocks like `SELECT * FROM {{ source_table }}`
Add `source_table` value to `/folder1/folder2/folder3/.sqlfluff` and leave `/folder1/.sqlfluff` empty.


Run 
`sqllineage -d athena -f  /folder1/folder2/folder3/foo.sql` 

As result we will get Error `Line 1, Position 10: Found unparsable section: 'FROM'`

**Expected behavior**

`Sqllineage` uses `sqlfluff` correctly to read neasted configs.

**Additional context**

To fix that issue we have to change [this function](https://github.com/reata/sqllineage/blob/master/sqllineage/core/parser/sqlfluff/analyzer.py#L83) to `FluffConfig.from_path(path=file_path, overrides={"dialect": self._dialect})` and throw validating file path.

Discussion:
codecov[bot]: ## [Codecov](https://app.codecov.io/gh/reata/sqllineage/pull/637?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 99.55%. Comparing base [(`10442e5`)](https://app.codecov.io/gh/reata/sqllineage/commit/10442e52aed3d937b252293d556ed1d0afa8b622?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None) to head [(`c37c9ae`)](https://app.codecov.io/gh/reata/sqllineage/commit/c37c9ae5eeca3d30e431cec7f3e128295125d9ef?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None).

<details><summary>Additional details and impacted files</summary>


```diff
@@           Coverage Diff           @@
##           master     #637   +/-   ##
=======================================
  Coverage   99.55%   99.55%           
=======================================
  Files          41       41           
  Lines        2237     2245    +8     
=======================================
+ Hits         2227     2235    +8     
  Misses         10       10           
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/reata/sqllineage/pull/637?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None).


Patch:
diff --git a/sqllineage/cli.py b/sqllineage/cli.py
index 857de5c..e4630a0 100644
--- a/sqllineage/cli.py
+++ b/sqllineage/cli.py
@@ -16,7 +16,7 @@ from sqllineage.core.metadata.sqlalchemy import SQLAlchemyMetaDataProvider
 from sqllineage.drawing import draw_lineage_graph
 from sqllineage.runner import LineageRunner
 from sqllineage.utils.constant import LineageLevel
-from sqllineage.utils.helpers import extract_sql_from_args
+from sqllineage.utils.helpers import extract_file_path_from_args, extract_sql_from_args
 
 logger = logging.getLogger(__name__)
 
@@ -108,8 +108,10 @@ def main(args=None) -> None:
         warnings.warn("Both -e and -f options are specified. -e option will be ignored")
     if args.f or args.e:
         sql = extract_sql_from_args(args)
+        file_path = extract_file_path_from_args(args)
         runner = LineageRunner(
             sql,
+            file_path=file_path,
             dialect=args.dialect,
             metadata_provider=metadata_provider,
             verbose=args.verbose,
diff --git a/sqllineage/core/parser/sqlfluff/analyzer.py b/sqllineage/core/parser/sqlfluff/analyzer.py
index 74e0ace..9fd3b10 100644
--- a/sqllineage/core/parser/sqlfluff/analyzer.py
+++ b/sqllineage/core/parser/sqlfluff/analyzer.py
@@ -27,8 +27,10 @@ class SqlFluffLineageAnalyzer(LineageAnalyzer):
     PARSER_NAME = "sqlfluff"
     SUPPORTED_DIALECTS = list(dialect.label for dialect in dialect_readout())
 
-    def __init__(self, dialect: str, silent_mode: bool = False):
-        self._dialect = dialect
+    def __init__(self, file_path: str, dialect: str, silent_mode: bool = False):
+        self._sqlfluff_config = FluffConfig.from_path(
+            path=file_path, overrides={"dialect": dialect}
+        )
         self._silent_mode = silent_mode
         self.tsql_split_cache: Dict[str, BaseSegment] = {}
 
@@ -57,7 +59,7 @@ class SqlFluffLineageAnalyzer(LineageAnalyzer):
         else:
             statement_segment = statement_segments[0]
             for extractor in [
-                extractor_cls(self._dialect, metadata_provider)
+                extractor_cls(self._sqlfluff_config.get("dialect"), metadata_provider)
                 for extractor_cls in BaseExtractor.__subclasses__()
             ]:
                 if extractor.can_extract(statement_segment.type):
@@ -79,9 +81,7 @@ class SqlFluffLineageAnalyzer(LineageAnalyzer):
                     )
 
     def _list_specific_statement_segment(self, sql: str):
-        parsed = Linter(
-            config=FluffConfig.from_root(overrides={"dialect": self._dialect})
-        ).parse_string(sql)
+        parsed = Linter(config=self._sqlfluff_config).parse_string(sql)
         violations = [
             str(e)
             for e in parsed.violations
diff --git a/sqllineage/runner.py b/sqllineage/runner.py
index 9d666b5..113d036 100644
--- a/sqllineage/runner.py
+++ b/sqllineage/runner.py
@@ -42,6 +42,7 @@ class LineageRunner(object):
         verbose: bool = False,
         silent_mode: bool = False,
         draw_options: Optional[Dict[str, Any]] = None,
+        file_path: str = ".",
     ):
         """
         The entry point of SQLLineage after command line options are parsed.
@@ -51,6 +52,7 @@ class LineageRunner(object):
         :param metadata_provider: metadata service object providing table schema
         :param verbose: verbose flag indicating whether statement-wise lineage result will be shown
         :param silent_mode: boolean flag indicating whether to skip lineage analysis for unknown statement types
+        :param file_path: path of the SQL file.
         """
         if dialect == SQLPARSE_DIALECT:
             warnings.warn(
@@ -60,6 +62,7 @@ class LineageRunner(object):
                 stacklevel=2,
             )
         self._sql = sql
+        self._file_path = file_path
         self._verbose = verbose
         self._draw_options = draw_options if draw_options else {}
         self._evaluated = False
@@ -183,7 +186,9 @@ Target Tables:
         analyzer = (
             SqlParseLineageAnalyzer()
             if self._dialect == SQLPARSE_DIALECT
-            else SqlFluffLineageAnalyzer(self._dialect, self._silent_mode)
+            else SqlFluffLineageAnalyzer(
+                self._file_path, self._dialect, self._silent_mode
+            )
         )
         if SQLLineageConfig.TSQL_NO_SEMICOLON and self._dialect == "tsql":
             self._stmt = analyzer.split_tsql(self._sql.strip())
diff --git a/sqllineage/utils/helpers.py b/sqllineage/utils/helpers.py
index df9c3ed..86fbbda 100644
--- a/sqllineage/utils/helpers.py
+++ b/sqllineage/utils/helpers.py
@@ -46,6 +46,13 @@ def extract_sql_from_args(args: Namespace) -> str:
     return sql
 
 
+def extract_file_path_from_args(args: Namespace) -> str:
+    file_path = "."
+    if getattr(args, "f", None):
+        file_path = args.f
+    return file_path
+
+
 def split(sql: str) -> List[str]:
     # TODO: we need a parser independent split function
     import sqlparse


Test patch:
diff --git a/tests/core/test_runner.py b/tests/core/test_runner.py
index 23210cf..aec0d52 100644
--- a/tests/core/test_runner.py
+++ b/tests/core/test_runner.py
@@ -1,6 +1,7 @@
 import os
 import tempfile
 
+from sqllineage.cli import main
 from sqllineage.core.models import SubQuery
 from sqllineage.runner import LineageRunner
 from sqllineage.utils.constant import LineageLevel
@@ -51,3 +52,24 @@ tbl_name=my_table"""
             assert_table_lineage_equal(sql, {"my_table"}, test_sqlparse=False)
         finally:
             os.chdir(cwd)
+
+
+def test_respect_nested_sqlfluff_configuration_file():
+    sqlfluff_config = """[sqlfluff:templater:jinja:context]
+num_things=456
+tbl_name=my_table"""
+    cwd = os.getcwd()
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        try:
+            os.chdir(tmpdirname)
+            nested_dir = os.path.join(tmpdirname, "nested_dir")
+            os.mkdir(nested_dir)
+            with open(os.path.join(nested_dir, ".sqlfluff"), "w") as f:
+                f.write(sqlfluff_config)
+            with open(os.path.join(nested_dir, "nested.sql"), "w") as f:
+                f.write(
+                    "SELECT {{ num_things }} FROM {{ tbl_name }} WHERE id > 10 LIMIT 5"
+                )
+            main(["-f", nested_dir + "/nested.sql"])
+        finally:
+            os.chdir(cwd)