Repository: TobikoData/sqlmesh
Pull request #3263 (2024-10-16 23:09:37)

Fix: Prevent extraction of dependencies from a rendered query for dbt models
The previous workaround, which involved manually setting the `_full_depends_on` cache when converting a dbt model, didn’t work well because this logic didn’t apply when the model definition was loaded from the state, which can happen when a selector is used.


Patch:
diff --git a/sqlmesh/core/model/definition.py b/sqlmesh/core/model/definition.py
index c29c7ce3..8fbe82c6 100644
--- a/sqlmesh/core/model/definition.py
+++ b/sqlmesh/core/model/definition.py
@@ -116,12 +116,14 @@ class _Model(ModelMeta, frozen=True):
         clustered_by: The cluster columns, only applicable in certain engines. (eg. (ds, hour))
         python_env: Dictionary containing all global variables needed to render the model's macros.
         mapping_schema: The schema of table names to column and types.
+        extract_dependencies_from_query: Whether to extract additional dependencies from the rendered model's query.
         physical_schema_override: The desired physical schema name override.
     """
 
     python_env_: t.Optional[t.Dict[str, Executable]] = Field(default=None, alias="python_env")
     jinja_macros: JinjaMacroRegistry = JinjaMacroRegistry()
     mapping_schema: t.Dict[str, t.Any] = {}
+    extract_dependencies_from_query: bool = True
 
     _full_depends_on: t.Optional[t.Set[str]] = None
     _statement_renderer_cache: t.Dict[int, ExpressionRenderer] = {}
@@ -969,6 +971,8 @@ class _Model(ModelMeta, frozen=True):
 
     @property
     def full_depends_on(self) -> t.Set[str]:
+        if not self.extract_dependencies_from_query:
+            return self.depends_on_ or set()
         if self._full_depends_on is None:
             depends_on = self.depends_on_ or set()
 
diff --git a/sqlmesh/dbt/model.py b/sqlmesh/dbt/model.py
index 2f1b2428..c864cd7f 100644
--- a/sqlmesh/dbt/model.py
+++ b/sqlmesh/dbt/model.py
@@ -428,14 +428,13 @@ class ModelConfig(BaseModelConfig):
             dialect=model_dialect,
             kind=self.model_kind(context),
             start=self.start,
+            # This ensures that we bypass query rendering that would otherwise be required to extract additional
+            # dependencies from the model's SQL.
+            # Note: any table dependencies that are not referenced using the `ref` macro will not be included.
+            extract_dependencies_from_query=False,
             **optional_kwargs,
             **model_kwargs,
         )
-        # Prepopulate the _full_depends_on cache with dependencies sourced directly from the manifest.
-        # This ensures that we bypass query rendering that would otherwise be required to extract additional
-        # dependencies from the model's SQL.
-        # Note: any table dependencies that are not referenced using the `ref` macro will not be included.
-        model._full_depends_on = model.depends_on_
         return model
 
     def _dbt_max_partition_blob(self) -> t.Optional[str]:


Test patch:
diff --git a/tests/core/test_snapshot.py b/tests/core/test_snapshot.py
index d995457f..6e08bcf4 100644
--- a/tests/core/test_snapshot.py
+++ b/tests/core/test_snapshot.py
@@ -139,6 +139,7 @@ def test_json(snapshot: Snapshot):
             "allow_partials": False,
             "signals": [],
             "enabled": True,
+            "extract_dependencies_from_query": True,
         },
         "audits": [],
         "name": '"name"',
diff --git a/tests/schedulers/airflow/test_client.py b/tests/schedulers/airflow/test_client.py
index 6471c125..9618f6cf 100644
--- a/tests/schedulers/airflow/test_client.py
+++ b/tests/schedulers/airflow/test_client.py
@@ -132,6 +132,7 @@ def test_apply_plan(mocker: MockerFixture, snapshot: Snapshot):
                         "allow_partials": False,
                         "signals": [],
                         "enabled": True,
+                        "extract_dependencies_from_query": True,
                     },
                     "audits": [],
                     "name": '"test_model"',