Repository: TobikoData/sqlmesh
Pull request #3233 (2024-10-09 16:45:53)

Fix: Modify dlt query filter not to use alias reference
Fix issue with generated dlt incremental query for dialects that don't support using alias reference for filtering by using the table column from the from clause


Patch:
diff --git a/sqlmesh/core/loader.py b/sqlmesh/core/loader.py
index 1b2f2632..beb8d035 100644
--- a/sqlmesh/core/loader.py
+++ b/sqlmesh/core/loader.py
@@ -93,11 +93,18 @@ class Loader(abc.ABC):
         self._config_mtimes = {path: max(mtimes) for path, mtimes in config_mtimes.items()}
 
         macros, jinja_macros = self._load_scripts()
+
+        # All macros need to be known at parse time
+        standard_macros = macro.get_registry()
+        macro.set_registry(macros)
+
         audits = self._load_audits(macros=macros, jinja_macros=jinja_macros)
         models = self._load_models(
             macros, jinja_macros, context.gateway or context.config.default_gateway, audits or None
         )
 
+        macro.set_registry(standard_macros)
+
         for model in models.values():
             self._add_model_to_dag(model)
 
diff --git a/sqlmesh/integrations/dlt.py b/sqlmesh/integrations/dlt.py
index a26669f1..942259a3 100644
--- a/sqlmesh/integrations/dlt.py
+++ b/sqlmesh/integrations/dlt.py
@@ -111,7 +111,7 @@ SELECT
 FROM
   {from_table}
 WHERE
-  _dlt_load_time BETWEEN @start_ds AND @end_ds
+  {time_column} BETWEEN @start_ds AND @end_ds
 """
 
 


Test patch:
diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py
index 8cbad11d..7a55b4ba 100644
--- a/tests/cli/test_cli.py
+++ b/tests/cli/test_cli.py
@@ -728,7 +728,7 @@ SELECT
 FROM
   sushi_dataset.sushi_types
 WHERE
-  _dlt_load_time BETWEEN @start_ds AND @end_ds
+  TO_TIMESTAMP(CAST(_dlt_load_id AS DOUBLE)) BETWEEN @start_ds AND @end_ds
 """
 
     with open(tmp_path / "models/incremental_sushi_types.sql") as file:
@@ -759,7 +759,7 @@ SELECT
 FROM
   sushi_dataset._dlt_loads
 WHERE
-  _dlt_load_time BETWEEN @start_ds AND @end_ds
+  TO_TIMESTAMP(CAST(load_id AS DOUBLE)) BETWEEN @start_ds AND @end_ds
 """
 
     with open(tmp_path / "models/incremental__dlt_loads.sql") as file:
diff --git a/tests/core/test_model.py b/tests/core/test_model.py
index b62abf4e..279701c1 100644
--- a/tests/core/test_model.py
+++ b/tests/core/test_model.py
@@ -5922,3 +5922,38 @@ def test_cache():
     model = load_sql_based_model(expressions)
     assert model.depends_on == {'"y"'}
     assert model.copy(update={"depends_on_": {'"z"'}}).depends_on == {'"z"', '"y"'}
+
+
+def test_snowflake_macro_func_as_table(tmp_path: Path):
+    init_example_project(tmp_path, dialect="duckdb")
+
+    custom_macro_file = tmp_path / "macros/custom_macros.py"
+    custom_macro_file.parent.mkdir(parents=True, exist_ok=True)
+    custom_macro_file.write_text("""
+from sqlmesh import macro
+
+@macro()
+def custom_macro(evaluator, arg1, arg2):
+    return f"{arg1}{arg2}"
+    """)
+
+    new_snowflake_model_file = tmp_path / "models/new_model.sql"
+    new_snowflake_model_file.parent.mkdir(parents=True, exist_ok=True)
+    new_snowflake_model_file.write_text("""
+MODEL (
+  name sqlmesh_example.test,
+  dialect snowflake,
+);
+
+@DEF(foo, foo);
+@DEF(bar, bar);
+
+SELECT * FROM @custom_macro(@foo, @bar)
+    """)
+
+    config = Config(model_defaults=ModelDefaultsConfig(dialect="duckdb"))
+    context = Context(paths=tmp_path, config=config)
+
+    query = context.get_model("sqlmesh_example.test").render_query()
+
+    assert t.cast(exp.Query, query).sql("snowflake") == 'SELECT * FROM "FOOBAR" AS "FOOBAR"'