Repository: TobikoData/sqlmesh
Pull request #3194 (2024-09-27 15:59:09)

Fix: Invalidate the optimized query cache entry when a macro definition changes within the model model definition
Model example:
```
MODEL (
  name agreement.stg_testing,
  kind FULL,
);

@DEF(filter_, txmetasource IN ('A'));

SELECT
  *
FROM iceberg.dbo.table1
WHERE
  @filter_
LIMIT 10
```
Prior to this fix, changing the `filter_` macro resulted in a metadata change unless the cache was manually cleaned up.


Patch:
diff --git a/sqlmesh/core/model/cache.py b/sqlmesh/core/model/cache.py
index 7b31a988..198569fc 100644
--- a/sqlmesh/core/model/cache.py
+++ b/sqlmesh/core/model/cache.py
@@ -133,6 +133,7 @@ class OptimizedQueryCache:
     def _entry_name(model: SqlModel) -> str:
         hash_data = _mapping_schema_hash_data(model.mapping_schema)
         hash_data.append(gen(model.query))
+        hash_data.append(str([gen(d) for d in model.macro_definitions]))
         hash_data.append(str([(k, v) for k, v in model.sorted_python_env]))
         hash_data.extend(model.jinja_macros.data_hash_values)
         return f"{model.name}_{crc32(hash_data)}"


Test patch:
diff --git a/tests/utils/test_cache.py b/tests/utils/test_cache.py
index baf29dbf..cd1fdb01 100644
--- a/tests/utils/test_cache.py
+++ b/tests/utils/test_cache.py
@@ -1,9 +1,11 @@
+import typing as t
 from pathlib import Path
 
 from pytest_mock.plugin import MockerFixture
 from sqlglot import parse_one
 
-from sqlmesh.core.model import SqlModel
+from sqlmesh.core import dialect as d
+from sqlmesh.core.model import SqlModel, load_sql_based_model
 from sqlmesh.core.model.cache import OptimizedQueryCache
 from sqlmesh.utils.cache import FileCache
 from sqlmesh.utils.pydantic import PydanticModel
@@ -79,3 +81,52 @@ def test_optimized_query_cache_missing_rendered_query(tmp_path: Path, mocker: Mo
 
     assert model._query_renderer._cache == [None]
     assert model._query_renderer._optimized_cache is None
+
+
+def test_optimized_query_cache_macro_def_change(tmp_path: Path, mocker: MockerFixture):
+    expressions = d.parse(
+        """
+        MODEL (name db.table);
+
+        @DEF(filter_, a = 1);
+
+        SELECT a FROM (SELECT 1 AS a) WHERE @filter_;
+        """
+    )
+    model = t.cast(SqlModel, load_sql_based_model(expressions))
+
+    cache = OptimizedQueryCache(tmp_path)
+
+    assert not cache.with_optimized_query(model)
+
+    model._query_renderer._cache = []
+    model._query_renderer._optimized_cache = None
+
+    assert cache.with_optimized_query(model)
+    assert (
+        model.render_query_or_raise().sql()
+        == 'SELECT "_q_0"."a" AS "a" FROM (SELECT 1 AS "a") AS "_q_0" WHERE "_q_0"."a" = 1'
+    )
+
+    # Change the filter_ definition
+    new_expressions = d.parse(
+        """
+        MODEL (name db.table);
+
+        @DEF(filter_, a = 2);
+
+        SELECT a FROM (SELECT 1 AS a) WHERE @filter_;
+        """
+    )
+    new_model = t.cast(SqlModel, load_sql_based_model(new_expressions))
+
+    assert not cache.with_optimized_query(new_model)
+
+    new_model._query_renderer._cache = []
+    new_model._query_renderer._optimized_cache = None
+
+    assert cache.with_optimized_query(new_model)
+    assert (
+        new_model.render_query_or_raise().sql()
+        == 'SELECT "_q_0"."a" AS "a" FROM (SELECT 1 AS "a") AS "_q_0" WHERE "_q_0"."a" = 2'
+    )