nebius/SWE-rebench-V2-PRs huggingface dataset:
Repository: TobikoData/sqlmesh
Pull request #2343 (2024-03-26 15:32:20)

Fix: Use a correct deoployability index when running the scheduler against non-prod environments
I've realized that deployability index would be wrong when running the `sqlmesh run` command against non-prod environments.


Patch:
diff --git a/sqlmesh/core/model/definition.py b/sqlmesh/core/model/definition.py
index 46c3e38e..56e2b1f6 100644
--- a/sqlmesh/core/model/definition.py
+++ b/sqlmesh/core/model/definition.py
@@ -755,6 +755,7 @@ class _Model(ModelMeta, frozen=True):
             json.dumps(self.column_descriptions, sort_keys=True),
             self.cron,
             str(self.start) if self.start else None,
+            str(self.end) if self.end else None,
             str(self.retention) if self.retention else None,
             str(self.batch_size) if self.batch_size is not None else None,
             json.dumps(self.mapping_schema, sort_keys=True),
diff --git a/sqlmesh/core/scheduler.py b/sqlmesh/core/scheduler.py
index 634d4d3b..d5a2fcaa 100644
--- a/sqlmesh/core/scheduler.py
+++ b/sqlmesh/core/scheduler.py
@@ -5,6 +5,7 @@ import traceback
 import typing as t
 from datetime import datetime
 
+from sqlmesh.core import constants as c
 from sqlmesh.core.console import Console, get_console
 from sqlmesh.core.environment import EnvironmentNamingInfo
 from sqlmesh.core.model import SeedModel
@@ -243,7 +244,11 @@ class Scheduler:
         else:
             environment_naming_info = environment
 
-        deployability_index = deployability_index or DeployabilityIndex.all_deployable()
+        deployability_index = deployability_index or (
+            DeployabilityIndex.create(self.snapshots.values())
+            if environment_naming_info.name != c.PROD
+            else DeployabilityIndex.all_deployable()
+        )
         execution_time = execution_time or now()
         batches = self.batches(
             start,


Test patch:
diff --git a/tests/core/test_snapshot.py b/tests/core/test_snapshot.py
index 8ad01eb8..88ae4a78 100644
--- a/tests/core/test_snapshot.py
+++ b/tests/core/test_snapshot.py
@@ -606,7 +606,7 @@ def test_fingerprint(model: Model, parent_model: Model):
 
     original_fingerprint = SnapshotFingerprint(
         data_hash="372035512",
-        metadata_hash="3575901791",
+        metadata_hash="490962550",
     )
 
     assert fingerprint == original_fingerprint
@@ -666,7 +666,7 @@ def test_fingerprint_seed_model():
 
     expected_fingerprint = SnapshotFingerprint(
         data_hash="2156038176",
-        metadata_hash="3058233394",
+        metadata_hash="3304326900",
     )
 
     model = load_sql_based_model(expressions, path=Path("./examples/sushi/models/test_model.sql"))
@@ -705,7 +705,7 @@ def test_fingerprint_jinja_macros(model: Model):
     )
     original_fingerprint = SnapshotFingerprint(
         data_hash="900027747",
-        metadata_hash="3575901791",
+        metadata_hash="490962550",
     )
 
     fingerprint = fingerprint_from_node(model, nodes={})
Repository: TobikoData/sqlmesh
Pull request #2344 (2024-03-26 16:10:30)

fix!: include end in metadata hash
Missed in the original PR: https://github.com/TobikoData/sqlmesh/pull/2287

Breaking because we need to calculate new fingerprints. 


Patch:
diff --git a/sqlmesh/core/model/definition.py b/sqlmesh/core/model/definition.py
index 46c3e38e..56e2b1f6 100644
--- a/sqlmesh/core/model/definition.py
+++ b/sqlmesh/core/model/definition.py
@@ -755,6 +755,7 @@ class _Model(ModelMeta, frozen=True):
             json.dumps(self.column_descriptions, sort_keys=True),
             self.cron,
             str(self.start) if self.start else None,
+            str(self.end) if self.end else None,
             str(self.retention) if self.retention else None,
             str(self.batch_size) if self.batch_size is not None else None,
             json.dumps(self.mapping_schema, sort_keys=True),


Test patch:
diff --git a/tests/core/test_snapshot.py b/tests/core/test_snapshot.py
index 8ad01eb8..88ae4a78 100644
--- a/tests/core/test_snapshot.py
+++ b/tests/core/test_snapshot.py
@@ -606,7 +606,7 @@ def test_fingerprint(model: Model, parent_model: Model):
 
     original_fingerprint = SnapshotFingerprint(
         data_hash="372035512",
-        metadata_hash="3575901791",
+        metadata_hash="490962550",
     )
 
     assert fingerprint == original_fingerprint
@@ -666,7 +666,7 @@ def test_fingerprint_seed_model():
 
     expected_fingerprint = SnapshotFingerprint(
         data_hash="2156038176",
-        metadata_hash="3058233394",
+        metadata_hash="3304326900",
     )
 
     model = load_sql_based_model(expressions, path=Path("./examples/sushi/models/test_model.sql"))
@@ -705,7 +705,7 @@ def test_fingerprint_jinja_macros(model: Model):
     )
     original_fingerprint = SnapshotFingerprint(
         data_hash="900027747",
-        metadata_hash="3575901791",
+        metadata_hash="490962550",
     )
 
     fingerprint = fingerprint_from_node(model, nodes={})
Repository: TobikoData/sqlmesh
Pull request #2371 (2024-03-30 01:24:52)

fix: don't warn about state


Patch:
diff --git a/sqlmesh/core/config/connection.py b/sqlmesh/core/config/connection.py
index a8217525..26f36aca 100644
--- a/sqlmesh/core/config/connection.py
+++ b/sqlmesh/core/config/connection.py
@@ -36,7 +36,7 @@ else:
 
 logger = logging.getLogger(__name__)
 
-RECOMMENDED_STATE_SYNC_ENGINES = {"postgres", "gcp_postgres", "mysql"}
+RECOMMENDED_STATE_SYNC_ENGINES = {"postgres", "gcp_postgres", "mysql", "duckdb"}
 
 
 class ConnectionConfig(abc.ABC, BaseConfig):
diff --git a/sqlmesh/core/config/scheduler.py b/sqlmesh/core/config/scheduler.py
index 6244efa2..8934142f 100644
--- a/sqlmesh/core/config/scheduler.py
+++ b/sqlmesh/core/config/scheduler.py
@@ -79,10 +79,6 @@ class _EngineAdapterStateSyncSchedulerConfig(_SchedulerConfig):
                 f"The {engine_adapter.DIALECT.upper()} engine cannot be used to store SQLMesh state - please specify a different `state_connection` engine."
                 + " See https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#gateways for more information."
             )
-        if not state_connection.is_recommended_for_state_sync:
-            logger.warning(
-                f"{state_connection.type_} is not recommended to be used as a state sync for production deployments. Please see documentation ( https://sqlmesh.readthedocs.io/en/latest/guides/configuration/#state-connection) for list of recommended engines to be used for storing state and further details."
-            )
         schema = context.config.get_state_schema(context.gateway)
         return EngineAdapterStateSync(
             engine_adapter, schema=schema, context_path=context.path, console=context.console


Test patch:
diff --git a/tests/core/test_connection_config.py b/tests/core/test_connection_config.py
index 7fd81f0d..3fa6624c 100644
--- a/tests/core/test_connection_config.py
+++ b/tests/core/test_connection_config.py
@@ -380,7 +380,7 @@ def test_duckdb(make_config):
         connector_config={"foo": "bar"},
     )
     assert isinstance(config, DuckDBConnectionConfig)
-    assert config.is_recommended_for_state_sync is False
+    assert config.is_recommended_for_state_sync is True
 
 
 @pytest.mark.parametrize(
Repository: TobikoData/sqlmesh
Pull request #2389 (2024-04-03 12:48:55)

Update README.md (make email link a mailto)

CLAassistant: [![CLA assistant check](https://cla-assistant.io/pull/badge/not_signed)](https://cla-assistant.io/TobikoData/sqlmesh?pullRequest=2389) <br/>Thank you for your submission! We really appreciate it. Like many open source projects, we ask that you sign our [Contributor License Agreement](https://cla-assistant.io/TobikoData/sqlmesh?pullRequest=2389) before we can accept your contribution.<br/><sub>You have signed the CLA already but the status is still pending? Let us [recheck](https://cla-assistant.io/check/TobikoData/sqlmesh?pullRequest=2389) it.</sub>

Patch:
diff --git a/README.md b/README.md
index 9d4600c3..5e299a5f 100644
--- a/README.md
+++ b/README.md
@@ -34,7 +34,7 @@ We'd love to join you on your data journey. Connect with us in the following way
 
 * Join the [Tobiko Slack community](https://tobikodata.com/slack) to ask questions, or just to say hi!
 * File an issue on our [GitHub](https://github.com/TobikoData/sqlmesh/issues/new).
-* Send us an email at [hello@tobikodata.com](hello@tobikodata.com) with your questions or feedback.
+* Send us an email at [hello@tobikodata.com](mailto:hello@tobikodata.com) with your questions or feedback.
 
 ## Contribution
 Contributions in the form of issues or pull requests are greatly appreciated. [Read more](https://sqlmesh.readthedocs.io/en/stable/development/) about how to develop for SQLMesh.
diff --git a/sqlmesh/core/plan/builder.py b/sqlmesh/core/plan/builder.py
index 89376c78..4b87f5d9 100644
--- a/sqlmesh/core/plan/builder.py
+++ b/sqlmesh/core/plan/builder.py
@@ -615,7 +615,7 @@ class PlanBuilder:
     def _ensure_no_broken_references(self) -> None:
         for snapshot in self._context_diff.snapshots.values():
             broken_references = {x.name for x in self._context_diff.removed_snapshots} & {
-                x.name for x in snapshot.parents
+                x for x in snapshot.node.depends_on
             }
             if broken_references:
                 broken_references_msg = ", ".join(f"'{x}'" for x in broken_references)


Test patch:
diff --git a/tests/core/test_plan.py b/tests/core/test_plan.py
index e92f9595..cdd660f4 100644
--- a/tests/core/test_plan.py
+++ b/tests/core/test_plan.py
@@ -668,9 +668,7 @@ def test_broken_references(make_snapshot, mocker: MockerFixture):
     snapshot_a = make_snapshot(SqlModel(name="a", query=parse_one("select 1, ds")))
     snapshot_a.categorize_as(SnapshotChangeCategory.BREAKING)
 
-    snapshot_b = make_snapshot(
-        SqlModel(name="b", query=parse_one("select 2, ds FROM a")), nodes={'"a"': snapshot_a.node}
-    )
+    snapshot_b = make_snapshot(SqlModel(name="b", query=parse_one("select 2, ds FROM a")))
     snapshot_b.categorize_as(SnapshotChangeCategory.BREAKING)
 
     context_diff = ContextDiff(
Repository: TobikoData/sqlmesh
Pull request #2390 (2024-04-03 14:50:28)

Fix: Detection of broken model references when building a plan
Using `snapshot.parents` was incorrect for this check since the removed model wouldn't be present in the list of Snapshot parents (since there's no more snapshot), but is still available in the model's `depends_on` set.


Patch:
diff --git a/sqlmesh/core/plan/builder.py b/sqlmesh/core/plan/builder.py
index 89376c78..4b87f5d9 100644
--- a/sqlmesh/core/plan/builder.py
+++ b/sqlmesh/core/plan/builder.py
@@ -615,7 +615,7 @@ class PlanBuilder:
     def _ensure_no_broken_references(self) -> None:
         for snapshot in self._context_diff.snapshots.values():
             broken_references = {x.name for x in self._context_diff.removed_snapshots} & {
-                x.name for x in snapshot.parents
+                x for x in snapshot.node.depends_on
             }
             if broken_references:
                 broken_references_msg = ", ".join(f"'{x}'" for x in broken_references)


Test patch:
diff --git a/tests/core/test_plan.py b/tests/core/test_plan.py
index e92f9595..cdd660f4 100644
--- a/tests/core/test_plan.py
+++ b/tests/core/test_plan.py
@@ -668,9 +668,7 @@ def test_broken_references(make_snapshot, mocker: MockerFixture):
     snapshot_a = make_snapshot(SqlModel(name="a", query=parse_one("select 1, ds")))
     snapshot_a.categorize_as(SnapshotChangeCategory.BREAKING)
 
-    snapshot_b = make_snapshot(
-        SqlModel(name="b", query=parse_one("select 2, ds FROM a")), nodes={'"a"': snapshot_a.node}
-    )
+    snapshot_b = make_snapshot(SqlModel(name="b", query=parse_one("select 2, ds FROM a")))
     snapshot_b.categorize_as(SnapshotChangeCategory.BREAKING)
 
     context_diff = ContextDiff(
Repository: TobikoData/sqlmesh
Pull request #2412 (2024-04-09 09:33:52)

Fix: make Snowflake table part parsing more lenient


Patch:
diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py
index d935fab5..a24db88f 100644
--- a/sqlmesh/core/dialect.py
+++ b/sqlmesh/core/dialect.py
@@ -382,10 +382,16 @@ def _parse_types(
 def _parse_table_parts(
     self: Parser, schema: bool = False, is_db_reference: bool = False
 ) -> exp.Table:
+    index = self._index
     table = self.__parse_table_parts(schema=schema, is_db_reference=is_db_reference)  # type: ignore
     table_arg = table.this
 
     if isinstance(table_arg, exp.Var) and table_arg.name.startswith(SQLMESH_MACRO_PREFIX):
+        # Macro functions do not clash with the staged file syntax, so we can safely parse them
+        if "(" in table_arg.name:
+            self._retreat(index)
+            return Parser._parse_table_parts(self, schema=schema, is_db_reference=is_db_reference)
+
         table_arg.replace(MacroVar(this=table_arg.name[1:]))
         return StagedFilePath(**table.args)
 


Test patch:
diff --git a/tests/core/test_dialect.py b/tests/core/test_dialect.py
index bf4db7cf..b4770a30 100644
--- a/tests/core/test_dialect.py
+++ b/tests/core/test_dialect.py
@@ -207,6 +207,19 @@ def test_parse():
     assert isinstance(expressions[1], exp.Select)
     assert expressions[0].expressions[2].args["value"] == exp.to_identifier("metric")
 
+    expressions = parse(
+        """
+        MODEL(
+          name model1,
+          dialect snowflake
+        );
+
+        SELECT a FROM @if(true, m2, m3)
+        """,
+    )
+    assert len(expressions) == 2
+    assert expressions[1].sql(dialect="snowflake") == "SELECT a FROM @IF(TRUE, m2, m3)"
+
 
 def test_parse_jinja_with_semicolons():
     expressions = parse(
Repository: TobikoData/sqlmesh
Pull request #2419 (2024-04-10 06:04:41)

chore: add error message for untyped self references
https://github.com/TobikoData/sqlmesh/issues/2413


Patch:
diff --git a/sqlmesh/core/model/definition.py b/sqlmesh/core/model/definition.py
index f7478264..bb8845c3 100644
--- a/sqlmesh/core/model/definition.py
+++ b/sqlmesh/core/model/definition.py
@@ -1057,6 +1057,12 @@ class SqlModel(_SqlBasedModel):
             if count > 1:
                 raise_config_error(f"Found duplicate outer select name '{name}'", self._path)
 
+        if self.depends_on_past and not self.annotated:
+            raise_config_error(
+                "Self-referencing models require inferrable column types. There are three options available to mitigate this issue: add explicit types to all projections in the outermost SELECT statement, leverage external models (https://sqlmesh.readthedocs.io/en/stable/concepts/models/external_models/), or use the `columns` model attribute (https://sqlmesh.readthedocs.io/en/stable/concepts/models/overview/#columns).",
+                self._path,
+            )
+
         super().validate_definition()
 
     def is_breaking_change(self, previous: Model) -> t.Optional[bool]:


Test patch:
diff --git a/tests/core/test_model.py b/tests/core/test_model.py
index 3e943c58..fb54ddcd 100644
--- a/tests/core/test_model.py
+++ b/tests/core/test_model.py
@@ -172,6 +172,7 @@ def test_model_multiple_select_statements():
     "query, error",
     [
         ("y::int, x::int AS y", "duplicate"),
+        ("* FROM db.table", "require inferrable column types"),
     ],
 )
 def test_model_validation(query, error):
Repository: TobikoData/sqlmesh
Pull request #2446 (2024-04-15 17:53:46)

Fix: skip DataTypeParam when checking type inference


Patch:
diff --git a/sqlmesh/utils/__init__.py b/sqlmesh/utils/__init__.py
index 85c58d4c..8e6eb04e 100644
--- a/sqlmesh/utils/__init__.py
+++ b/sqlmesh/utils/__init__.py
@@ -326,6 +326,8 @@ def columns_to_types_all_known(columns_to_types: t.Dict[str, exp.DataType]) -> b
             if not d_type.kind:
                 return False
             d_type = d_type.kind
+        if isinstance(d_type, exp.DataTypeParam):
+            return True
         if d_type.is_type(exp.DataType.Type.UNKNOWN, exp.DataType.Type.NULL):
             return False
         if d_type.expressions:


Test patch:
diff --git a/tests/utils/test_helpers.py b/tests/utils/test_helpers.py
index dc41b430..48008064 100644
--- a/tests/utils/test_helpers.py
+++ b/tests/utils/test_helpers.py
@@ -1,5 +1,6 @@
 import pytest
 from sqlglot import expressions
+from sqlglot.optimizer.annotate_types import annotate_types
 
 from sqlmesh.utils import columns_to_types_all_known
 
@@ -63,6 +64,10 @@ from sqlmesh.utils import columns_to_types_all_known
             },
             False,
         ),
+        (
+            {"a": annotate_types(expressions.DataType.build("VARCHAR(MAX)", dialect="redshift"))},
+            True,
+        ),
     ],
 )
 def test_columns_to_types_all_known(columns_to_types, expected) -> None:
Repository: TobikoData/sqlmesh
Pull request #2452 (2024-04-16 03:08:17)

fix: don't call template if identifier has no variables


Patch:
diff --git a/sqlmesh/core/macros.py b/sqlmesh/core/macros.py
index ac185026..19f1b8aa 100644
--- a/sqlmesh/core/macros.py
+++ b/sqlmesh/core/macros.py
@@ -189,7 +189,7 @@ class MacroEvaluator:
                 return exp.convert(
                     self.transform(value) if isinstance(value, exp.Expression) else value
                 )
-            if isinstance(node, exp.Identifier):
+            if isinstance(node, exp.Identifier) and "@" in node.this:
                 text = self.template(node.this, self.locals)
                 if node.this != text:
                     changed = True


Test patch:
diff --git a/tests/core/test_macros.py b/tests/core/test_macros.py
index 0c57f487..588854c7 100644
--- a/tests/core/test_macros.py
+++ b/tests/core/test_macros.py
@@ -348,3 +348,9 @@ def test_macro_functions(macro_evaluator, assert_exp_eq, sql, expected, args):
 
 def test_macro_returns_none(macro_evaluator):
     assert macro_evaluator.transform(parse_one("@NOOP()")) is None
+
+
+def test_transform_no_template(mocker, macro_evaluator):
+    spy = mocker.spy(macro_evaluator, "template")
+    macro_evaluator.transform(parse_one("select a, b"))
+    spy.assert_not_called()
Repository: TobikoData/sqlmesh
Pull request #2516 (2024-04-25 15:56:25)

fix: this_model with snapshots wasnt working properly


Patch:
diff --git a/sqlmesh/core/renderer.py b/sqlmesh/core/renderer.py
index e94f6918..a0b6457c 100644
--- a/sqlmesh/core/renderer.py
+++ b/sqlmesh/core/renderer.py
@@ -100,12 +100,10 @@ class BaseExpressionRenderer:
             return self._cache
 
         if self._model_fqn and "this_model" not in kwargs:
-            if snapshots:
-                kwargs["this_model"] = self._to_table_mapping(
-                    [snapshots[self._model_fqn]], deployability_index
-                )
-            else:
-                kwargs["this_model"] = self._model_fqn
+            kwargs["this_model"] = self._to_table_mapping(
+                [snapshots[self._model_fqn]] if snapshots and self._model_fqn in snapshots else [],
+                deployability_index,
+            ).get(self._model_fqn, self._model_fqn)
 
         expressions = [self._expression]
 


Test patch:
diff --git a/tests/core/test_model.py b/tests/core/test_model.py
index f18dd875..46b47a02 100644
--- a/tests/core/test_model.py
+++ b/tests/core/test_model.py
@@ -36,7 +36,7 @@ from sqlmesh.core.model.common import parse_expression
 from sqlmesh.core.model.kind import _model_kind_validator
 from sqlmesh.core.model.seed import CsvSettings
 from sqlmesh.core.node import IntervalUnit, _Node
-from sqlmesh.core.snapshot import SnapshotChangeCategory
+from sqlmesh.core.snapshot import Snapshot, SnapshotChangeCategory
 from sqlmesh.utils.date import TimeLike, to_datetime, to_ds, to_timestamp
 from sqlmesh.utils.errors import ConfigError, SQLMeshError
 from sqlmesh.utils.jinja import JinjaMacroRegistry, MacroInfo
@@ -3810,6 +3810,15 @@ def test_this_model() -> None:
     assert model.render_pre_statements()[0].sql() == """COPY "db"."table" TO 'a'"""
     assert model.render_post_statements()[0].sql() == """COPY "db"."table" TO 'b'"""
 
+    snapshot = Snapshot.from_node(model, nodes={})
+    assert (
+        model.render_query_or_raise(
+            start="2020-01-01",
+            snapshots={snapshot.name: snapshot},
+        ).sql()
+        == '''SELECT '"db"."table"' AS "x"'''
+    )
+
 
 def test_macros_in_model_statement(sushi_context, assert_exp_eq):
     expressions = d.parse(
Repository: TobikoData/sqlmesh
Pull request #2517 (2024-04-25 17:52:52)

Fix: Propagate table properties as options when creating views in BQ
BigQuery supports the `OPTIONS` clause when creating views. Previously we weren't propagating `table_properties` set at the model level into the view creation statement.


Patch:
diff --git a/sqlmesh/core/engine_adapter/bigquery.py b/sqlmesh/core/engine_adapter/bigquery.py
index 636bab25..a006f164 100644
--- a/sqlmesh/core/engine_adapter/bigquery.py
+++ b/sqlmesh/core/engine_adapter/bigquery.py
@@ -568,6 +568,8 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin):
                 ),
             )
 
+        properties.extend(self._table_properties_to_expressions(table_properties))
+
         if properties:
             return exp.Properties(expressions=properties)
         return None


Test patch:
diff --git a/tests/core/engine_adapter/test_bigquery.py b/tests/core/engine_adapter/test_bigquery.py
index 707616f4..d33f8282 100644
--- a/tests/core/engine_adapter/test_bigquery.py
+++ b/tests/core/engine_adapter/test_bigquery.py
@@ -719,3 +719,36 @@ def test_drop_schema(
     sql_calls = _to_sql_calls(execute_mock)
 
     assert sql_calls == ensure_list(expected)
+
+
+def test_view_table_properties(make_mocked_engine_adapter: t.Callable, mocker: MockerFixture):
+    adapter = make_mocked_engine_adapter(BigQueryEngineAdapter)
+    execute_mock = mocker.patch(
+        "sqlmesh.core.engine_adapter.bigquery.BigQueryEngineAdapter.execute"
+    )
+
+    adapter.create_view(
+        "test_table",
+        parse_one("SELECT 1"),
+        table_description="some description",
+        table_properties={
+            "labels": exp.Array(
+                expressions=[
+                    exp.Tuple(
+                        expressions=[
+                            exp.Literal.string("test-label"),
+                            exp.Literal.string("label-value"),
+                        ]
+                    )
+                ]
+            ),
+        },
+    )
+
+    adapter.create_view("test_table", parse_one("SELECT 1"), table_properties={})
+
+    sql_calls = _to_sql_calls(execute_mock)
+    assert sql_calls == [
+        "CREATE OR REPLACE VIEW `test_table` OPTIONS (description='some description', labels=[('test-label', 'label-value')]) AS SELECT 1",
+        "CREATE OR REPLACE VIEW `test_table` AS SELECT 1",
+    ]