Repository: TobikoData/sqlmesh
Pull request #2905 (2024-07-15 19:13:09)

feat: support upstream/downstream git selector
Prior to this change the git branch selector didn't support upstream/downstream selection (using `+`) so this PR adds that functionality. This is part of work to allow the Github Action CI/CD bot to be able to compare against git main instead of prod. Slack context for that change: https://tobiko-data.slack.com/archives/C044BRE5W4S/p1720792390212429


Patch:
diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py
index 33bd240b..19407487 100644
--- a/sqlmesh/core/dialect.py
+++ b/sqlmesh/core/dialect.py
@@ -241,7 +241,7 @@ def _parse_macro(self: Parser, keyword_macro: str = "") -> t.Optional[exp.Expres
             return field
         return self.expression(MacroVar, this=field.this, comments=comments)
 
-    if isinstance(field, exp.Window):
+    if isinstance(field, (exp.Window, exp.IgnoreNulls, exp.RespectNulls)):
         field.set("this", _build_macro(field.this))
     else:
         field = _build_macro(field)
diff --git a/sqlmesh/core/selector.py b/sqlmesh/core/selector.py
index e1c6bd66..3c4a844d 100644
--- a/sqlmesh/core/selector.py
+++ b/sqlmesh/core/selector.py
@@ -158,8 +158,9 @@ class Selector:
                     add_sub_results(
                         self._expand_model_tag(sub_selection[4:], models, models_by_tag)
                     )
-                elif sub_selection.startswith("git:"):
-                    add_sub_results(self._expand_git(sub_selection[4:]))
+                elif sub_selection.startswith(("git:", "+git:")):
+                    sub_selection = sub_selection.replace("git:", "")
+                    add_sub_results(self._expand_git(sub_selection, models))
                 else:
                     add_sub_results(self._expand_model_name(sub_selection, models))
 
@@ -170,7 +171,15 @@ class Selector:
 
         return results
 
-    def _expand_git(self, target_branch: str) -> t.Set[str]:
+    def _expand_git(self, target_branch: str, models: t.Dict[str, Model]) -> t.Set[str]:
+        results: t.Set[str] = set()
+
+        (
+            target_branch,
+            include_upstream,
+            include_downstream,
+        ) = self._get_value_and_dependency_inclusion(target_branch)
+
         git_modified_files = {
             *self._git_client.list_untracked_files(),
             *self._git_client.list_uncommitted_changed_files(),
@@ -180,8 +189,14 @@ class Selector:
 
         if not matched_models:
             logger.warning(f"Expression 'git:{target_branch}' doesn't match any models.")
+            return matched_models
 
-        return matched_models
+        for model_fqn in matched_models:
+            results.update(
+                self._get_models(model_fqn, include_upstream, include_downstream, models)
+            )
+
+        return results
 
     def _expand_model_name(self, selection: str, models: t.Dict[str, Model]) -> t.Set[str]:
         results = set()


Test patch:
diff --git a/tests/core/test_macros.py b/tests/core/test_macros.py
index 72bde9ce..6b1fba20 100644
--- a/tests/core/test_macros.py
+++ b/tests/core/test_macros.py
@@ -3,7 +3,7 @@ import typing as t
 import pytest
 from sqlglot import MappingSchema, ParseError, exp, parse_one
 
-from sqlmesh.core import constants as c
+from sqlmesh.core import constants as c, dialect as d
 from sqlmesh.core.dialect import StagedFilePath
 from sqlmesh.core.macros import SQL, MacroEvalError, MacroEvaluator, macro
 from sqlmesh.utils.errors import SQLMeshError
@@ -638,3 +638,24 @@ def test_macro_metadata_flag():
 
     assert getattr(noop_metadata_only, c.SQLMESH_METADATA) is True
     assert macro.get_registry()["noop_metadata_only"].metadata_only is True
+
+
+def test_macro_first_value_ignore_respect_nulls(assert_exp_eq) -> None:
+    schema = MappingSchema({}, dialect="duckdb")
+    evaluator = MacroEvaluator(schema=schema, dialect="duckdb")
+
+    evaluator.evaluate(t.cast(d.MacroDef, d.parse_one("@DEF(test, x -> x)")))
+
+    expected_sql = "SELECT FIRST_VALUE(x IGNORE NULLS) OVER (ORDER BY y NULLS FIRST) AS column_test"
+    actual_expr = d.parse_one(
+        "SELECT FIRST_VALUE(@test(x) IGNORE NULLS) OVER (ORDER BY y) AS column_test"
+    )
+    assert_exp_eq(evaluator.transform(actual_expr), expected_sql, dialect="duckdb")
+
+    expected_sql = (
+        "SELECT FIRST_VALUE(x RESPECT NULLS) OVER (ORDER BY y NULLS FIRST) AS column_test"
+    )
+    actual_expr = d.parse_one(
+        "SELECT FIRST_VALUE(@test(x) RESPECT NULLS) OVER (ORDER BY y) AS column_test"
+    )
+    assert_exp_eq(evaluator.transform(actual_expr), expected_sql, dialect="duckdb")
diff --git a/tests/core/test_selector.py b/tests/core/test_selector.py
index da8e2e61..f43e918b 100644
--- a/tests/core/test_selector.py
+++ b/tests/core/test_selector.py
@@ -392,6 +392,9 @@ def test_model_selection_normalized(mocker: MockerFixture, make_snapshot):
     [
         (["git:main"], {'"test_model_a"', '"test_model_c"'}),
         (["git:main & +*model_c"], {'"test_model_c"'}),
+        (["git:main+"], {'"test_model_a"', '"test_model_c"', '"test_model_d"'}),
+        (["+git:main"], {'"test_model_a"', '"test_model_c"', '"test_model_b"'}),
+        (["+git:main+"], {'"test_model_a"', '"test_model_c"', '"test_model_b"', '"test_model_d"'}),
     ],
 )
 def test_expand_git_selection(
@@ -407,7 +410,11 @@ def test_expand_git_selection(
     model_b._path = Path("/path/to/test_model_b.sql")
     models[model_b.fqn] = model_b
 
-    model_c = SqlModel(name="test_model_c", query=d.parse_one("SELECT 3 AS c"))
+    model_c = SqlModel(
+        name="test_model_c",
+        query=d.parse_one("SELECT b AS c FROM test_model_b"),
+        depends_on={"test_model_b"},
+    )
     model_c._path = Path("/path/to/test_model_c.sql")
     models[model_c.fqn] = model_c