Repository: TobikoData/sqlmesh
Pull request #2906 (2024-07-15 23:32:28)

Docs: Add description for the environment_catalog_mapping setting to the configuration reference


Patch:
diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md
index e8c1d0ac..7c93d254 100644
--- a/docs/reference/configuration.md
+++ b/docs/reference/configuration.md
@@ -25,16 +25,17 @@ Configuration options for SQLMesh project directories.
 
 Configuration options for SQLMesh environment creation and promotion.
 
-| Option                       | Description                                                                                                                                                                                                                                                                                        | Type         | Required |
-|------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------:|:--------:|
-| `snapshot_ttl`               | The period of time that a model snapshot not a part of any environment should exist before being deleted. This is defined as a string with the default `in 1 week`. Other [relative dates](https://dateparser.readthedocs.io/en/latest/) can be used, such as `in 30 days`. (Default: `in 1 week`) | string       | N        |
-| `environment_ttl`            | The period of time that a development environment should exist before being deleted. This is defined as a string with the default `in 1 week`. Other [relative dates](https://dateparser.readthedocs.io/en/latest/) can be used, such as `in 30 days`. (Default: `in 1 week`)                      | string       | N        |
-| `pinned_environments`        | The list of development environments that are exempt from deletion due to expiration                                                                                                                                                                                                               | list[string] | N        |
-| `time_column_format`         | The default format to use for all model time columns. This time format uses [python format codes](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes) (Default: `%Y-%m-%d`)                                                                                        | string       | N        |
-| `default_target_environment` | The name of the environment that will be the default target for the `sqlmesh plan` and `sqlmesh run` commands. (Default: `prod`)                                                                                                                                                                   | string       | N        |
-| `physical_schema_override`   | A mapping from model schema names to names of schemas in which physical tables for the corresponding models will be placed - [addition details](../guides/configuration.md#physical-schema-override). (Default physical schema name: `sqlmesh__[model schema]`)                                    | string       | N        |
-| `environment_suffix_target`  | Whether SQLMesh views should append their environment name to the `schema` or `table` - [additional details](../guides/configuration.md#view-schema-override). (Default: `schema`)                                                                                                                 | string       | N        |
-| `log_limit`                  | The default number of logs to keep (Default: `20`)                                                                                                                                                                                                                                                 | int          | N        |
+| Option                        | Description                                                                                                                                                                                                                                                                                        | Type                 | Required |
+|-------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------------------:|:--------:|
+| `snapshot_ttl`                | The period of time that a model snapshot not a part of any environment should exist before being deleted. This is defined as a string with the default `in 1 week`. Other [relative dates](https://dateparser.readthedocs.io/en/latest/) can be used, such as `in 30 days`. (Default: `in 1 week`) | string               | N        |
+| `environment_ttl`             | The period of time that a development environment should exist before being deleted. This is defined as a string with the default `in 1 week`. Other [relative dates](https://dateparser.readthedocs.io/en/latest/) can be used, such as `in 30 days`. (Default: `in 1 week`)                      | string               | N        |
+| `pinned_environments`         | The list of development environments that are exempt from deletion due to expiration                                                                                                                                                                                                               | list[string]         | N        |
+| `time_column_format`          | The default format to use for all model time columns. This time format uses [python format codes](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes) (Default: `%Y-%m-%d`)                                                                                        | string               | N        |
+| `default_target_environment`  | The name of the environment that will be the default target for the `sqlmesh plan` and `sqlmesh run` commands. (Default: `prod`)                                                                                                                                                                   | string               | N        |
+| `physical_schema_override`    | A mapping from model schema names to names of schemas in which physical tables for the corresponding models will be placed - [addition details](../guides/configuration.md#physical-schema-override). (Default physical schema name: `sqlmesh__[model schema]`)                                    | string               | N        |
+| `environment_suffix_target`   | Whether SQLMesh views should append their environment name to the `schema` or `table` - [additional details](../guides/configuration.md#view-schema-override). (Default: `schema`)                                                                                                                 | string               | N        |
+| `environment_catalog_mapping` | A mapping from regular expressions to catalog names. The catalog name is used to determine the target catalog for a given environment.                                                                                                                                                             | dict[string, string] | N        |
+| `log_limit`                   | The default number of logs to keep (Default: `20`)                                                                                                                                                                                                                                                 | int                  | N        |
 
 ### Model defaults
 
diff --git a/docs/reference/model_configuration.md b/docs/reference/model_configuration.md
index db89b024..48f9e624 100644
--- a/docs/reference/model_configuration.md
+++ b/docs/reference/model_configuration.md
@@ -23,7 +23,7 @@ Configuration options for SQLMesh model properties. Supported by all model kinds
 | `end`                 | The date/time that determines the latest date interval that should be processed by a model. Can be a datetime string, epoch time in milliseconds, or a relative datetime such as `1 year ago`.                                                                                                                                                              | str \| int        | N        |
 | `grains`              | The column(s) whose combination uniquely identifies each row in the model                                                                                                                                                                                                                                                                                   | str \| array[str] | N        |
 | `references`          | The model column(s) used to join to other models' grains                                                                                                                                                                                                                                                                                                    | str \| array[str] | N        |
-| `depends_on`          | Models on which this model depends. (Default: dependencies inferred from model code)                                                                                                                                                                                                                                                                        | array[str]        | N        |
+| `depends_on`          | Models on which this model depends in addition to the ones inferred from the model's query. (Default: dependencies inferred from model code)                                                                                                                                                                                                                | array[str]        | N        |
 | `storage_format`      | The storage format that should be used to store physical tables; only applicable to engines such as Spark                                                                                                                                                                                                                                                   | str               | N        |
 | `partitioned_by`      | The column(s) and/or column expressions used define a model's partitioning key. Required for the `INCREMENTAL_BY_PARTITION` model kind. Optional for all other model kinds; used to partition the model's physical table in engines that support partitioning.                                                                                              | str \| array[str] | N        |
 | `clustered_by`        | The column(s) used to cluster the model's physical table; only applicable to engines that support clustering                                                                                                                                                                                                                                                | str               | N        |
diff --git a/sqlmesh/cli/main.py b/sqlmesh/cli/main.py
index 4bc28920..3d0ed10a 100644
--- a/sqlmesh/cli/main.py
+++ b/sqlmesh/cli/main.py
@@ -702,12 +702,17 @@ def rollback(obj: Context) -> None:
 
 
 @cli.command("create_external_models")
+@click.option(
+    "--strict",
+    is_flag=True,
+    help="Raise an error if the external model is missing in the database",
+)
 @click.pass_obj
 @error_handler
 @cli_analytics
-def create_external_models(obj: Context) -> None:
+def create_external_models(obj: Context, **kwargs: t.Any) -> None:
     """Create a schema file containing external model schemas."""
-    obj.create_external_models()
+    obj.create_external_models(**kwargs)
 
 
 @cli.command("table_diff")
diff --git a/sqlmesh/core/config/root.py b/sqlmesh/core/config/root.py
index bd6f7638..a2e18e84 100644
--- a/sqlmesh/core/config/root.py
+++ b/sqlmesh/core/config/root.py
@@ -67,6 +67,7 @@ class Config(BaseConfig):
         model_defaults: Default values for model definitions.
         physical_schema_override: A mapping from model schema names to names of schemas in which physical tables for corresponding models will be placed.
         environment_suffix_target: Indicates whether to append the environment name to the schema or table name.
+        environment_catalog_mapping: A mapping from regular expressions to catalog names. The catalog name is used to determine the target catalog for a given environment.
         default_target_environment: The name of the environment that will be the default target for the `sqlmesh plan` and `sqlmesh run` commands.
         log_limit: The default number of logs to keep.
         format: The formatting options for SQL code.
diff --git a/sqlmesh/core/context.py b/sqlmesh/core/context.py
index 14783d00..06b86506 100644
--- a/sqlmesh/core/context.py
+++ b/sqlmesh/core/context.py
@@ -1655,11 +1655,14 @@ class GenericContext(BaseContext, t.Generic[C]):
         self._new_state_sync().rollback()
 
     @python_api_analytics
-    def create_external_models(self) -> None:
+    def create_external_models(self, strict: bool = False) -> None:
         """Create a file to document the schema of external models.
 
         The external models file contains all columns and types of external models, allowing for more
         robust lineage, validation, and optimizations.
+
+        Args:
+            strict: If True, raise an error if the external model is missing in the database.
         """
         if not self._models:
             self.load(update_schemas=False)
@@ -1685,6 +1688,7 @@ class GenericContext(BaseContext, t.Generic[C]):
                 dialect=config.model_defaults.dialect,
                 gateway=self.gateway,
                 max_workers=self.concurrent_tasks,
+                strict=strict,
             )
 
     @python_api_analytics
diff --git a/sqlmesh/core/schema_loader.py b/sqlmesh/core/schema_loader.py
index c4704050..1b00c7bc 100644
--- a/sqlmesh/core/schema_loader.py
+++ b/sqlmesh/core/schema_loader.py
@@ -12,6 +12,7 @@ from sqlmesh.core.engine_adapter import EngineAdapter
 from sqlmesh.core.model.definition import Model
 from sqlmesh.core.state_sync import StateReader
 from sqlmesh.utils import UniqueKeyDict, yaml
+from sqlmesh.utils.errors import SQLMeshError
 
 logger = logging.getLogger(__name__)
 
@@ -24,6 +25,7 @@ def create_external_models_file(
     dialect: DialectType,
     gateway: t.Optional[str] = None,
     max_workers: int = 1,
+    strict: bool = False,
 ) -> None:
     """Create or replace a YAML file with column and types of all columns in all external models.
 
@@ -35,6 +37,7 @@ def create_external_models_file(
         dialect: The dialect to serialize the schema as.
         gateway: If the model should be associated with a specific gateway; the gateway key
         max_workers: The max concurrent workers to fetch columns.
+        strict: If True, raise an error if the external model is missing in the database.
     """
     external_model_fqns = set()
 
@@ -61,7 +64,10 @@ def create_external_models_file(
             try:
                 return adapter.columns(table, include_pseudo_columns=True)
             except Exception as e:
-                logger.warning(f"Unable to get schema for '{table}': '{e}'.")
+                msg = f"Unable to get schema for '{table}': '{e}'."
+                if strict:
+                    raise SQLMeshError(msg) from e
+                logger.warning(msg)
                 return None
 
         gateway_part = {"gateway": gateway} if gateway else {}
diff --git a/sqlmesh/magics.py b/sqlmesh/magics.py
index e1c1c801..51652fc0 100644
--- a/sqlmesh/magics.py
+++ b/sqlmesh/magics.py
@@ -558,11 +558,17 @@ class SQLMeshMagics(Magics):
         context.console.log_success("Migration complete")
 
     @magic_arguments()
+    @argument(
+        "--strict",
+        action="store_true",
+        help="Raise an error if the external model is missing in the database",
+    )
     @line_magic
     @pass_sqlmesh_context
     def create_external_models(self, context: Context, line: str) -> None:
         """Create a schema file containing external model schemas."""
-        context.create_external_models()
+        args = parse_argstring(self.create_external_models, line)
+        context.create_external_models(strict=args.strict)
 
     @magic_arguments()
     @argument(


Test patch:
diff --git a/tests/core/test_schema_loader.py b/tests/core/test_schema_loader.py
index 2e970914..ada27dfd 100644
--- a/tests/core/test_schema_loader.py
+++ b/tests/core/test_schema_loader.py
@@ -1,3 +1,4 @@
+import pytest
 import logging
 import typing as t
 from pathlib import Path
@@ -16,6 +17,7 @@ from sqlmesh.core.model.definition import ExternalModel
 from sqlmesh.core.schema_loader import create_external_models_file
 from sqlmesh.core.snapshot import SnapshotChangeCategory
 from sqlmesh.utils.yaml import YAML
+from sqlmesh.utils.errors import SQLMeshError
 
 
 def test_create_external_models(tmpdir, assert_exp_eq):
@@ -354,3 +356,13 @@ def test_missing_table(tmp_path: Path):
     with open(filename, "r", encoding="utf8") as fd:
         schema = YAML().load(fd)
     assert len(schema) == 0
+
+    with pytest.raises(SQLMeshError, match=r"""Unable to get schema for '"tbl_source"'.*"""):
+        create_external_models_file(
+            filename,
+            {"a": model},  # type: ignore
+            context.engine_adapter,
+            context.state_reader,
+            "",
+            strict=True,
+        )