Repository: TobikoData/sqlmesh
Pull request #2219 (2024-03-05 15:45:47)

Docs: clarify which general properties apply to which kinds in model configuration reference
And expand seed model properties


Patch:
diff --git a/docs/reference/model_configuration.md b/docs/reference/model_configuration.md
index 909960b7..f5b92997 100644
--- a/docs/reference/model_configuration.md
+++ b/docs/reference/model_configuration.md
@@ -6,12 +6,13 @@ Learn more about specifying SQLMesh model properties in the [model concepts over
 
 ## General model properties
 
-Configuration options for SQLMesh model properties supported by all model kinds.
+Configuration options for SQLMesh model properties. Supported by all model kinds other than [`SEED` models](#seed-models).
 
 | Option             | Description                                                                                                                                                                                                                                                                                                                      |       Type        | Required |
 | ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :---------------: | :------: |
 | `name`             | The model name. Must include at least a qualifying schema (`<schema>.<model>`) and may include a catalog (`<catalog>.<schema>.<model>`). If any project model name includes a catalog, all model names must include a catalog or a default catalog [must be set](#default-schema--catalog).                                      |        str        |    Y     |
 | `kind`             | The model kind ([Additional Details](#model-kind-properties)) (Default: `VIEW`)                                                                                                                                                                                                                                                  |    str \| dict    |    N     |
+| `audits`           | SQLMesh [audits](../concepts/audits.md) that should run against the model's output                                                                                                                                                                                                                                               |    array[str]     |    N     |
 | `dialect`          | The SQL dialect in which the model's query is written. All SQL dialects [supported by the SQLGlot library](https://github.com/tobymao/sqlglot/blob/main/sqlglot/dialects/dialect.py) are allowed.                                                                                                                                |        str        |    N     |
 | `owner`            | The owner of a model; may be used for notification purposes                                                                                                                                                                                                                                                                      |        str        |    N     |
 | `stamp`            | Arbitrary string used to indicate a model's version without changing the model name                                                                                                                                                                                                                                              |        str        |    N     |
@@ -29,7 +30,7 @@ Configuration options for SQLMesh model properties supported by all model kinds.
 | `columns`          | The column names and data types returned by the model. Disables [automatic inference of column names and types](../concepts/models/overview.md#conventions) from the SQL query.                                                                                                                                                  |    array[str]     |    N     |
 | `table_properties` | Arbitrary table properties specific to the target engine. Specified as key-value pairs (`key = value`)                                                                                                                                                                                                                           |       dict        |    N     |
 | `allow_partials`   | Whether this model can process partial (incomplete) data intervals                                                                                                                                                                                                                                                               |       bool        |    N     |
-| `description`   | Description of the model. Automatically registered in the SQL engine's table COMMENT field or equivalent (if supported by the engine).                                                                                                                                                                                                                                                               |       str        |    N     |
+| `description`      | Description of the model. Automatically registered in the SQL engine's table COMMENT field or equivalent (if supported by the engine).                                                                                                                                                                                           |        str        |    N     |
 
 ### Model defaults
 
@@ -45,17 +46,15 @@ The SQLMesh project-level `model_defaults` key supports the following options, d
 - batch_size
 - storage_format
 
-The project-level `model_defaults` key also supports two keys for specifying a default catalog or schema, described [below](#default-schema--catalog).
-
 ## Model kind properties
 
-Configuration options for kind-specific SQLMesh model properties.
+Configuration options for kind-specific SQLMesh model properties, in addition to the [general model properties](#general-model-properties) listed above.
 
 Learn more about model kinds at the [model kind concepts page](../concepts/models/model_kinds.md). Learn more about specifying model kind in Python models at the [Python models concepts page](../concepts/models/python_models.md#model-specification).
 
 ### `VIEW` models
 
-Configuration options for models of the [`VIEW` kind](../concepts/models/model_kinds.md#view).
+Configuration options for models of the [`VIEW` kind](../concepts/models/model_kinds.md#view) (in addition to [general model properties](#general-model-properties)).
 
 | Option         | Description                                                                                          | Type | Required |
 | -------------- | ---------------------------------------------------------------------------------------------------- | :--: | :------: |
@@ -65,13 +64,13 @@ Python model configuration object: [ViewKind()](https://sqlmesh.readthedocs.io/e
 
 ### `FULL` models
 
-The [`FULL` model kind](../concepts/models/model_kinds.md#full) does not support any configuration options.
+The [`FULL` model kind](../concepts/models/model_kinds.md#full) does not support any configuration options other than the [general model properties listed above](#general-model-properties).
 
 Python model configuration object: [FullKind()](https://sqlmesh.readthedocs.io/en/stable/_readthedocs/html/sqlmesh/core/model/kind.html#FullKind)
 
 ### Incremental models
 
-Configuration options for all incremental models.
+Configuration options for all incremental models (in addition to [general model properties](#general-model-properties)).
 
 | Option       | Description                                                                                                                                                                                                                                                                                                                      | Type | Required |
 | ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :--: | :------: |
@@ -80,7 +79,7 @@ Configuration options for all incremental models.
 
 #### Incremental by time range
 
-Configuration options for [`INCREMENTAL_BY_TIME_RANGE` models](../concepts/models/model_kinds.md#incremental_by_time_range).
+Configuration options for [`INCREMENTAL_BY_TIME_RANGE` models](../concepts/models/model_kinds.md#incremental_by_time_range) (in addition to [general model properties](#general-model-properties) and [incremental model properties](#incremental-models)).
 
 | Option                | Description                                                                                                                             | Type | Required |
 | --------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | :--: | :------: |
@@ -93,7 +92,7 @@ Python model configuration object: [IncrementalByTimeRangeKind()](https://sqlmes
 
 #### Incremental by unique key
 
-Configuration options for [`INCREMENTAL_BY_UNIQUE_KEY` models](../concepts/models/model_kinds.md#incremental_by_unique_key).
+Configuration options for [`INCREMENTAL_BY_UNIQUE_KEY` models](../concepts/models/model_kinds.md#incremental_by_unique_key) (in addition to [general model properties](#general-model-properties) and [incremental model properties](#incremental-models)).
 
 | Option                | Description                                                                                                                             |       Type       | Required |
 | --------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | :--------------: | :------: |
@@ -103,27 +102,72 @@ Configuration options for [`INCREMENTAL_BY_UNIQUE_KEY` models](../concepts/model
 
 Python model configuration object: [IncrementalByUniqueKeyKind()](https://sqlmesh.readthedocs.io/en/stable/_readthedocs/html/sqlmesh/core/model/kind.html#IncrementalByUniqueKeyKind)
 
-### `SEED` models
+#### SCD Type 2 models
+
+Configuration options for [`SCD_TYPE_2` models](../concepts/models/model_kinds.md#scd-type-2) (in addition to [general model properties](#general-model-properties) and [incremental model properties](#incremental-models)).
+
+| Option                    | Description                                                                                                                                                                                 |   Type    | Required |
+| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :-------: | :------: |
+| `unique_key`              | The model column(s) containing each row's unique key.                                                                                                                                       | list[str] |    Y     |
+| `valid_from_name`         | The model column containing each row's valid from date. (Default: `valid_from`)                                                                                                             |    str    |    N     |
+| `valid_to_name`           | The model column containing each row's valid to date. (Default: `valid_to`)                                                                                                                 |    str    |    N     |
+| `invalidate_hard_deletes` | If set to true, when a record is missing from the source table it will be marked as invalid - see [here](../concepts/models/model_kinds.md#deletes) for more information. (Default: `True`) |   bool    |    N     |
 
-Configuration options for [`SEED` models](../concepts/models/model_kinds.md#seed).
+##### SCD Type 2 By Time
 
-| Option | Description            | Type | Required |
-| ------ | ---------------------- | :--: | :------: |
-| `path` | Path to seed CSV file. | str  |    Y     |
+Configuration options for [`SCD_TYPE_2_BY_TIME` models](../concepts/models/model_kinds.md#scd-type-2) (in addition to [general model properties](#general-model-properties), [incremental model properties](#incremental-models), and [SCD Type 2 properties](#scd-type-2-models)).
 
-Python model configuration object: [SeedKind()](https://sqlmesh.readthedocs.io/en/stable/_readthedocs/html/sqlmesh/core/model/kind.html#SeedKind)
+| Option                     | Description                                                                                                                                                                      | Type | Required |
+| -------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :--: | :------: |
+| `updated_at_name`          | The model column containing each row's updated at date. (Default: `updated_at`)                                                                                                  | str  |    N     |
+| `updated_at_as_valid_from` | By default, for new rows the `valid_from` column is set to 1970-01-01 00:00:00. This sets `valid_from` to the value of `updated_at` when the row is inserted. (Default: `False`) | bool |    N     |
 
-### SCD Type 2 models
+##### SCD Type 2 By Column
 
-Configuration options for [`SCD_TYPE_2` models](../concepts/models/model_kinds.md#scd-type-2).
+Configuration options for [`SCD_TYPE_2_BY_COLUMN` models](../concepts/models/model_kinds.md#scd-type-2) (in addition to [general model properties](#general-model-properties), [incremental model properties](#incremental-models), and [SCD Type 2 properties](#scd-type-2-models)).
 
-| Option                | Description                                                                                                                            |   Type    | Required |
-| --------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | :-------: | :------: |
-| `unique_key`          | The model column(s) containing each row's unique key.                                                                                  | list[str] |    Y     |
-| `valid_from_name`     | The model column containing each row's valid from date. (Default: `valid_from`)                                                        |    str    |    N     |
-| `valid_to_name`       | The model column containing each row's valid to date. (Default: `valid_to`)                                                            |    str    |    N     |
-| `updated_at_name`     | The model column containing each row's updated at date. (Default: `updated_at`)                                                        |    str    |    N     |
-| `forward_only`        | Whether the model's changes should always be classified as [forward-only](../concepts/plans.md#forward-only-change). (Default: `True`) |   bool    |    N     |
-| `disable_restatement` | Whether [restatements](../concepts/plans.md#restatement-plans) should be disabled for the model. (Default: `True`)                     |   bool    |    N     |
+| Option                         | Description                                                                                                                                                                 |       Type        | Required |
+| ------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :---------------: | :------: |
+| `columns`                      | Columns whose changed data values indicate a data update (instead of an `updated_at` column). `*` to represent that all columns should be checked.                          | str \| array[str] |    Y     |
+| `execution_time_as_valid_from` | By default, for new rows `valid_from` is set to 1970-01-01 00:00:00. This changes the behavior to set it to the execution_time of when the pipeline ran. (Default: `False`) |       bool        |    N     |
+
+Python model configuration object: [SCDType2Kind()](https://sqlmesh.readthedocs.io/en/stable/_readthedocs/html/sqlmesh/core/model/kind.html#SCDType2Kind)
+
+### `SEED` models
 
-Python model configuration object: [SCDType2Kind()](https://sqlmesh.readthedocs.io/en/stable/_readthedocs/html/sqlmesh/core/model/kind.html#SCDType2Kind)
\ No newline at end of file
+Configuration options for [`SEED` models](../concepts/models/model_kinds.md#seed). `SEED` models do not support all the general properties supported by other models; they only support the properties listed in this table.
+
+Top-level options inside the MODEL DDL:
+
+| Option        | Description                                                                                                                                                                                                                                                                                 |    Type    | Required |
+| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :--------: | :------: |
+| `name`        | The model name. Must include at least a qualifying schema (`<schema>.<model>`) and may include a catalog (`<catalog>.<schema>.<model>`). If any project model name includes a catalog, all model names must include a catalog or a default catalog [must be set](#default-schema--catalog). |    str     |    Y     |
+| `kind`        | The model kind. Must be `SEED`.                                                                                                                                                                                                                                                             |            |          |
+| `columns`     | The column names and data types in the CSV file. Disables automatic inference of column names and types by the pandas CSV reader. NOTE: order of columns overrides the order specified in the CSV header row (if present).                                                                  | array[str] |    N     |
+| `audits`      | SQLMesh [audits](../concepts/audits.md) that should run against the model's output                                                                                                                                                                                                          | array[str] |    N     |
+| `owner`       | The owner of a model; may be used for notification purposes                                                                                                                                                                                                                                 |    str     |    N     |
+| `stamp`       | Arbitrary string used to indicate a model's version without changing the model name                                                                                                                                                                                                         |    str     |    N     |
+| `tags`        | Arbitrary strings used to organize or classify a model                                                                                                                                                                                                                                      | array[str] |    N     |
+| `description` | Description of the model. Automatically registered in the SQL engine's table COMMENT field or equivalent (if supported by the engine).                                                                                                                                                      |    str     |    N     |
+
+Options specified within the top-level `kind` property:
+
+| Option         | Description                                                                                                                                                                             | Type | Required |
+| -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---- | -------- |
+| `path`         | Path to seed CSV file.                                                                                                                                                                  | str  | Y        |
+| `batch_size`   | The maximum number of CSV rows ingested in each batch. All rows ingested in one batch if not specified.                                                                                 | int  | N        |
+| `csv_settings` | Pandas CSV reader settings (overrides [default values](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html)). Specified as key-value pairs (`key = value`). | dict | N        |
+
+Options specified within the `kind` property's `csv_settings` property (overrides [default Pandas CSV reader settings](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html)):
+
+| Option             | Description                                                                                                                                                                                                                                                                         | Type | Required |
+| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---- | -------- |
+| `delimiter`        | Character or regex pattern to treat as the delimiter. More information at the [Pandas documentation](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html).                                                                                              | str  | N        |
+| `quotechar`        | Character used to denote the start and end of a quoted item. More information at the [Pandas documentation](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html).                                                                                       | str  | N        |
+| `doublequote`      | When quotechar is specified, indicate whether or not to interpret two consecutive quotechar elements INSIDE a field as a single quotechar element. More information at the [Pandas documentation](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html). | bool | N        |
+| `escapechar`       | Character used to escape other characters. More information at the [Pandas documentation](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html).                                                                                                         | str  | N        |
+| `skipinitialspace` | Skip spaces after delimiter. More information at the [Pandas documentation](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html).                                                                                                                       | bool | N        |
+| `lineterminator`   | Character used to denote a line break. More information at the [Pandas documentation](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html).                                                                                                             | str  | N        |
+| `encoding`         | Encoding to use for UTF when reading/writing (ex. 'utf-8'). More information at the [Pandas documentation](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html).                                                                                        | str  | N        |
+
+Python model configuration object: [SeedKind()](https://sqlmesh.readthedocs.io/en/stable/_readthedocs/html/sqlmesh/core/model/kind.html#SeedKind)
\ No newline at end of file
diff --git a/sqlmesh/core/engine_adapter/redshift.py b/sqlmesh/core/engine_adapter/redshift.py
index cc99a031..6db265ce 100644
--- a/sqlmesh/core/engine_adapter/redshift.py
+++ b/sqlmesh/core/engine_adapter/redshift.py
@@ -344,4 +344,10 @@ def parse_plan(plan: str) -> t.Optional[t.Dict]:
         match(TokenType.R_BRACE, True)
         return query_plan
 
-    return parse_nested()
+    while curr():
+        nested = parse_nested()
+
+        if nested and nested.get("name") == "RESULT":
+            return nested
+        advance()
+    return None


Test patch:
diff --git a/tests/core/engine_adapter/test_redshift.py b/tests/core/engine_adapter/test_redshift.py
index 2480a536..32d2b402 100644
--- a/tests/core/engine_adapter/test_redshift.py
+++ b/tests/core/engine_adapter/test_redshift.py
@@ -840,3 +840,110 @@ def test_parse_plan():
             },
         },
     ]
+
+
+def test_parse_plan_limit():
+    plan = parse_plan(
+        """
+{ LIMIT
+  :startup_cost 0.00
+  :total_cost 0.00
+  :plan_rows 1
+  :node_id 1
+  :parent_id 0
+  :plan_width 0
+  :best_pathkeys <>
+  :phys_properties <>
+  :reqd_phys_properties <>
+  :targetlist (
+    { TARGETENTRY
+    :resdom
+      { RESDOM
+      :resno 1
+      :restype 20
+      :restypmod -1
+      :resname ?column?
+      :ressortgroupref 0
+      :resorigtbl 0
+      :resorigcol 0
+      :resjunk false
+      }
+    :expr
+      { CONST
+      :consttype 20
+      :constlen 8
+      :constbyval true
+      :constisnull true
+      :constvalue <>
+      }
+    }
+  )
+  }
+
+XN Limit  (cost=0.00..0.00 rows=1 width=0)
+  ->  XN Result  (cost=0.00..0.01 rows=1 width=0)
+        One-Time Filter: false
+
+  { RESULT
+  :targetlist (
+    { TARGETENTRY
+    :resdom
+      { RESDOM
+      :resno 1
+      :restype 1043
+      :restypmod 36
+      :resname surrogate_key
+      :ressortgroupref 0
+      :resorigtbl 0
+      :resorigcol 0
+      :resjunk false
+      }
+    :expr
+      { VAR
+      :varno 65001
+      :varattno 1
+      :vartype 1043
+      :vartypmod 36
+      :varlevelsup 0
+      :varnoold 1
+      :varoattno 1
+      }
+    }
+  )
+  :scanrelid 0
+  :values_lists <>
+  }
+        """
+    )
+
+    assert plan == {
+        "name": "RESULT",
+        "scanrelid": "0",
+        "targetlist": [
+            {
+                "expr": {
+                    "name": "VAR",
+                    "varattno": "1",
+                    "varlevelsup": "0",
+                    "varno": "65001",
+                    "varnoold": "1",
+                    "varoattno": "1",
+                    "vartype": "1043",
+                    "vartypmod": "36",
+                },
+                "name": "TARGETENTRY",
+                "resdom": {
+                    "name": "RESDOM",
+                    "resjunk": "false",
+                    "resname": "surrogate_key",
+                    "resno": "1",
+                    "resorigcol": "0",
+                    "resorigtbl": "0",
+                    "ressortgroupref": "0",
+                    "restype": "1043",
+                    "restypmod": "36",
+                },
+            }
+        ],
+        "values_lists": "<>",
+    }