Repository: dbt-labs/dbt-core

Issue:
[CT-3299] [SL] Support Hierarchical Configs for Saved Queries and their Exports
### Housekeeping

- [X] I am a maintainer of dbt-core

### Short description

In #8950 we added `Exports` to `SavedQueries`. `Exports` have configs and we want these configs to be setable from a project's `dbt_project.yml`, i.e. hierarchical. Originally this work was to be part of #8892 in #8950, however due to deadlines and some pesky edge cases, we separated out this work. With that said, we've thought through (and tried some) approaches

1. Write an entire bespoke config compiler just for `Exports`
2. Make an `Export` a node type (but don't put it in the DAG), and then use [calculate_node_config](https://github.com/dbt-labs/dbt-core/blob/bb21403c9e906097c4913edcf7a013d404e93d80/core/dbt/context/context_config.py#L134) to generate the config
3. Abstract pretty much all of `calculate_node_config` except the config class getter into a separate function which would then be called by `calculate_node_config` and directly from the export parsing

We haven't attempted option (1) because it seemed wrong to write a whole bunch of bespoke code for this. Options (2) and (3) suffer from an issue where-in the `ExportConfig` has a `schema_name` attr, but the `UnparsedExportConfig` uses `schema` (this is necessary for other reasons). Additionally option (2) would make an export a pseudo node, which we don't want to do unintentionally.

As it stands it seems like there are two paths forward: option (3) but use the `UnrenderedExportConfig` during the config generation process and then after the fact map things to the `ExportConfig` or a forth option would be to refactor our config parsing process to be less clever.

### Acceptance criteria

We have implemented a way for `Export` configs to be set hierarchically AND it is tested

### Impact to Other Teams

Semantic Layer, maybe  also Cloud Artifacts?

### Will backports be required?

Yes 1.7

### Context

_No response_


Discussion:

Resolving patch:
diff --git a/core/dbt/contracts/graph/model_config.py b/core/dbt/contracts/graph/model_config.py
--- a/core/dbt/contracts/graph/model_config.py
+++ b/core/dbt/contracts/graph/model_config.py
@@ -15,6 +15,7 @@
 from dbt.exceptions import DbtInternalError, CompilationError
 from dbt import hooks
 from dbt.node_types import NodeType, AccessType
+from dbt_semantic_interfaces.type_enums.export_destination_type import ExportDestinationType
 from mashumaro.jsonschema.annotations import Pattern
 
 
@@ -407,6 +408,8 @@ class SavedQueryConfig(BaseConfig):
         default_factory=dict,
         metadata=MergeBehavior.Update.meta(),
     )
+    export_as: Optional[ExportDestinationType] = None
+    schema: Optional[str] = None
 
 
 @dataclass
diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py
--- a/core/dbt/contracts/graph/unparsed.py
+++ b/core/dbt/contracts/graph/unparsed.py
@@ -19,7 +19,6 @@
 from dbt.exceptions import CompilationError, ParsingError, DbtInternalError
 
 from dbt.dataclass_schema import dbtClassMixin, StrEnum, ExtensibleDbtClassMixin, ValidationError
-from dbt_semantic_interfaces.type_enums.export_destination_type import ExportDestinationType
 
 from dataclasses import dataclass, field
 from datetime import timedelta
@@ -729,21 +728,12 @@ class UnparsedQueryParams(dbtClassMixin):
     where: Optional[Union[str, List[str]]] = None
 
 
-@dataclass
-class UnparsedExportConfig(dbtClassMixin):
-    """Nested configuration attributes for exports."""
-
-    export_as: ExportDestinationType
-    schema: Optional[str] = None
-    alias: Optional[str] = None
-
-
 @dataclass
 class UnparsedExport(dbtClassMixin):
     """Configuration for writing query results to a table."""
 
     name: str
-    config: UnparsedExportConfig
+    config: Dict[str, Any] = field(default_factory=dict)
 
 
 @dataclass
diff --git a/core/dbt/parser/schema_yaml_readers.py b/core/dbt/parser/schema_yaml_readers.py
--- a/core/dbt/parser/schema_yaml_readers.py
+++ b/core/dbt/parser/schema_yaml_readers.py
@@ -6,7 +6,6 @@
     UnparsedDimensionTypeParams,
     UnparsedEntity,
     UnparsedExport,
-    UnparsedExportConfig,
     UnparsedExposure,
     UnparsedGroup,
     UnparsedMeasure,
@@ -19,6 +18,7 @@
     UnparsedSavedQuery,
     UnparsedSemanticModel,
 )
+from dbt.contracts.graph.model_config import SavedQueryConfig
 from dbt.contracts.graph.nodes import (
     Exposure,
     Group,
@@ -57,7 +57,7 @@
     MetricType,
     TimeGranularity,
 )
-from typing import List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 
 def parse_where_filter(
@@ -669,16 +669,25 @@ def _generate_saved_query_config(
 
         return config
 
-    def _get_export_config(self, unparsed: UnparsedExportConfig) -> ExportConfig:
-        return ExportConfig(
-            export_as=unparsed.export_as,
-            schema_name=unparsed.schema,
-            alias=unparsed.alias,
+    def _get_export_config(
+        self, unparsed_export_config: Dict[str, Any], saved_query_config: SavedQueryConfig
+    ) -> ExportConfig:
+        # Combine the two dictionaries using dictionary unpacking
+        # the second dictionary is the one whose keys take priority
+        combined = {**saved_query_config.__dict__, **unparsed_export_config}
+        # `schema` is the user facing attribute, but for DSI protocol purposes we track it as `schema_name`
+        if combined.get("schema") is not None and combined.get("schema_name") is None:
+            combined["schema_name"] = combined["schema"]
+
+        return ExportConfig.from_dict(combined)
+
+    def _get_export(
+        self, unparsed: UnparsedExport, saved_query_config: SavedQueryConfig
+    ) -> Export:
+        return Export(
+            name=unparsed.name, config=self._get_export_config(unparsed.config, saved_query_config)
         )
 
-    def _get_export(self, unparsed: UnparsedExport) -> Export:
-        return Export(name=unparsed.name, config=self._get_export_config(unparsed.config))
-
     def _get_query_params(self, unparsed: UnparsedQueryParams) -> QueryParams:
         return QueryParams(
             group_by=unparsed.group_by,
@@ -721,7 +730,7 @@ def parse_saved_query(self, unparsed: UnparsedSavedQuery) -> None:
             resource_type=NodeType.SavedQuery,
             unique_id=unique_id,
             query_params=self._get_query_params(unparsed.query_params),
-            exports=[self._get_export(export) for export in unparsed.exports],
+            exports=[self._get_export(export, config) for export in unparsed.exports],
             config=config,
             unrendered_config=unrendered_config,
             group=config.group,


Test patch:
diff --git a/tests/functional/saved_queries/fixtures.py b/tests/functional/saved_queries/fixtures.py
--- a/tests/functional/saved_queries/fixtures.py
+++ b/tests/functional/saved_queries/fixtures.py
@@ -24,3 +24,70 @@
             export_as: table
             schema: my_export_schema_name
 """
+
+saved_query_with_extra_config_attributes_yml = """
+version: 2
+
+saved_queries:
+  - name: test_saved_query
+    description: "{{ doc('saved_query_description') }}"
+    label: Test Saved Query
+    query_params:
+        metrics:
+            - simple_metric
+        group_by:
+            - "Dimension('user__ds')"
+        where:
+            - "{{ Dimension('user__ds', 'DAY') }} <= now()"
+            - "{{ Dimension('user__ds', 'DAY') }} >= '2023-01-01'"
+    exports:
+        - name: my_export
+          config:
+            my_random_config: 'I have this for some reason'
+            export_as: table
+"""
+
+saved_query_with_export_configs_defined_at_saved_query_level_yml = """
+version: 2
+
+saved_queries:
+  - name: test_saved_query
+    description: "{{ doc('saved_query_description') }}"
+    label: Test Saved Query
+    config:
+      export_as: table
+      schema: my_default_export_schema
+    query_params:
+        metrics:
+            - simple_metric
+        group_by:
+            - "Dimension('user__ds')"
+        where:
+            - "{{ Dimension('user__ds', 'DAY') }} <= now()"
+            - "{{ Dimension('user__ds', 'DAY') }} >= '2023-01-01'"
+    exports:
+        - name: my_export
+          config:
+            export_as: view
+            schema: my_custom_export_schema
+        - name: my_export2
+"""
+
+saved_query_without_export_configs_defined_yml = """
+version: 2
+
+saved_queries:
+  - name: test_saved_query
+    description: "{{ doc('saved_query_description') }}"
+    label: Test Saved Query
+    query_params:
+        metrics:
+            - simple_metric
+        group_by:
+            - "Dimension('user__ds')"
+        where:
+            - "{{ Dimension('user__ds', 'DAY') }} <= now()"
+            - "{{ Dimension('user__ds', 'DAY') }} >= '2023-01-01'"
+    exports:
+        - name: my_export
+"""
diff --git a/tests/functional/saved_queries/test_configs.py b/tests/functional/saved_queries/test_configs.py
new file mode 100644
--- /dev/null
+++ b/tests/functional/saved_queries/test_configs.py
@@ -0,0 +1,186 @@
+import pytest
+
+from dbt.contracts.graph.manifest import Manifest
+from dbt.tests.util import update_config_file
+from dbt_semantic_interfaces.type_enums.export_destination_type import ExportDestinationType
+from tests.functional.assertions.test_runner import dbtTestRunner
+from tests.functional.configs.fixtures import BaseConfigProject
+from tests.functional.saved_queries.fixtures import (
+    saved_queries_yml,
+    saved_query_description,
+    saved_query_with_extra_config_attributes_yml,
+    saved_query_with_export_configs_defined_at_saved_query_level_yml,
+    saved_query_without_export_configs_defined_yml,
+)
+from tests.functional.semantic_models.fixtures import (
+    fct_revenue_sql,
+    metricflow_time_spine_sql,
+    schema_yml,
+)
+
+
+class TestSavedQueryConfigs(BaseConfigProject):
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {
+            "saved-queries": {
+                "test": {
+                    "test_saved_query": {
+                        "+enabled": True,
+                        "+export_as": ExportDestinationType.VIEW.value,
+                        "+schema": "my_default_export_schema",
+                    }
+                },
+            },
+        }
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "saved_queries.yml": saved_query_with_extra_config_attributes_yml,
+            "schema.yml": schema_yml,
+            "fct_revenue.sql": fct_revenue_sql,
+            "metricflow_time_spine.sql": metricflow_time_spine_sql,
+            "docs.md": saved_query_description,
+        }
+
+    def test_basic_saved_query_config(
+        self,
+        project,
+    ):
+        runner = dbtTestRunner()
+
+        # parse with default fixture project config
+        result = runner.invoke(["parse"])
+        assert result.success
+        assert isinstance(result.result, Manifest)
+        assert len(result.result.saved_queries) == 1
+        saved_query = result.result.saved_queries["saved_query.test.test_saved_query"]
+        assert saved_query.config.export_as == ExportDestinationType.VIEW
+        assert saved_query.config.schema == "my_default_export_schema"
+
+        # disable the saved_query via project config and rerun
+        config_patch = {"saved-queries": {"test": {"test_saved_query": {"+enabled": False}}}}
+        update_config_file(config_patch, project.project_root, "dbt_project.yml")
+        result = runner.invoke(["parse"])
+        assert result.success
+        assert len(result.result.saved_queries) == 0
+
+
+class TestExportConfigsWithAdditionalProperties(BaseConfigProject):
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "saved_queries.yml": saved_queries_yml,
+            "schema.yml": schema_yml,
+            "fct_revenue.sql": fct_revenue_sql,
+            "metricflow_time_spine.sql": metricflow_time_spine_sql,
+            "docs.md": saved_query_description,
+        }
+
+    def test_extra_config_properties_dont_break_parsing(self, project):
+        runner = dbtTestRunner()
+
+        # parse with default fixture project config
+        result = runner.invoke(["parse"])
+        assert result.success
+        assert isinstance(result.result, Manifest)
+        assert len(result.result.saved_queries) == 1
+        saved_query = result.result.saved_queries["saved_query.test.test_saved_query"]
+        assert len(saved_query.exports) == 1
+        assert saved_query.exports[0].config.__dict__.get("my_random_config") is None
+
+
+class TestInheritingExportConfigFromSavedQueryConfig(BaseConfigProject):
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "saved_queries.yml": saved_query_with_export_configs_defined_at_saved_query_level_yml,
+            "schema.yml": schema_yml,
+            "fct_revenue.sql": fct_revenue_sql,
+            "metricflow_time_spine.sql": metricflow_time_spine_sql,
+            "docs.md": saved_query_description,
+        }
+
+    def test_export_config_inherits_from_saved_query(self, project):
+        runner = dbtTestRunner()
+
+        # parse with default fixture project config
+        result = runner.invoke(["parse"])
+        assert result.success
+        assert isinstance(result.result, Manifest)
+        assert len(result.result.saved_queries) == 1
+        saved_query = result.result.saved_queries["saved_query.test.test_saved_query"]
+        assert len(saved_query.exports) == 2
+
+        # assert Export `my_export` has its configs defined from itself because they should take priority
+        export1 = next(
+            (export for export in saved_query.exports if export.name == "my_export"), None
+        )
+        assert export1 is not None
+        assert export1.config.export_as == ExportDestinationType.VIEW
+        assert export1.config.export_as != saved_query.config.export_as
+        assert export1.config.schema_name == "my_custom_export_schema"
+        assert export1.config.schema_name != saved_query.config.schema
+
+        # assert Export `my_export` has its configs defined from the saved_query because they should take priority
+        export2 = next(
+            (export for export in saved_query.exports if export.name == "my_export2"), None
+        )
+        assert export2 is not None
+        assert export2.config.export_as == ExportDestinationType.TABLE
+        assert export2.config.export_as == saved_query.config.export_as
+        assert export2.config.schema_name == "my_default_export_schema"
+        assert export2.config.schema_name == saved_query.config.schema
+
+
+class TestInheritingExportConfigsFromProject(BaseConfigProject):
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {
+            "saved-queries": {
+                "test": {
+                    "test_saved_query": {
+                        "+export_as": ExportDestinationType.VIEW.value,
+                    }
+                },
+            },
+        }
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "saved_queries.yml": saved_query_without_export_configs_defined_yml,
+            "schema.yml": schema_yml,
+            "fct_revenue.sql": fct_revenue_sql,
+            "metricflow_time_spine.sql": metricflow_time_spine_sql,
+            "docs.md": saved_query_description,
+        }
+
+    def test_export_config_inherits_from_project(
+        self,
+        project,
+    ):
+        runner = dbtTestRunner()
+
+        # parse with default fixture project config
+        result = runner.invoke(["parse"])
+        assert result.success
+        assert isinstance(result.result, Manifest)
+        assert len(result.result.saved_queries) == 1
+        saved_query = result.result.saved_queries["saved_query.test.test_saved_query"]
+        assert saved_query.config.export_as == ExportDestinationType.VIEW
+
+        # change export's `export_as` to `TABLE` via project config
+        config_patch = {
+            "saved-queries": {
+                "test": {"test_saved_query": {"+export_as": ExportDestinationType.TABLE.value}}
+            }
+        }
+        update_config_file(config_patch, project.project_root, "dbt_project.yml")
+        result = runner.invoke(["parse"])
+        assert result.success
+        assert isinstance(result.result, Manifest)
+        assert len(result.result.saved_queries) == 1
+        saved_query = result.result.saved_queries["saved_query.test.test_saved_query"]
+        assert saved_query.config.export_as == ExportDestinationType.TABLE