Repository: dbt-labs/dbt-core

Issue:
[CT-3005] Support dictionary of additional configs in `dbt_project.yml`
### Housekeeping

- [X] I am a maintainer of dbt-core

### Short description

Allow setting a dictionary of arbitrary configs in `dbt_project.yml`, which are to be read & used by dbt Cloud.

We don't need to validate the keys in this dictionary, nor store them in the project config class.

Let's call this dictionary `dbt-cloud`:
```yml
# dbt_project.yml
dbt-cloud:
  project-id: 1234
  ...
```

### Acceptance criteria

It is possible to set a dictionary of arbitrary configs in `dbt_project.yml`, as metadata for dbt Cloud.

### Impact to Other Teams

Cloud CLI impacted (see jira ink below), ensure this is completed within this sprint

### Context

[internal slack thread](https://dbt-labs.slack.com/archives/C044WU55X7Y/p1691671419131039)
[internal jira ticket](https://dbtlabs.atlassian.net/browse/CLI-9)


Discussion:
Open question:
- how will they access these keys? are they intending on parsing these themselves? 
  - via reading the `dbt_project.yml` file
  - yes, they would parse these themselves
- what should we name this? 
  - let's go with ~`cloud`~ `dbt-cloud`

response discussed [here](https://dbt-labs.slack.com/archives/C051TUB7S9W/p1692631638644559) and [here](https://dbt-labs.slack.com/archives/C051TUB7S9W/p1693236807990039?thread_ts=1692971547.200749&cid=C051TUB7S9W)

Resolving patch:
diff --git a/core/dbt/config/project.py b/core/dbt/config/project.py
--- a/core/dbt/config/project.py
+++ b/core/dbt/config/project.py
@@ -429,6 +429,7 @@ def create_project(self, rendered: RenderComponents) -> "Project":
         semantic_models: Dict[str, Any]
         exposures: Dict[str, Any]
         vars_value: VarProvider
+        dbt_cloud: Dict[str, Any]
 
         dispatch = cfg.dispatch
         models = cfg.models
@@ -461,6 +462,8 @@ def create_project(self, rendered: RenderComponents) -> "Project":
             manifest_selectors = SelectorDict.parse_from_selectors_list(
                 rendered.selectors_dict["selectors"]
             )
+        dbt_cloud = cfg.dbt_cloud
+
         project = Project(
             project_name=name,
             version=version,
@@ -501,6 +504,7 @@ def create_project(self, rendered: RenderComponents) -> "Project":
             unrendered=unrendered,
             project_env_vars=project_env_vars,
             restrict_access=cfg.restrict_access,
+            dbt_cloud=dbt_cloud,
         )
         # sanity check - this means an internal issue
         project.validate()
@@ -613,6 +617,7 @@ class Project:
     unrendered: RenderComponents
     project_env_vars: Dict[str, Any]
     restrict_access: bool
+    dbt_cloud: Dict[str, Any]
 
     @property
     def all_source_paths(self) -> List[str]:
@@ -683,6 +688,7 @@ def to_project_config(self, with_packages=False):
                 "require-dbt-version": [v.to_version_string() for v in self.dbt_version],
                 "config-version": self.config_version,
                 "restrict-access": self.restrict_access,
+                "dbt-cloud": self.dbt_cloud,
             }
         )
         if self.query_comment:
diff --git a/core/dbt/config/runtime.py b/core/dbt/config/runtime.py
--- a/core/dbt/config/runtime.py
+++ b/core/dbt/config/runtime.py
@@ -183,6 +183,7 @@ def from_parts(
             args=args,
             cli_vars=cli_vars,
             dependencies=dependencies,
+            dbt_cloud=project.dbt_cloud,
         )
 
     # Called by 'load_projects' in this class
diff --git a/core/dbt/contracts/project.py b/core/dbt/contracts/project.py
--- a/core/dbt/contracts/project.py
+++ b/core/dbt/contracts/project.py
@@ -225,6 +225,7 @@ class Project(dbtClassMixin, Replaceable):
     packages: List[PackageSpec] = field(default_factory=list)
     query_comment: Optional[Union[QueryComment, NoValue, str]] = field(default_factory=NoValue)
     restrict_access: bool = False
+    dbt_cloud: Optional[Dict[str, Any]] = None
 
     class Config(dbtMashConfig):
         # These tell mashumaro to use aliases for jsonschema and for "from_dict"
@@ -251,6 +252,7 @@ class Config(dbtMashConfig):
             "query_comment": "query-comment",
             "restrict_access": "restrict-access",
             "semantic_models": "semantic-models",
+            "dbt_cloud": "dbt-cloud",
         }
 
     @classmethod
@@ -268,6 +270,10 @@ def validate(cls, data):
                     or not isinstance(entry["search_order"], list)
                 ):
                     raise ValidationError(f"Invalid project dispatch config: {entry}")
+        if "dbt_cloud" in data and not isinstance(data["dbt_cloud"], dict):
+            raise ValidationError(
+                f"Invalid dbt_cloud config. Expected a 'dict' but got '{type(data['dbt_cloud'])}'"
+            )
 
 
 @dataclass


Test patch:
diff --git a/tests/functional/basic/test_project.py b/tests/functional/basic/test_project.py
--- a/tests/functional/basic/test_project.py
+++ b/tests/functional/basic/test_project.py
@@ -1,5 +1,8 @@
+import os
 import pytest
-from dbt.tests.util import run_dbt, update_config_file
+import yaml
+from pathlib import Path
+from dbt.tests.util import run_dbt, update_config_file, write_config_file
 from dbt.exceptions import ProjectContractError
 
 
@@ -62,3 +65,50 @@ def test_invalid_version(self, project):
         assert "at path ['version']: 'invalid' is not valid under any of the given schemas" in str(
             excinfo.value
         )
+
+
+class TestProjectDbtCloudConfig:
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {"simple_model.sql": simple_model_sql, "simple_model.yml": simple_model_yml}
+
+    def test_dbt_cloud(self, project):
+        run_dbt(["parse"], expect_pass=True)
+        conf = yaml.safe_load(
+            Path(os.path.join(project.project_root, "dbt_project.yml")).read_text()
+        )
+        assert conf == {"name": "test", "profile": "test"}
+
+        config = {
+            "name": "test",
+            "profile": "test",
+            "dbt-cloud": {
+                "account_id": "123",
+                "application": "test",
+                "environment": "test",
+                "api_key": "test",
+            },
+        }
+        write_config_file(config, project.project_root, "dbt_project.yml")
+        run_dbt(["parse"], expect_pass=True)
+        conf = yaml.safe_load(
+            Path(os.path.join(project.project_root, "dbt_project.yml")).read_text()
+        )
+        assert conf == config
+
+
+class TestProjectDbtCloudConfigString:
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {"simple_model.sql": simple_model_sql, "simple_model.yml": simple_model_yml}
+
+    def test_dbt_cloud_invalid(self, project):
+        run_dbt()
+        config = {"name": "test", "profile": "test", "dbt-cloud": "Some string"}
+        update_config_file(config, "dbt_project.yml")
+        expected_err = (
+            "at path ['dbt-cloud']: 'Some string' is not valid under any of the given schemas"
+        )
+        with pytest.raises(ProjectContractError) as excinfo:
+            run_dbt()
+        assert expected_err in str(excinfo.value)