Repository: dbt-labs/dbt-core

Issue:
Custom environment metadata in JSON artifacts
### Describe the feature
Include custom environment-sourced metadata in the JSON artifacts.

Inside the `metadata` field (or at the root if #2761 is not done yet), dbt should add a field - tentatively `env`. This field should be a dictionary, populated from the contents of dbt's environment that start with `DBT_ENV_CUSTOM_ENV_`. The keys should be the environment variables without the prefix, and the values the values. For example:
DBT_ENV_CUSTOM_ENV_run_id=1 DBT_ENV_CUSTOM_ENV_run_env=ci dbt run
will result in a manifest with `{"env": {"run_id": "1", "run_env": "ci"}}` in its metadata field.

See [this comment](https://github.com/fishtown-analytics/dbt/issues/2671#issuecomment-685892719) for a tiny bit of context.

Split off from #2671 



Discussion:

Resolving patch:
diff --git a/core/dbt/contracts/util.py b/core/dbt/contracts/util.py
--- a/core/dbt/contracts/util.py
+++ b/core/dbt/contracts/util.py
@@ -1,4 +1,5 @@
 import dataclasses
+import os
 from datetime import datetime
 from typing import (
     List, Tuple, ClassVar, Type, TypeVar, Dict, Any, Optional
@@ -123,6 +124,16 @@ def __str__(self) -> str:
 SCHEMA_VERSION_KEY = 'dbt_schema_version'
 
 
+METADATA_ENV_PREFIX = 'DBT_ENV_CUSTOM_ENV_'
+
+
+def get_metadata_env() -> Dict[str, str]:
+    return {
+        k[len(METADATA_ENV_PREFIX):]: v for k, v in os.environ.items()
+        if k.startswith(METADATA_ENV_PREFIX)
+    }
+
+
 @dataclasses.dataclass
 class BaseArtifactMetadata(JsonSchemaMixin):
     dbt_schema_version: str
@@ -133,6 +144,7 @@ class BaseArtifactMetadata(JsonSchemaMixin):
     invocation_id: Optional[str] = dataclasses.field(
         default_factory=get_invocation_id
     )
+    env: Dict[str, str] = dataclasses.field(default_factory=get_metadata_env)
 
 
 def schema_version(name: str, version: int):


Test patch:
diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py
--- a/test/integration/029_docs_generate_tests/test_docs_generate.py
+++ b/test/integration/029_docs_generate_tests/test_docs_generate.py
@@ -5,7 +5,6 @@
 import shutil
 import tempfile
 import time
-import uuid
 from datetime import datetime
 from unittest.mock import ANY, patch
 
@@ -112,6 +111,12 @@ def setUp(self):
             self.alternate_schema = self.alternate_schema.upper()
 
         self._created_schemas.add(self.alternate_schema)
+        os.environ['DBT_ENV_CUSTOM_ENV_env_key'] = 'env_value'
+
+    def tearDown(self):
+        super().tearDown()
+        del os.environ['DBT_ENV_CUSTOM_ENV_env_key']
+
 
     @property
     def schema(self):
@@ -2563,6 +2568,12 @@ def verify_metadata(self, metadata, dbt_schema_version):
         assert 'dbt_schema_version' in metadata
         assert metadata['dbt_schema_version'] == dbt_schema_version
         assert metadata['invocation_id'] == dbt.tracking.active_user.invocation_id
+        key = 'env_key'
+        if os.name == 'nt':
+            key = key.upper()
+        assert metadata['env'] == {
+            key: 'env_value'
+        }
 
     def verify_manifest(self, expected_manifest):
         self.assertTrue(os.path.exists('./target/manifest.json'))
diff --git a/test/integration/042_sources_test/test_sources.py b/test/integration/042_sources_test/test_sources.py
--- a/test/integration/042_sources_test/test_sources.py
+++ b/test/integration/042_sources_test/test_sources.py
@@ -56,6 +56,11 @@ def setUp(self):
         self._id = 101
         # this is the db initial value
         self.last_inserted_time = "2016-09-19T14:45:51+00:00"
+        os.environ['DBT_ENV_CUSTOM_ENV_key'] = 'value'
+
+    def tearDown(self):
+        super().tearDown()
+        del os.environ['DBT_ENV_CUSTOM_ENV_key']
 
     def _set_updated_at_to(self, delta):
         insert_time = datetime.utcnow() + delta
@@ -244,6 +249,10 @@ def _assert_freshness_results(self, path, state):
         assert data['metadata']['dbt_schema_version'] == 'https://schemas.getdbt.com/dbt/sources/v1.json'
         assert data['metadata']['dbt_version'] == dbt.version.__version__
         assert data['metadata']['invocation_id'] == dbt.tracking.active_user.invocation_id
+        key = 'key'
+        if os.name == 'nt':
+            key = key.upper()
+        assert data['metadata']['env'] == {key: 'value'}
 
 
         last_inserted_time = self.last_inserted_time
diff --git a/test/unit/test_manifest.py b/test/unit/test_manifest.py
--- a/test/unit/test_manifest.py
+++ b/test/unit/test_manifest.py
@@ -1,3 +1,4 @@
+import os
 import unittest
 from unittest import mock
 
@@ -41,6 +42,9 @@
 })
 
 
+ENV_KEY_NAME = 'KEY' if os.name == 'nt' else 'key'
+
+
 class ManifestTest(unittest.TestCase):
     def setUp(self):
         dbt.flags.STRICT_MODE = True
@@ -212,6 +216,11 @@ def setUp(self):
         for source in self.sources.values():
             source.validate(source.to_dict())
 
+        os.environ['DBT_ENV_CUSTOM_ENV_key'] = 'value'
+
+    def tearDown(self):
+        del os.environ['DBT_ENV_CUSTOM_ENV_key']
+
     @freezegun.freeze_time('2018-02-14T09:15:13Z')
     def test__no_nodes(self):
         manifest = Manifest(
@@ -231,6 +240,7 @@ def test__no_nodes(self):
                     'generated_at': '2018-02-14T09:15:13Z',
                     'dbt_schema_version': 'https://schemas.getdbt.com/dbt/manifest/v1.json',
                     'dbt_version': dbt.version.__version__,
+                    'env': {ENV_KEY_NAME: 'value'},
                     # invocation_id is None, so it will not be present
                 },
                 'docs': {},
@@ -376,6 +386,7 @@ def test_no_nodes_with_metadata(self, mock_user):
                     'send_anonymous_usage_stats': False,
                     'adapter_type': 'postgres',
                     'invocation_id': '01234567-0123-0123-0123-0123456789ab',
+                    'env': {ENV_KEY_NAME: 'value'},
                 },
                 'disabled': [],
             }
@@ -586,6 +597,10 @@ def setUp(self):
                 checksum=FileHash.empty(),
             ),
         }
+        os.environ['DBT_ENV_CUSTOM_ENV_key'] = 'value'
+
+    def tearDown(self):
+        del os.environ['DBT_ENV_CUSTOM_ENV_key']
 
     @freezegun.freeze_time('2018-02-14T09:15:13Z')
     def test__no_nodes(self):
@@ -607,6 +622,7 @@ def test__no_nodes(self):
                     'dbt_schema_version': 'https://schemas.getdbt.com/dbt/manifest/v1.json',
                     'dbt_version': dbt.version.__version__,
                     'invocation_id': '01234567-0123-0123-0123-0123456789ab',
+                    'env': {ENV_KEY_NAME: 'value'},
                 },
                 'docs': {},
                 'disabled': [],