Repository: dbt-labs/dbt-core

Issue:
[CT-896] [Feature] Add Jinja function which returns all `DBT_ENV_CUSTOM_ENV_ ` and `DBT_CLOUD_` environment variables
### Is this your first time opening an issue?

- [X] I have read the [expectations for open source contributors](https://docs.getdbt.com/docs/contributing/oss-expectations)

### Describe the Feature

Right now, you have to know which environment variables might be set in order to get the values for them using the `env_var` function. For the dbt_artifacts package (and other like it) it would be helpful to be able to call a single function which returns all `DBT_ENV_CUSTOM_ENV_` and `DBT_CLOUD_` environment variables, in the same way they are all present in the manifest.json's metadata env field.

### Describe alternatives you've considered

Asking users to configure variables to describe their in-use env vars

### Who will this benefit?

_No response_

### Are you interested in contributing this feature?

_No response_

### Anything else?

_No response_


Discussion:
Legit! Thanks for opening @NiallRees.

I don't think there would be any security concerns related to this, as long as we keep it tightly locked down to _just_ the environment variables with the appropriate prefix.

It just needs to be `DBT_ENV_CUSTOM_ENV_`: That's all we document [here](https://docs.getdbt.com/reference/artifacts/dbt-artifacts#common-metadata), and have the logic for here:
https://github.com/dbt-labs/dbt-core/blob/94a7cfa58df3df5d3b07b619c0510fead9195700/core/dbt/contracts/util.py#L153-L161

The way that dbt Cloud sets the environment variables is: `DBT_ENV_CUSTOM_ENV_DBT_CLOUD_JOB_ID`.

We have a few different definitions of the `env_var` context method, in order to ensure that it works properly with partial parsing. (We need to know whether the `env_var` call happened in a yaml file, a SQL file, or the profile / project file, in order to know what needs re-parsing when the env var value changes.) I think we do _not_ care about partial parsing for this method: The whole point is just to pass along metadata, not to use those metadata vars to actually set project configuration.

So, I'd be inclined to say:
- We add `metadata_env()` as a function in the [ProviderContext](https://github.com/dbt-labs/dbt-core/blob/94a7cfa58df3df5d3b07b619c0510fead9195700/core/dbt/context/providers.py#L678), a.k.a. the runtime context. It won't be accessible at parse time, which is intentional. (I could be convinced by a member of Core-Language that the BaseContext is just better for simplicity, though.)
- It returns _exactly_ the value of `get_metadata_env()` above. Heck, it may as well import and use the function.

I'm going to mark this a `help_wanted`, since it will also require updating a few tests. Is that something you'd be interested in contributing?

Resolving patch:
diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py
--- a/core/dbt/context/providers.py
+++ b/core/dbt/context/providers.py
@@ -41,6 +41,7 @@
     ParsedSourceDefinition,
 )
 from dbt.contracts.graph.metrics import MetricReference, ResolvedMetricReference
+from dbt.contracts.util import get_metadata_env
 from dbt.exceptions import (
     CompilationException,
     ParsingException,
@@ -710,6 +711,10 @@ def _get_namespace_builder(self):
             self.model,
         )
 
+    @contextproperty
+    def dbt_metadata_envs(self) -> Dict[str, str]:
+        return get_metadata_env()
+
     @contextproperty
     def invocation_args_dict(self):
         return args_to_dict(self.config.args)


Test patch:
diff --git a/test/unit/test_context.py b/test/unit/test_context.py
--- a/test/unit/test_context.py
+++ b/test/unit/test_context.py
@@ -200,7 +200,7 @@ def assert_has_keys(required_keys: Set[str], maybe_keys: Set[str], ctx: Dict[str
         "modules",
         "flags",
         "print",
-        "diff_of_two_dicts"
+        "diff_of_two_dicts",
     }
 )
 
@@ -239,6 +239,7 @@ def assert_has_keys(required_keys: Set[str], maybe_keys: Set[str], ctx: Dict[str
     "adapter_macro",
     "selected_resources",
     "invocation_args_dict",
+    "dbt_metadata_envs"
 }
 REQUIRED_MODEL_KEYS = REQUIRED_MACRO_KEYS | {"this", "compiled_code",}
 MAYBE_KEYS = frozenset({"debug"})
@@ -499,3 +500,21 @@ def test_macro_namespace(config_postgres, manifest_fx):
         assert result["dbt"]["some_macro"].macro is pg_macro
         assert result["root"]["some_macro"].macro is package_macro
         assert result["some_macro"].macro is package_macro
+
+def test_dbt_metadata_envs(monkeypatch, config_postgres, manifest_fx, get_adapter, get_include_paths):
+    envs = {
+        "DBT_ENV_CUSTOM_ENV_RUN_ID": 1234,
+        "DBT_ENV_CUSTOM_ENV_JOB_ID": 5678,
+        "DBT_ENV_RUN_ID": 91011,
+        "RANDOM_ENV": 121314
+    }
+    monkeypatch.setattr(os, 'environ', envs)
+
+    ctx = providers.generate_runtime_macro_context(
+        macro=manifest_fx.macros["macro.root.macro_a"],
+        config=config_postgres,
+        manifest=manifest_fx,
+        package_name="root",
+    ) 
+
+    assert ctx["dbt_metadata_envs"] == {'JOB_ID': 5678, 'RUN_ID': 1234}
diff --git a/tests/functional/context_methods/test_builtin_functions.py b/tests/functional/context_methods/test_builtin_functions.py
--- a/tests/functional/context_methods/test_builtin_functions.py
+++ b/tests/functional/context_methods/test_builtin_functions.py
@@ -1,5 +1,6 @@
 import pytest
 import json
+import os
 
 from dbt.tests.util import run_dbt, run_dbt_and_capture, write_file
 from dbt.exceptions import CompilationException
@@ -30,6 +31,12 @@
 {% endmacro %}
 """
 
+macros__validate_dbt_metadata_envs_sql = """
+{% macro validate_dbt_metadata_envs() %}
+    {{ log("dbt_metadata_envs_result:"~ dbt_metadata_envs) }}
+{% endmacro %}
+"""
+
 models__set_exception_sql = """
 {% set set_strict_result = set_strict(1) %}
 """
@@ -39,6 +46,26 @@
 """
 
 
+def parse_json_logs(json_log_output):
+    parsed_logs = []
+    for line in json_log_output.split("\n"):
+        try:
+            log = json.loads(line)
+        except ValueError:
+            continue
+
+        parsed_logs.append(log)
+
+    return parsed_logs
+
+
+def find_result_in_parsed_logs(parsed_logs, result_name):
+    return next(
+        (item for item in parsed_logs if result_name in item["data"].get("msg", "msg")),
+        False,
+    )
+
+
 class TestContextBuiltins:
     @pytest.fixture(scope="class")
     def macros(self):
@@ -46,6 +73,7 @@ def macros(self):
             "validate_set.sql": macros__validate_set_sql,
             "validate_zip.sql": macros__validate_zip_sql,
             "validate_invocation.sql": macros__validate_invocation_sql,
+            "validate_dbt_metadata_envs.sql": macros__validate_dbt_metadata_envs_sql,
         }
 
     def test_builtin_set_function(self, project):
@@ -75,24 +103,8 @@ def test_builtin_invocation_args_dict_function(self, project):
             ]
         )
 
-        parsed_logs = []
-        for line in log_output.split("\n"):
-            try:
-                log = json.loads(line)
-            except ValueError:
-                continue
-
-            parsed_logs.append(log)
-
-        # Value of msg is a string
-        result = next(
-            (
-                item
-                for item in parsed_logs
-                if "invocation_result" in item["data"].get("msg", "msg")
-            ),
-            False,
-        )
+        parsed_logs = parse_json_logs(log_output)
+        result = find_result_in_parsed_logs(parsed_logs, "invocation_result")
 
         assert result
 
@@ -103,6 +115,27 @@ def test_builtin_invocation_args_dict_function(self, project):
         expected = "'send_anonymous_usage_stats': False, 'event_buffer_size': 100000, 'quiet': False, 'no_print': False, 'macro': 'validate_invocation', 'args': '{my_variable: test_variable}', 'which': 'run-operation', 'rpc_method': 'run-operation', 'indirect_selection': 'eager'}"
         assert expected in str(result)
 
+    def test_builtin_dbt_metadata_envs_function(self, project, monkeypatch):
+        envs = {
+            "DBT_ENV_CUSTOM_ENV_RUN_ID": 1234,
+            "DBT_ENV_CUSTOM_ENV_JOB_ID": 5678,
+            "DBT_ENV_RUN_ID": 91011,
+            "RANDOM_ENV": 121314,
+        }
+        monkeypatch.setattr(os, "environ", envs)
+
+        _, log_output = run_dbt_and_capture(
+            ["--debug", "--log-format=json", "run-operation", "validate_dbt_metadata_envs"]
+        )
+
+        parsed_logs = parse_json_logs(log_output)
+        result = find_result_in_parsed_logs(parsed_logs, "dbt_metadata_envs_result")
+
+        assert result
+
+        expected = "dbt_metadata_envs_result:{'RUN_ID': 1234, 'JOB_ID': 5678}"
+        assert expected in str(result)
+
 
 class TestContextBuiltinExceptions:
     # Assert compilation errors are raised with _strict equivalents