Repository: dbt-labs/dbt-core

Issue:
[CT-293] add context README
Fill out `core/dbt/context/README.md`


Discussion:

Resolving patch:
diff --git a/core/dbt/config/renderer.py b/core/dbt/config/renderer.py
--- a/core/dbt/config/renderer.py
+++ b/core/dbt/config/renderer.py
@@ -1,12 +1,15 @@
 from typing import Dict, Any, Tuple, Optional, Union, Callable
+import re
+import os
 
 from dbt.clients.jinja import get_rendered, catch_jinja
 from dbt.context.target import TargetContext
-from dbt.context.secret import SecretContext
+from dbt.context.secret import SecretContext, SECRET_PLACEHOLDER
 from dbt.context.base import BaseContext
 from dbt.contracts.connection import HasCredentials
 from dbt.exceptions import DbtProjectError, CompilationException, RecursionException
 from dbt.utils import deep_map_render
+from dbt.logger import SECRET_ENV_PREFIX
 
 
 Keypath = Tuple[Union[str, int], ...]
@@ -174,6 +177,23 @@ def __init__(self, cli_vars: Dict[str, Any] = {}) -> None:
     def name(self):
         return "Secret"
 
+    def render_value(self, value: Any, keypath: Optional[Keypath] = None) -> Any:
+        rendered = super().render_value(value, keypath)
+        if SECRET_ENV_PREFIX in str(rendered):
+            search_group = f"({SECRET_ENV_PREFIX}(.*))"
+            pattern = SECRET_PLACEHOLDER.format(search_group).replace("$", r"\$")
+            m = re.search(
+                pattern,
+                rendered,
+            )
+            if m:
+                found = m.group(1)
+                value = os.environ[found]
+                replace_this = SECRET_PLACEHOLDER.format(found)
+                return rendered.replace(replace_this, value)
+        else:
+            return rendered
+
 
 class ProfileRenderer(SecretRenderer):
     @property
diff --git a/core/dbt/context/base.py b/core/dbt/context/base.py
--- a/core/dbt/context/base.py
+++ b/core/dbt/context/base.py
@@ -26,38 +26,7 @@
 import re
 import itertools
 
-# Contexts in dbt Core
-# Contexts are used for Jinja rendering. They include context methods,
-# executable macros, and various settings that are available in Jinja.
-#
-# Different contexts are used in different places because we allow access
-# to different methods and data in different places. Executable SQL, for
-# example, includes the available macros and the model, while Jinja in
-# yaml files is more limited.
-#
-# The context that is passed to Jinja is always in a dictionary format,
-# not an actual class, so a 'to_dict()' is executed on a context class
-# before it is used for rendering.
-#
-# Each context has a generate_<name>_context function to create the context.
-# ProviderContext subclasses have different generate functions for
-# parsing and for execution.
-#
-# Context class hierarchy
-#
-#   BaseContext -- core/dbt/context/base.py
-#     SecretContext -- core/dbt/context/secret.py
-#     TargetContext -- core/dbt/context/target.py
-#       ConfiguredContext -- core/dbt/context/configured.py
-#         SchemaYamlContext -- core/dbt/context/configured.py
-#           DocsRuntimeContext -- core/dbt/context/configured.py
-#         MacroResolvingContext -- core/dbt/context/configured.py
-#         ManifestContext -- core/dbt/context/manifest.py
-#           QueryHeaderContext -- core/dbt/context/manifest.py
-#           ProviderContext -- core/dbt/context/provider.py
-#             MacroContext -- core/dbt/context/provider.py
-#             ModelContext -- core/dbt/context/provider.py
-#             TestContext -- core/dbt/context/provider.py
+# See the `contexts` module README for more information on how contexts work
 
 
 def get_pytz_module_context() -> Dict[str, Any]:
diff --git a/core/dbt/context/secret.py b/core/dbt/context/secret.py
--- a/core/dbt/context/secret.py
+++ b/core/dbt/context/secret.py
@@ -7,6 +7,9 @@
 from dbt.logger import SECRET_ENV_PREFIX
 
 
+SECRET_PLACEHOLDER = "$$$DBT_SECRET_START$$${}$$$DBT_SECRET_END$$$"
+
+
 class SecretContext(BaseContext):
     """This context is used in profiles.yml + packages.yml. It can render secret
     env vars that aren't usable elsewhere"""
@@ -18,21 +21,29 @@ def env_var(self, var: str, default: Optional[str] = None) -> str:
 
         If the default is None, raise an exception for an undefined variable.
 
-        In this context *only*, env_var will return the actual values of
-        env vars prefixed with DBT_ENV_SECRET_
+        In this context *only*, env_var will accept env vars prefixed with DBT_ENV_SECRET_.
+        It will return the name of the secret env var, wrapped in 'start' and 'end' identifiers.
+        The actual value will be subbed in later in SecretRenderer.render_value()
         """
         return_value = None
-        if var in os.environ:
+
+        # if this is a 'secret' env var, just return the name of the env var
+        # instead of rendering the actual value here, to avoid any risk of
+        # Jinja manipulation. it will be subbed out later, in SecretRenderer.render_value
+        if var in os.environ and var.startswith(SECRET_ENV_PREFIX):
+            return SECRET_PLACEHOLDER.format(var)
+
+        elif var in os.environ:
             return_value = os.environ[var]
         elif default is not None:
             return_value = default
 
         if return_value is not None:
-            # do not save secret environment variables
+            # store env vars in the internal manifest to power partial parsing
+            # if it's a 'secret' env var, we shouldn't even get here
+            # but just to be safe — don't save secrets
             if not var.startswith(SECRET_ENV_PREFIX):
                 self.env_vars[var] = return_value
-
-            # return the value even if its a secret
             return return_value
         else:
             msg = f"Env var required but not provided: '{var}'"


Test patch: