Repository: dbt-labs/dbt-core

Issue:
Scrub secret env vars from targeted exceptions
In v1:
- secret env vars (prefixed `DBT_ENV_SECRET_`) are for use exclusively in `packages.yml` + `profiles.yml`
- We scrub those secret values from log messages produced by all events

However, exceptions still do not pass through the event logger, and so they may still raise/print unscrubbed information. This came up in a test case with `bad_package_spec()`.

- Temporary fix (#4507): Let's scrub all exceptions that are raised by `deps` for `git`-based packages, since this is the use case we know about
- Long-term fix: Exception messages should pass through the same centralized event logger


Discussion:

Resolving patch:
diff --git a/core/dbt/clients/git.py b/core/dbt/clients/git.py
--- a/core/dbt/clients/git.py
+++ b/core/dbt/clients/git.py
@@ -8,7 +8,10 @@
     GitProgressUpdatingExistingDependency, GitProgressPullingNewDependency,
     GitNothingToDo, GitProgressUpdatedCheckoutRange, GitProgressCheckedOutAt
 )
-import dbt.exceptions
+from dbt.exceptions import (
+    CommandResultError, RuntimeException, bad_package_spec, raise_git_cloning_error,
+    raise_git_cloning_problem
+)
 from packaging import version
 
 
@@ -22,9 +25,9 @@ def _raise_git_cloning_error(repo, revision, error):
     if 'usage: git' in stderr:
         stderr = stderr.split('\nusage: git')[0]
     if re.match("fatal: destination path '(.+)' already exists", stderr):
-        raise error
+        raise_git_cloning_error(error)
 
-    dbt.exceptions.bad_package_spec(repo, revision, stderr)
+    bad_package_spec(repo, revision, stderr)
 
 
 def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirectory=None):
@@ -53,7 +56,7 @@ def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirec
         clone_cmd.append(dirname)
     try:
         result = run_cmd(cwd, clone_cmd, env={'LC_ALL': 'C'})
-    except dbt.exceptions.CommandResultError as exc:
+    except CommandResultError as exc:
         _raise_git_cloning_error(repo, revision, exc)
 
     if subdirectory:
@@ -61,7 +64,7 @@ def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirec
         clone_cmd_subdir = ['git', 'sparse-checkout', 'set', subdirectory]
         try:
             run_cmd(cwd_subdir, clone_cmd_subdir)
-        except dbt.exceptions.CommandResultError as exc:
+        except CommandResultError as exc:
             _raise_git_cloning_error(repo, revision, exc)
 
     if remove_git_dir:
@@ -105,9 +108,9 @@ def checkout(cwd, repo, revision=None):
         revision = 'HEAD'
     try:
         return _checkout(cwd, repo, revision)
-    except dbt.exceptions.CommandResultError as exc:
+    except CommandResultError as exc:
         stderr = exc.stderr.decode('utf-8').strip()
-    dbt.exceptions.bad_package_spec(repo, revision, stderr)
+    bad_package_spec(repo, revision, stderr)
 
 
 def get_current_sha(cwd):
@@ -131,14 +134,11 @@ def clone_and_checkout(repo, cwd, dirname=None, remove_git_dir=False,
             remove_git_dir=remove_git_dir,
             subdirectory=subdirectory,
         )
-    except dbt.exceptions.CommandResultError as exc:
+    except CommandResultError as exc:
         err = exc.stderr.decode('utf-8')
         exists = re.match("fatal: destination path '(.+)' already exists", err)
         if not exists:
-            print(
-                '\nSomething went wrong while cloning {}'.format(repo) +
-                '\nCheck the debug logs for more information')
-            raise
+            raise_git_cloning_problem(repo)
 
     directory = None
     start_sha = None
@@ -148,7 +148,7 @@ def clone_and_checkout(repo, cwd, dirname=None, remove_git_dir=False,
     else:
         matches = re.match("Cloning into '(.+)'", err.decode('utf-8'))
         if matches is None:
-            raise dbt.exceptions.RuntimeException(
+            raise RuntimeException(
                 f'Error cloning {repo} - never saw "Cloning into ..." from git'
             )
         directory = matches.group(1)
diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py
--- a/core/dbt/exceptions.py
+++ b/core/dbt/exceptions.py
@@ -2,8 +2,7 @@
 import functools
 from typing import NoReturn, Optional, Mapping, Any
 
-from dbt.logger import get_secret_env
-from dbt.events.functions import fire_event
+from dbt.events.functions import fire_event, scrub_secrets, env_secrets
 from dbt.events.types import GeneralWarningMsg, GeneralWarningException
 from dbt.node_types import NodeType
 from dbt import flags
@@ -54,7 +53,7 @@ class RuntimeException(RuntimeError, Exception):
     def __init__(self, msg, node=None):
         self.stack = []
         self.node = node
-        self.msg = msg
+        self.msg = scrub_secrets(msg, env_secrets())
 
     def add_node(self, node=None):
         if node is not None and node is not self.node:
@@ -401,8 +400,6 @@ def __init__(self, cwd, cmd, message='Error running command'):
         super().__init__(message)
         self.cwd = cwd
         self.cmd = cmd
-        for secret in get_secret_env():
-            self.cmd = str(self.cmd).replace(secret, "*****")
         self.args = (cwd, cmd, message)
 
     def __str__(self):
@@ -466,7 +463,21 @@ def raise_database_error(msg, node=None) -> NoReturn:
 
 
 def raise_dependency_error(msg) -> NoReturn:
-    raise DependencyException(msg)
+    raise DependencyException(scrub_secrets(msg, env_secrets()))
+
+
+def raise_git_cloning_error(error: CommandResultError) -> NoReturn:
+    error.cmd = scrub_secrets(str(error.cmd), env_secrets())
+    raise error
+
+
+def raise_git_cloning_problem(repo) -> NoReturn:
+    repo = scrub_secrets(repo, env_secrets())
+    msg = '''\
+    Something went wrong while cloning {}
+    Check the debug logs for more information
+    '''
+    raise RuntimeException(msg.format(repo))
 
 
 def disallow_secret_env_var(env_var_name) -> NoReturn:
@@ -692,9 +703,9 @@ def missing_materialization(model, adapter_type):
 
 
 def bad_package_spec(repo, spec, error_message):
-    raise InternalException(
-        "Error checking out spec='{}' for repo {}\n{}".format(
-            spec, repo, error_message))
+    msg = "Error checking out spec='{}' for repo {}\n{}".format(spec, repo, error_message)
+
+    raise InternalException(scrub_secrets(msg, env_secrets()))
 
 
 def raise_cache_inconsistent(message):
@@ -999,7 +1010,7 @@ def raise_duplicate_alias(
 
 def warn_or_error(msg, node=None, log_fmt=None):
     if flags.WARN_ERROR:
-        raise_compiler_error(msg, node)
+        raise_compiler_error(scrub_secrets(msg, env_secrets()), node)
     else:
         fire_event(GeneralWarningMsg(msg=msg, log_fmt=log_fmt))
 


Test patch: