Repository: dbt-labs/dbt-core

Issue:
[CT-25] [Bug] dbt deps should not require a profile
### Is there an existing issue for this?

- [X] I have searched the existing issues

### Current Behavior

We have defined a profile like that :
```
    prod:
      type: redshift
      threads: "{{ env_var('DBT_THREADS', 1)| int }}"
      host: "{{ env_var('REDSHIFT_HOST') }}"
      port: 5439
      user: "{{ env_var('REDSHIFT_USER') }}"
      pass: "{{ env_var('REDSHIFT_PASSWORD') }}"
      dbname: "{{ env_var('REDSHIFT_DATABASE') }}"
      schema: "bi"
```
If one of the env is not defined  ( ex : REDSHIFT_DATABASE) we now have the following error message while running `dbt deps`
```
13:26:05  Encountered an error:
Parsing Error
  Env var required but not provided: 'REDSHIFT_DATABASE'
```
If it is defined with a dummy data, it works properly. ( So it does not check connection)

### Expected Behavior

`dbt deps`should not return any issue when a env variable is not defined.

### Steps To Reproduce

1. Define a profile with env variable 
2. Do not define the given env variable 
3. run `dbt deps`

### Relevant log output

_No response_

### Environment

```markdown
- OS:container python:3.9.6-slim-buster
- Python: 3.9.6
- dbt:1.0.0
```


### What database are you using dbt with?

redshift

### Additional Context

_No response_


Discussion:
When you say "we now have the following error message", do you mean that this used to work? In what version? A quick glance at the code shows that throwing an error for a missing env_var is not new to 1.0.0.

I think the ability to set a default for the env_var, like you do for DBT_THREADS, is designed for this case. You could do env_var('REDSHIFT_DATABASE, ''). 
Hello, it was working perfectly for the version before the 1.0.0. 
We are using this logic for in the version 0.21.1 (and before also)
This bug is seen when we try to move to version 1.0.0
This bug is due to changing the env_var exception to a parsing exception, so that the exception is not caught in _get_rendered_profile in core/dbt/config/runtime.py. The error that is issued in 0.21 is not correct however, and this is not a good way to handle suppressing profile errors because the final error reported is not correct.

This should be fixed, however, since we should not need to load a profile for the dbt deps command.
We should also check `dbt clean` and `dbt debug`

Resolving patch:
diff --git a/core/dbt/config/runtime.py b/core/dbt/config/runtime.py
--- a/core/dbt/config/runtime.py
+++ b/core/dbt/config/runtime.py
@@ -1,7 +1,7 @@
 import itertools
 import os
 from copy import deepcopy
-from dataclasses import dataclass, fields
+from dataclasses import dataclass
 from pathlib import Path
 from typing import (
     Dict, Any, Optional, Mapping, Iterator, Iterable, Tuple, List, MutableSet,
@@ -13,20 +13,17 @@
 from .renderer import DbtProjectYamlRenderer, ProfileRenderer
 from .utils import parse_cli_vars
 from dbt import flags
-from dbt import tracking
 from dbt.adapters.factory import get_relation_class_by_name, get_include_paths
 from dbt.helper_types import FQNPath, PathSet
+from dbt.config.profile import read_user_config
 from dbt.contracts.connection import AdapterRequiredConfig, Credentials
 from dbt.contracts.graph.manifest import ManifestMetadata
 from dbt.contracts.relation import ComponentName
-from dbt.events.types import ProfileLoadError, ProfileNotFound
-from dbt.events.functions import fire_event
 from dbt.ui import warning_tag
 
 from dbt.contracts.project import Configuration, UserConfig
 from dbt.exceptions import (
     RuntimeException,
-    DbtProfileError,
     DbtProjectError,
     validator_error_message,
     warn_or_error,
@@ -191,6 +188,7 @@ def _get_rendered_profile(
         profile_renderer: ProfileRenderer,
         profile_name: Optional[str],
     ) -> Profile:
+
         return Profile.render_from_args(
             args, profile_renderer, profile_name
         )
@@ -412,21 +410,12 @@ def _connection_keys(self):
         return ()
 
 
-class UnsetConfig(UserConfig):
-    def __getattribute__(self, name):
-        if name in {f.name for f in fields(UserConfig)}:
-            raise AttributeError(
-                f"'UnsetConfig' object has no attribute {name}"
-            )
-
-    def __post_serialize__(self, dct):
-        return {}
-
-
+# This is used by UnsetProfileConfig, for commands which do
+# not require a profile, i.e. dbt deps and clean
 class UnsetProfile(Profile):
     def __init__(self):
         self.credentials = UnsetCredentials()
-        self.user_config = UnsetConfig()
+        self.user_config = UserConfig()  # This will be read in _get_rendered_profile
         self.profile_name = ''
         self.target_name = ''
         self.threads = -1
@@ -443,6 +432,8 @@ def __getattribute__(self, name):
         return Profile.__getattribute__(self, name)
 
 
+# This class is used by the dbt deps and clean commands, because they don't
+# require a functioning profile.
 @dataclass
 class UnsetProfileConfig(RuntimeConfig):
     """This class acts a lot _like_ a RuntimeConfig, except if your profile is
@@ -525,7 +516,7 @@ def from_parts(
             profile_env_vars=profile.profile_env_vars,
             profile_name='',
             target_name='',
-            user_config=UnsetConfig(),
+            user_config=UserConfig(),
             threads=getattr(args, 'threads', 1),
             credentials=UnsetCredentials(),
             args=args,
@@ -540,21 +531,12 @@ def _get_rendered_profile(
         profile_renderer: ProfileRenderer,
         profile_name: Optional[str],
     ) -> Profile:
-        try:
-            profile = Profile.render_from_args(
-                args, profile_renderer, profile_name
-            )
-        except (DbtProjectError, DbtProfileError) as exc:
-            selected_profile_name = Profile.pick_profile_name(
-                args_profile_name=getattr(args, 'profile', None),
-                project_profile_name=profile_name
-            )
-            fire_event(ProfileLoadError(exc=exc))
-            fire_event(ProfileNotFound(profile_name=selected_profile_name))
-            # return the poisoned form
-            profile = UnsetProfile()
-            # disable anonymous usage statistics
-            tracking.disable_tracking()
+
+        profile = UnsetProfile()
+        # The profile (for warehouse connection) is not needed, but we want
+        # to get the UserConfig, which is also in profiles.yml
+        user_config = read_user_config(flags.PROFILES_DIR)
+        profile.user_config = user_config
         return profile
 
     @classmethod
@@ -569,9 +551,6 @@ def from_args(cls: Type[RuntimeConfig], args: Any) -> 'RuntimeConfig':
         :raises ValidationException: If the cli variables are invalid.
         """
         project, profile = cls.collect_parts(args)
-        if not isinstance(profile, UnsetProfile):
-            # if it's a real profile, return a real config
-            cls = RuntimeConfig
 
         return cls.from_parts(
             project=project,
diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py
--- a/core/dbt/events/types.py
+++ b/core/dbt/events/types.py
@@ -795,24 +795,6 @@ def message(self) -> str:
         return f"target not specified in profile '{self.profile_name}', using '{self.target_name}'"
 
 
-@dataclass
-class ProfileLoadError(ShowException, DebugLevel, Cli, File):
-    exc: Exception
-    code: str = "A006"
-
-    def message(self) -> str:
-        return f"Profile not loaded due to error: {self.exc}"
-
-
-@dataclass
-class ProfileNotFound(InfoLevel, Cli, File):
-    profile_name: Optional[str]
-    code: str = "A007"
-
-    def message(self) -> str:
-        return f'No profile "{self.profile_name}" found, continuing with no target'
-
-
 @dataclass
 class InvalidVarsYAML(ErrorLevel, Cli, File):
     code: str = "A008"
@@ -2528,8 +2510,6 @@ def message(self) -> str:
     TimingInfoCollected()
     MergedFromState(nbr_merged=0, sample=[])
     MissingProfileTarget(profile_name='', target_name='')
-    ProfileLoadError(exc=Exception(''))
-    ProfileNotFound(profile_name='')
     InvalidVarsYAML()
     GenericTestFileParse(path='')
     MacroFileParse(path='')


Test patch:
diff --git a/test/integration/006_simple_dependency_test/test_simple_dependency.py b/test/integration/006_simple_dependency_test/test_simple_dependency.py
--- a/test/integration/006_simple_dependency_test/test_simple_dependency.py
+++ b/test/integration/006_simple_dependency_test/test_simple_dependency.py
@@ -246,3 +246,48 @@ def run_clean(self):
         with tempfile.TemporaryDirectory() as tmpdir:
             result = self.run_dbt(["clean", "--profiles-dir", tmpdir])
         return result
+
+class TestSimpleDependencyBadProfile(DBTIntegrationTest):
+
+    @property
+    def schema(self):
+        return "simple_dependency_006"
+
+    @property
+    def models(self):
+        return "models"
+
+    def postgres_profile(self):
+        # Need to set the environment variable here initially because
+        # the unittest setup does a load_config.
+        os.environ['PROFILE_TEST_HOST'] = self.database_host
+        return {
+            'config': {
+                'send_anonymous_usage_stats': False
+            },
+            'test': {
+                'outputs': {
+                    'default2': {
+                        'type': 'postgres',
+                        'threads': 4,
+                        'host': "{{ env_var('PROFILE_TEST_HOST') }}",
+                        'port': 5432,
+                        'user': 'root',
+                        'pass': 'password',
+                        'dbname': 'dbt',
+                        'schema': self.unique_schema()
+                    },
+                },
+                'target': 'default2'
+            }
+        }
+
+    @use_profile('postgres')
+    def test_postgres_deps_bad_profile(self):
+        del os.environ['PROFILE_TEST_HOST']
+        self.run_dbt(["deps"])
+
+    @use_profile('postgres')
+    def test_postgres_clean_bad_profile(self):
+        del os.environ['PROFILE_TEST_HOST']
+        self.run_dbt(["clean"])
diff --git a/test/unit/test_events.py b/test/unit/test_events.py
--- a/test/unit/test_events.py
+++ b/test/unit/test_events.py
@@ -233,8 +233,6 @@ def MockNode():
     TimingInfoCollected(),
     MergedFromState(nbr_merged=0, sample=[]),
     MissingProfileTarget(profile_name='', target_name=''),
-    ProfileLoadError(exc=Exception('')),
-    ProfileNotFound(profile_name=''),
     InvalidVarsYAML(),
     GenericTestFileParse(path=''),
     MacroFileParse(path=''),