Repository: dbt-labs/dbt-core (Python)

Issue:
[CT-665] [Feature] Allow control over the outer CTE identifier generated in ephemeral materialization
### Is this your first time opening an issue?

- [X] I have read the [expectations for open source contributors](https://docs.getdbt.com/docs/contributing/oss-expectations)

### Describe the Feature

We use a dot notation to group and organize dbt model files with a naming pattern of `{group}.{model}` such as `ingest.source` and `dw.dimension` etc. With specially complex models this provides a level of organization and consistency that keeps files organized, unique and faster to identify and find.

We rely on config aliases to then properly name relations - otherwise adding schema would be an invalid fully qualified relation name like `schema.layer.table`.  This works for both **table** and **view** materialization, however causes the generated sql to be invalid when using the **ephemeral** materialization due to the outer CTE identifier being generated as `__dbt__cte__ingest.source`.

If model `alias` is configured it should be used everywhere as the identifier of the output instead of the file name - regardless of materialization used to give end user better control over generated SQL code and avoid errors or potential invalid code being generated when using an _unconventional_ naming of model files.

### Describe alternatives you've considered

An alternative might be some form of sanitization of CTE name during generation to avoid generating invalid CTE identifier - like stripping non alphanumeric characters or generating a non model name linked name - like an incremental `__dbt__cte__1` - as long as its a valid and unique CTE name to enable stacking CTEs.

### Who will this benefit?

This feature would introduce consistency in behavior compared to other materialization strategies (alias being used as table or view name), give uses control over naming of models vs files and allow end users to come up with creative naming convention and organization of model files without having to worry that switching materialization will cause a failure because of the model file name.


### Are you interested in contributing this feature?

Happy to contribute, looking for feedback and some pointers on this from the team

### Anything else?

_No response_



Fix (patch):
diff --git a/dbt/main.py b/dbt/main.py
index f7984a2a77e..a1af0bfb53c 100644
--- a/dbt/main.py
+++ b/dbt/main.py
@@ -20,8 +20,8 @@
 import dbt.tracking
 
 
-def is_opted_out():
-    profiles = project.read_profiles()
+def is_opted_out(profiles_dir):
+    profiles = project.read_profiles(profiles_dir)
 
     if profiles is None or profiles.get("config") is None:
         return False
@@ -34,12 +34,8 @@ def main(args=None):
     if args is None:
         args = sys.argv[1:]
 
-    if is_opted_out():
-        dbt.tracking.do_not_track()
-
     try:
-        handle(args)
-        dbt.tracking.flush()
+        return handle(args)
 
     except RuntimeError as e:
         print("Encountered an error:")
@@ -47,11 +43,95 @@ def main(args=None):
         sys.exit(1)
 
 def handle(args):
+    parsed = parse_args(args)
+
+    # this needs to happen after args are parsed so we can determine the correct profiles.yml file
+    if is_opted_out(parsed.profiles_dir):
+        dbt.tracking.do_not_track()
+
+    res = run_from_args(parsed)
+    dbt.tracking.flush()
+
+    return res
+
+def get_nearest_project_dir():
+    root_path = os.path.abspath(os.sep)
+    cwd = os.getcwd()
+
+    while cwd != root_path:
+        project_file = os.path.join(cwd, "dbt_project.yml")
+        if os.path.exists(project_file):
+            return cwd
+        cwd = os.path.dirname(cwd)
+
+    return None
+
+def run_from_args(parsed):
+    nearest_project_dir = get_nearest_project_dir()
+
+    if nearest_project_dir is None:
+        raise RuntimeError("dbt must be run from a project root directory with a dbt_project.yml file")
+
+    os.chdir(nearest_project_dir)
+
+    if parsed.which == 'init':
+        # bypass looking for a project file if we're running `dbt init`
+        task = parsed.cls(args=parsed)
+    else:
+        task, proj = invoke_dbt(parsed)
+
+    dbt.tracking.track_invocation_start(project=proj, args=parsed)
+    try:
+        return task.run()
+        dbt.tracking.track_invocation_end(project=proj, args=parsed, result_type="ok", result=None)
+    except Exception as e:
+        dbt.tracking.track_invocation_end(project=proj, args=parsed, result_type="error", result=str(e))
+        raise
+
+def invoke_dbt(parsed):
+    task = None
+    proj = None
+
+    try:
+        proj = project.read_project('dbt_project.yml', parsed.profiles_dir, validate=False, profile_to_load=parsed.profile)
+        proj.validate()
+    except project.DbtProjectError as e:
+        print("Encountered an error while reading the project:")
+        print("  ERROR {}".format(str(e)))
+        print("Did you set the correct --profile? Using: {}".format(parsed.profile))
+        all_profiles = project.read_profiles(parsed.profiles_dir).keys()
+        profiles_string = "\n".join([" - " + key for key in all_profiles])
+        print("Valid profiles:\n{}".format(profiles_string))
+        dbt.tracking.track_invalid_invocation(project=proj, args=parsed, result_type="invalid_profile", result=str(e))
+        return None
+
+    if parsed.target is not None:
+        targets = proj.cfg.get('outputs', {}).keys()
+        if parsed.target in targets:
+            proj.cfg['target'] = parsed.target
+        else:
+            print("Encountered an error while reading the project:")
+            print("  ERROR Specified target {} is not a valid option for profile {}".format(parsed.target, proj.profile_to_load))
+            print("Valid targets are: {}".format(targets))
+            dbt.tracking.track_invalid_invocation(project=proj, args=parsed, result_type="invalid_target", result="target not found")
+            return None
+
+    log_dir = proj.get('log-path', 'logs')
+    logger = getLogger(log_dir, __name__)
+
+    logger.info("running dbt with arguments %s", parsed)
+
+    task = parsed.cls(args=parsed, project=proj)
+
+    return task, proj
+
+def parse_args(args):
     p = argparse.ArgumentParser(prog='dbt: data build tool', formatter_class=argparse.RawTextHelpFormatter)
     p.add_argument('--version', action='version', version=dbt.version.get_version_information(), help="Show version information")
     subs = p.add_subparsers()
 
     base_subparser = argparse.ArgumentParser(add_help=False)
+    base_subparser.add_argument('--profiles-dir', default=project.default_profiles_dir, type=str, help='Which dir to look in for the profiles.yml file. Default = {}'.format(project.default_profiles_dir))
     base_subparser.add_argument('--profile', required=False, type=str, help='Which profile to load (overrides profile setting in dbt_project.yml file)')
     base_subparser.add_argument('--target', default=None, type=str, help='Which target to load for the given profile')
 
@@ -95,53 +175,4 @@ def handle(args):
     if len(args) == 0: return p.print_help()
 
     parsed = p.parse_args(args)
-
-    task = None
-    proj = None
-
-    if parsed.which == 'init':
-        # bypass looking for a project file if we're running `dbt init`
-        task = parsed.cls(args=parsed)
-
-    elif os.path.isfile('dbt_project.yml'):
-        try:
-            proj = project.read_project('dbt_project.yml', validate=False, profile_to_load=parsed.profile)
-            proj.validate()
-        except project.DbtProjectError as e:
-            print("Encountered an error while reading the project:")
-            print("  ERROR {}".format(str(e)))
-            print("Did you set the correct --profile? Using: {}".format(parsed.profile))
-            all_profiles = project.read_profiles().keys()
-            profiles_string = "\n".join([" - " + key for key in all_profiles])
-            print("Valid profiles:\n{}".format(profiles_string))
-            dbt.tracking.track_invalid_invocation(project=proj, args=parsed, result_type="invalid_profile", result=str(e))
-            return None
-
-        if parsed.target is not None:
-            targets = proj.cfg.get('outputs', {}).keys()
-            if parsed.target in targets:
-                proj.cfg['target'] = parsed.target
-            else:
-                print("Encountered an error while reading the project:")
-                print("  ERROR Specified target {} is not a valid option for profile {}".format(parsed.target, proj.profile_to_load))
-                print("Valid targets are: {}".format(targets))
-                dbt.tracking.track_invalid_invocation(project=proj, args=parsed, result_type="invalid_target", result="target not found")
-                return None
-
-        log_dir = proj.get('log-path', 'logs')
-        logger = getLogger(log_dir, __name__)
-
-        logger.info("running dbt with arguments %s", parsed)
-
-        task = parsed.cls(args=parsed, project=proj)
-
-    else:
-        raise RuntimeError("dbt must be run from a project root directory with a dbt_project.yml file")
-
-    dbt.tracking.track_invocation_start(project=proj, args=parsed)
-    try:
-        task.run()
-        dbt.tracking.track_invocation_end(project=proj, args=parsed, result_type="ok", result=None)
-    except Exception as e:
-        dbt.tracking.track_invocation_end(project=proj, args=parsed, result_type="error", result=str(e))
-        raise
+    return parsed
diff --git a/dbt/project.py b/dbt/project.py
index 5b34480340d..b19e8d7a770 100644
--- a/dbt/project.py
+++ b/dbt/project.py
@@ -23,6 +23,8 @@
 
 default_profiles = {}
 
+default_profiles_dir = os.path.join(os.path.expanduser('~'), '.dbt')
+
 class DbtProjectError(Exception):
     def __init__(self, message, project):
         self.project = project
@@ -30,11 +32,12 @@ def __init__(self, message, project):
 
 class Project(object):
 
-    def __init__(self, cfg, profiles, profile_to_load=None):
+    def __init__(self, cfg, profiles, profiles_dir, profile_to_load=None):
         self.cfg = default_project_cfg.copy()
         self.cfg.update(cfg)
         self.profiles = default_profiles.copy()
         self.profiles.update(profiles)
+        self.profiles_dir = profiles_dir
         self.profile_to_load = profile_to_load
 
         # load profile from dbt_config.yml if cli arg isn't supplied
@@ -85,12 +88,6 @@ def context(self):
         filtered_target.pop('pass', None)
         return {'env': filtered_target}
 
-    def with_profiles(self, profile):
-        return Project(
-            copy.deepcopy(self.cfg),
-            copy.deepcopy(self.profiles),
-            profile)
-
     def validate(self):
         self.handle_deprecations()
 
@@ -117,10 +114,13 @@ def hashed_name(self):
         return hashlib.md5(project_name.encode('utf-8')).hexdigest()
 
 
-def read_profiles():
+def read_profiles(profiles_dir=None):
+    if profiles_dir is None:
+        profiles_dir = default_profiles_dir
+
     profiles = {}
     paths = [
-        os.path.join(os.path.expanduser('~'), '.dbt/profiles.yml')
+        os.path.join(profiles_dir, 'profiles.yml')
     ]
     for path in paths:
         if os.path.isfile(path):
@@ -131,12 +131,15 @@ def read_profiles():
 
     return profiles
 
-def read_project(filename, validate=True, profile_to_load=None):
+def read_project(filename, profiles_dir=None, validate=True, profile_to_load=None):
+    if profiles_dir is None:
+        profiles_dir = default_profiles_dir
+
     with open(filename, 'r') as f:
         project_cfg = yaml.safe_load(f)
         project_cfg['project-root'] = os.path.dirname(os.path.abspath(filename))
-        profiles = read_profiles()
-        proj = Project(project_cfg, profiles, profile_to_load)
+        profiles = read_profiles(profiles_dir)
+        proj = Project(project_cfg, profiles, profiles_dir, profile_to_load)
 
         if validate:
             proj.validate()
diff --git a/dbt/task/deps.py b/dbt/task/deps.py
index 4b51fd6dc37..1556213f3dd 100644
--- a/dbt/task/deps.py
+++ b/dbt/task/deps.py
@@ -103,7 +103,7 @@ def __pull_deps_recursive(self, repos, processed_repos = None, i=0):
                     dep_project = project.read_project(
                         os.path.join(self.project['modules-path'],
                                      dep_folder,
-                                     'dbt_project.yml'), profile_to_load=self.project.profile_to_load
+                                     'dbt_project.yml'), self.project.profiles_dir, profile_to_load=self.project.profile_to_load
                     )
                     processed_repos.add(dep_folder)
                     self.__pull_deps_recursive(dep_project['repositories'], processed_repos, i+1)
diff --git a/dbt/utils.py b/dbt/utils.py
index 1949dd90fde..1d17ffdca4c 100644
--- a/dbt/utils.py
+++ b/dbt/utils.py
@@ -86,7 +86,7 @@ def dependency_projects(project):
         full_obj = os.path.join(project['modules-path'], obj)
         if os.path.isdir(full_obj):
             try:
-                yield dbt.project.read_project(os.path.join(full_obj, 'dbt_project.yml'), profile_to_load=project.profile_to_load)
+                yield dbt.project.read_project(os.path.join(full_obj, 'dbt_project.yml'), project.profiles_dir, profile_to_load=project.profile_to_load)
             except dbt.project.DbtProjectError as e:
                 print("Error reading dependency project at {}".format(full_obj))
                 print(str(e))