Repository: dbt-labs/dbt-core

Issue:
CTE no longer interpolated into target compiled query for ephemeral model
### Describe the bug
Recently upgraded to 0.19. For some ephemeral models, I am no longer able to run the compiled query as-is. Sample error: `Invalid operation: relation "__dbt__cte__xyz" does not exist;`.

### Steps To Reproduce
Issue occurs when an ephemeral model references another ephemeral model. The compiled query contains `select * from __dbt__CTE__xyz`, but the CTE query is missing.

### Expected behavior
Be able to run all compiled queries as-is.

### Screenshots and log output
N/A

### System information
**Which database are you using dbt with?**
- [ ] postgres
- [x] redshift
- [ ] bigquery
- [ ] snowflake
- [ ] other (specify: ____________)


**The output of `dbt --version`:**
```
$ dbt --version
installed version: 0.19.0
   latest version: 0.19.0

Up to date!

Plugins:
  - bigquery: 0.19.0
  - postgres: 0.19.0
  - redshift: 0.19.0
  - snowflake: 0.19.0
```

**The operating system you're using:** Windows 10 Pro

**The output of `python --version`:** Python 3.8.7

### Additional context
N/A



Discussion:

Resolving patch:
diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py
--- a/core/dbt/compilation.py
+++ b/core/dbt/compilation.py
@@ -248,19 +248,19 @@ def _inject_ctes_into_sql(self, sql: str, ctes: List[InjectedCTE]) -> str:
     def _get_dbt_test_name(self) -> str:
         return 'dbt__CTE__INTERNAL_test'
 
-    # This method is called by the 'compile_node' method. Starting
-    # from the node that it is passed in, it will recursively call
-    # itself using the 'extra_ctes'.  The 'ephemeral' models do
-    # not produce SQL that is executed directly, instead they
-    # are rolled up into the models that refer to them by
-    # inserting CTEs into the SQL.
     def _recursively_prepend_ctes(
         self,
         model: NonSourceCompiledNode,
         manifest: Manifest,
         extra_context: Optional[Dict[str, Any]],
     ) -> Tuple[NonSourceCompiledNode, List[InjectedCTE]]:
-
+        """This method is called by the 'compile_node' method. Starting
+        from the node that it is passed in, it will recursively call
+        itself using the 'extra_ctes'.  The 'ephemeral' models do
+        not produce SQL that is executed directly, instead they
+        are rolled up into the models that refer to them by
+        inserting CTEs into the SQL.
+        """
         if model.compiled_sql is None:
             raise RuntimeException(
                 'Cannot inject ctes into an unparsed node', model
@@ -324,19 +324,19 @@ def _recursively_prepend_ctes(
                 _extend_prepended_ctes(prepended_ctes, new_prepended_ctes)
 
                 new_cte_name = self.add_ephemeral_prefix(cte_model.name)
-                sql = f' {new_cte_name} as (\n{cte_model.compiled_sql}\n)'
+                rendered_sql = (
+                    cte_model._pre_injected_sql or cte_model.compiled_sql
+                )
+                sql = f' {new_cte_name} as (\n{rendered_sql}\n)'
 
             _add_prepended_cte(prepended_ctes, InjectedCTE(id=cte.id, sql=sql))
 
-        # We don't save injected_sql into compiled sql for ephemeral models
-        # because it will cause problems with processing of subsequent models.
-        # Ephemeral models do not produce executable SQL of their own.
-        if not model.is_ephemeral_model:
-            injected_sql = self._inject_ctes_into_sql(
-                model.compiled_sql,
-                prepended_ctes,
-            )
-            model.compiled_sql = injected_sql
+        injected_sql = self._inject_ctes_into_sql(
+            model.compiled_sql,
+            prepended_ctes,
+        )
+        model._pre_injected_sql = model.compiled_sql
+        model.compiled_sql = injected_sql
         model.extra_ctes_injected = True
         model.extra_ctes = prepended_ctes
         model.validate(model.to_dict(omit_none=True))
@@ -487,11 +487,6 @@ def _write_node(self, node: NonSourceCompiledNode) -> ManifestNode:
             )
         return node
 
-    # This is the main entry point into this code. It's called by
-    # CompileRunner.compile, GenericRPCRunner.compile, and
-    # RunTask.get_hook_sql. It calls '_compile_node' to convert
-    # the node into a compiled node, and then calls the
-    # recursive method to "prepend" the ctes.
     def compile_node(
         self,
         node: ManifestNode,
@@ -499,6 +494,12 @@ def compile_node(
         extra_context: Optional[Dict[str, Any]] = None,
         write: bool = True,
     ) -> NonSourceCompiledNode:
+        """This is the main entry point into this code. It's called by
+        CompileRunner.compile, GenericRPCRunner.compile, and
+        RunTask.get_hook_sql. It calls '_compile_node' to convert
+        the node into a compiled node, and then calls the
+        recursive method to "prepend" the ctes.
+        """
         node = self._compile_node(node, manifest, extra_context)
 
         node, _ = self._recursively_prepend_ctes(
diff --git a/core/dbt/contracts/graph/compiled.py b/core/dbt/contracts/graph/compiled.py
--- a/core/dbt/contracts/graph/compiled.py
+++ b/core/dbt/contracts/graph/compiled.py
@@ -43,6 +43,7 @@ class CompiledNode(ParsedNode, CompiledNodeMixin):
     extra_ctes_injected: bool = False
     extra_ctes: List[InjectedCTE] = field(default_factory=list)
     relation_name: Optional[str] = None
+    _pre_injected_sql: Optional[str] = None
 
     def set_cte(self, cte_id: str, sql: str):
         """This is the equivalent of what self.extra_ctes[cte_id] = sql would
@@ -55,6 +56,12 @@ def set_cte(self, cte_id: str, sql: str):
         else:
             self.extra_ctes.append(InjectedCTE(id=cte_id, sql=sql))
 
+    def __post_serialize__(self, dct):
+        dct = super().__post_serialize__(dct)
+        if '_pre_injected_sql' in dct:
+            del dct['_pre_injected_sql']
+        return dct
+
 
 @dataclass
 class CompiledAnalysisNode(CompiledNode):


Test patch:
diff --git a/test/unit/test_compiler.py b/test/unit/test_compiler.py
--- a/test/unit/test_compiler.py
+++ b/test/unit/test_compiler.py
@@ -62,22 +62,12 @@ def setUp(self):
 
         self.config = config_from_parts_or_dicts(project_cfg, profile_cfg)
 
-        self._generate_runtime_model_patch = patch.object(dbt.compilation, 'generate_runtime_model')
+        self._generate_runtime_model_patch = patch.object(
+            dbt.compilation, 'generate_runtime_model')
         self.mock_generate_runtime_model = self._generate_runtime_model_patch.start()
 
         inject_adapter(Plugin.adapter(self.config), Plugin)
 
-        # self.mock_adapter = PostgresAdapter MagicMock(type=MagicMock(return_value='postgres'))
-        # self.mock_adapter.Relation =
-        # self.mock_adapter.get_compiler.return_value = dbt.compilation.Compiler
-        # self.mock_plugin = MagicMock(
-        #     adapter=MagicMock(
-        #         credentials=MagicMock(return_value='postgres')
-        #     )
-        # )
-        # inject_adapter(self.mock_adapter, self.mock_plugin)
-        # so we can make an adapter
-
         def mock_generate_runtime_model_context(model, config, manifest):
             def ref(name):
                 result = f'__dbt__CTE__{name}'
@@ -98,7 +88,7 @@ def test__prepend_ctes__already_has_cte(self):
         manifest = Manifest(
             macros={},
             nodes={
-                'model.root.view': CompiledModelNode(
+                'model.root.view': ParsedModelNode(
                     name='view',
                     database='dbt',
                     schema='analytics',
@@ -108,23 +98,13 @@ def test__prepend_ctes__already_has_cte(self):
                     fqn=['root', 'view'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(nodes=['model.root.ephemeral']),
                     config=self.model_config,
-                    tags=[],
                     path='view.sql',
                     original_file_path='view.sql',
-                    raw_sql='select * from {{ref("ephemeral")}}',
-                    compiled=True,
-                    extra_ctes_injected=False,
-                    extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')],
-                    compiled_sql=(
-                        'with cte as (select * from something_else) '
-                        'select * from __dbt__CTE__ephemeral'),
+                    raw_sql='with cte as (select * from something_else) select * from {{ref("ephemeral")}}',
                     checksum=FileHash.from_contents(''),
                 ),
-                'model.root.ephemeral': CompiledModelNode(
+                'model.root.ephemeral': ParsedModelNode(
                     name='ephemeral',
                     database='dbt',
                     schema='analytics',
@@ -134,18 +114,10 @@ def test__prepend_ctes__already_has_cte(self):
                     fqn=['root', 'ephemeral'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(),
                     config=ephemeral_config,
-                    tags=[],
                     path='ephemeral.sql',
                     original_file_path='ephemeral.sql',
                     raw_sql='select * from source_table',
-                    compiled=True,
-                    compiled_sql='select * from source_table',
-                    extra_ctes_injected=False,
-                    extra_ctes=[],
                     checksum=FileHash.from_contents(''),
                 ),
             },
@@ -158,11 +130,8 @@ def test__prepend_ctes__already_has_cte(self):
         )
 
         compiler = dbt.compilation.Compiler(self.config)
-        result, _ = compiler._recursively_prepend_ctes(
-            manifest.nodes['model.root.view'],
-            manifest,
-            {}
-        )
+        result = compiler.compile_node(
+            manifest.nodes['model.root.view'], manifest, write=False)
 
         self.assertEqual(result, manifest.nodes['model.root.view'])
         self.assertEqual(result.extra_ctes_injected, True)
@@ -172,16 +141,15 @@ def test__prepend_ctes__already_has_cte(self):
              'select * from source_table'
              '), cte as (select * from something_else) '
              'select * from __dbt__CTE__ephemeral'))
-
         self.assertEqual(
             manifest.nodes['model.root.ephemeral'].extra_ctes_injected,
-            False)
+            True)
 
     def test__prepend_ctes__no_ctes(self):
         manifest = Manifest(
             macros={},
             nodes={
-                'model.root.view': CompiledModelNode(
+                'model.root.view': ParsedModelNode(
                     name='view',
                     database='dbt',
                     schema='analytics',
@@ -191,23 +159,14 @@ def test__prepend_ctes__no_ctes(self):
                     fqn=['root', 'view'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(),
                     config=self.model_config,
-                    tags=[],
                     path='view.sql',
                     original_file_path='view.sql',
                     raw_sql=('with cte as (select * from something_else) '
                              'select * from source_table'),
-                    compiled=True,
-                    extra_ctes_injected=False,
-                    extra_ctes=[],
-                    compiled_sql=('with cte as (select * from something_else) '
-                                  'select * from source_table'),
                     checksum=FileHash.from_contents(''),
                 ),
-                'model.root.view_no_cte': CompiledModelNode(
+                'model.root.view_no_cte': ParsedModelNode(
                     name='view_no_cte',
                     database='dbt',
                     schema='analytics',
@@ -217,18 +176,10 @@ def test__prepend_ctes__no_ctes(self):
                     fqn=['root', 'view_no_cte'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(),
                     config=self.model_config,
-                    tags=[],
                     path='view.sql',
                     original_file_path='view.sql',
                     raw_sql='select * from source_table',
-                    compiled=True,
-                    extra_ctes_injected=False,
-                    extra_ctes=[],
-                    compiled_sql=('select * from source_table'),
                     checksum=FileHash.from_contents(''),
                 ),
             },
@@ -241,11 +192,8 @@ def test__prepend_ctes__no_ctes(self):
         )
 
         compiler = dbt.compilation.Compiler(self.config)
-        result, _ = compiler._recursively_prepend_ctes(
-            manifest.nodes['model.root.view'],
-            manifest,
-            {}
-        )
+        result = compiler.compile_node(
+            manifest.nodes['model.root.view'], manifest, write=False)
 
         self.assertEqual(
             result,
@@ -253,13 +201,13 @@ def test__prepend_ctes__no_ctes(self):
         self.assertTrue(result.extra_ctes_injected)
         self.assertEqualIgnoreWhitespace(
             result.compiled_sql,
-            manifest.nodes.get('model.root.view').compiled_sql)
+            ('with cte as (select * from something_else) '
+             'select * from source_table')
+        )
 
         compiler = dbt.compilation.Compiler(self.config)
-        result, _ = compiler._recursively_prepend_ctes(
-            manifest.nodes.get('model.root.view_no_cte'),
-            manifest,
-            {})
+        result = compiler.compile_node(
+            manifest.nodes['model.root.view_no_cte'], manifest, write=False)
 
         self.assertEqual(
             result,
@@ -267,7 +215,8 @@ def test__prepend_ctes__no_ctes(self):
         self.assertTrue(result.extra_ctes_injected)
         self.assertEqualIgnoreWhitespace(
             result.compiled_sql,
-            manifest.nodes.get('model.root.view_no_cte').compiled_sql)
+            'select * from source_table'
+        )
 
     def test__prepend_ctes(self):
         ephemeral_config = self.model_config.replace(materialized='ephemeral')
@@ -275,7 +224,7 @@ def test__prepend_ctes(self):
         manifest = Manifest(
             macros={},
             nodes={
-                'model.root.view': CompiledModelNode(
+                'model.root.view': ParsedModelNode(
                     name='view',
                     database='dbt',
                     schema='analytics',
@@ -285,21 +234,13 @@ def test__prepend_ctes(self):
                     fqn=['root', 'view'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(nodes=['model.root.ephemeral']),
                     config=self.model_config,
-                    tags=[],
                     path='view.sql',
                     original_file_path='view.sql',
                     raw_sql='select * from {{ref("ephemeral")}}',
-                    compiled=True,
-                    extra_ctes_injected=False,
-                    extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')],
-                    compiled_sql='select * from __dbt__CTE__ephemeral',
                     checksum=FileHash.from_contents(''),
                 ),
-                'model.root.ephemeral': CompiledModelNode(
+                'model.root.ephemeral': ParsedModelNode(
                     name='ephemeral',
                     database='dbt',
                     schema='analytics',
@@ -309,18 +250,10 @@ def test__prepend_ctes(self):
                     fqn=['root', 'ephemeral'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(),
                     config=ephemeral_config,
-                    tags=[],
                     path='ephemeral.sql',
                     original_file_path='ephemeral.sql',
                     raw_sql='select * from source_table',
-                    compiled=True,
-                    extra_ctes_injected=False,
-                    extra_ctes=[],
-                    compiled_sql='select * from source_table',
                     checksum=FileHash.from_contents(''),
                 ),
             },
@@ -333,15 +266,11 @@ def test__prepend_ctes(self):
         )
 
         compiler = dbt.compilation.Compiler(self.config)
-        result, _ = compiler._recursively_prepend_ctes(
+        result = compiler.compile_node(
             manifest.nodes['model.root.view'],
             manifest,
-            {}
+            write=False
         )
-
-        self.assertEqual(result,
-                         manifest.nodes.get('model.root.view'))
-
         self.assertTrue(result.extra_ctes_injected)
         self.assertEqualIgnoreWhitespace(
             result.compiled_sql,
@@ -349,9 +278,8 @@ def test__prepend_ctes(self):
              'select * from source_table'
              ') '
              'select * from __dbt__CTE__ephemeral'))
-        print(f"\n---- line 349 ----")
-
-        self.assertFalse(manifest.nodes['model.root.ephemeral'].extra_ctes_injected)
+        self.assertTrue(
+            manifest.nodes['model.root.ephemeral'].extra_ctes_injected)
 
     def test__prepend_ctes__cte_not_compiled(self):
         ephemeral_config = self.model_config.replace(materialized='ephemeral')
@@ -422,7 +350,8 @@ def test__prepend_ctes__cte_not_compiled(self):
                     raw_sql='select * from {{ref("ephemeral")}}',
                     compiled=True,
                     extra_ctes_injected=False,
-                    extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')],
+                    extra_ctes=[InjectedCTE(
+                        id='model.root.ephemeral', sql='select * from source_table')],
                     compiled_sql='select * from __dbt__CTE__ephemeral',
                     checksum=FileHash.from_contents(''),
                 ),
@@ -445,7 +374,8 @@ def test__prepend_ctes__cte_not_compiled(self):
                 manifest,
                 {}
             )
-            compile_node.assert_called_once_with(parsed_ephemeral, manifest, {})
+            compile_node.assert_called_once_with(
+                parsed_ephemeral, manifest, {})
 
         self.assertEqual(result,
                          manifest.nodes.get('model.root.view'))
@@ -459,7 +389,8 @@ def test__prepend_ctes__cte_not_compiled(self):
              ') '
              'select * from __dbt__CTE__ephemeral'))
 
-        self.assertTrue(manifest.nodes['model.root.ephemeral'].extra_ctes_injected)
+        self.assertTrue(
+            manifest.nodes['model.root.ephemeral'].extra_ctes_injected)
 
     def test__prepend_ctes__multiple_levels(self):
         ephemeral_config = self.model_config.replace(materialized='ephemeral')
@@ -467,7 +398,7 @@ def test__prepend_ctes__multiple_levels(self):
         manifest = Manifest(
             macros={},
             nodes={
-                'model.root.view': CompiledModelNode(
+                'model.root.view': ParsedModelNode(
                     name='view',
                     database='dbt',
                     schema='analytics',
@@ -477,18 +408,10 @@ def test__prepend_ctes__multiple_levels(self):
                     fqn=['root', 'view'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(nodes=['model.root.ephemeral']),
                     config=self.model_config,
-                    tags=[],
                     path='view.sql',
                     original_file_path='view.sql',
                     raw_sql='select * from {{ref("ephemeral")}}',
-                    compiled=True,
-                    extra_ctes_injected=False,
-                    extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql=None)],
-                    compiled_sql='select * from __dbt__CTE__ephemeral',
                     checksum=FileHash.from_contents(''),
 
                 ),
@@ -502,11 +425,7 @@ def test__prepend_ctes__multiple_levels(self):
                     fqn=['root', 'ephemeral'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(),
                     config=ephemeral_config,
-                    tags=[],
                     path='ephemeral.sql',
                     original_file_path='ephemeral.sql',
                     raw_sql='select * from {{ref("ephemeral_level_two")}}',
@@ -522,11 +441,7 @@ def test__prepend_ctes__multiple_levels(self):
                     fqn=['root', 'ephemeral_level_two'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(),
                     config=ephemeral_config,
-                    tags=[],
                     path='ephemeral_level_two.sql',
                     original_file_path='ephemeral_level_two.sql',
                     raw_sql='select * from source_table',
@@ -542,11 +457,7 @@ def test__prepend_ctes__multiple_levels(self):
         )
 
         compiler = dbt.compilation.Compiler(self.config)
-        result, _ = compiler._recursively_prepend_ctes(
-            manifest.nodes['model.root.view'],
-            manifest,
-            {}
-        )
+        result = compiler.compile_node(manifest.nodes['model.root.view'], manifest, write=False)
 
         self.assertEqual(result, manifest.nodes['model.root.view'])
         self.assertTrue(result.extra_ctes_injected)
@@ -560,6 +471,97 @@ def test__prepend_ctes__multiple_levels(self):
              'select * from __dbt__CTE__ephemeral'))
 
         self.assertTrue(manifest.nodes['model.root.ephemeral'].compiled)
-        self.assertTrue(manifest.nodes['model.root.ephemeral_level_two'].compiled)
-        self.assertTrue(manifest.nodes['model.root.ephemeral'].extra_ctes_injected)
-        self.assertTrue(manifest.nodes['model.root.ephemeral_level_two'].extra_ctes_injected)
+        self.assertTrue(
+            manifest.nodes['model.root.ephemeral_level_two'].compiled)
+        self.assertTrue(
+            manifest.nodes['model.root.ephemeral'].extra_ctes_injected)
+        self.assertTrue(
+            manifest.nodes['model.root.ephemeral_level_two'].extra_ctes_injected)
+
+    def test__prepend_ctes__valid_ephemeral_sql(self):
+        """Assert that the compiled sql for ephemeral models is valid and can be executed on its own"""
+        ephemeral_config = self.model_config.replace(materialized='ephemeral')
+
+        manifest = Manifest(
+            macros={},
+            nodes={
+                'model.root.view': ParsedModelNode(
+                    name='view',
+                    database='dbt',
+                    schema='analytics',
+                    alias='view',
+                    resource_type=NodeType.Model,
+                    unique_id='model.root.view',
+                    fqn=['root', 'view'],
+                    package_name='root',
+                    root_path='/usr/src/app',
+                    config=self.model_config,
+                    path='view.sql',
+                    original_file_path='view.sql',
+                    raw_sql='select * from {{ref("ephemeral")}}',
+                    checksum=FileHash.from_contents(''),
+                ),
+                'model.root.inner_ephemeral': ParsedModelNode(
+                    name='inner_ephemeral',
+                    database='dbt',
+                    schema='analytics',
+                    alias='inner_ephemeral',
+                    resource_type=NodeType.Model,
+                    unique_id='model.root.inner_ephemeral',
+                    fqn=['root', 'inner_ephemeral'],
+                    package_name='root',
+                    root_path='/usr/src/app',
+                    config=ephemeral_config,
+                    path='inner_ephemeral.sql',
+                    original_file_path='inner_ephemeral.sql',
+                    raw_sql='select * from source_table',
+                    checksum=FileHash.from_contents(''),
+                ),
+                 'model.root.ephemeral': ParsedModelNode(
+                    name='ephemeral',
+                    database='dbt',
+                    schema='analytics',
+                    alias='ephemeral',
+                    resource_type=NodeType.Model,
+                    unique_id='model.root.ephemeral',
+                    fqn=['root', 'ephemeral'],
+                    package_name='root',
+                    root_path='/usr/src/app',
+                    config=ephemeral_config,
+                    path='ephemeral.sql',
+                    original_file_path='ephemeral.sql',
+                    raw_sql='select * from {{ ref("inner_ephemeral") }}',
+                    checksum=FileHash.from_contents(''),
+                ),
+            },
+            sources={},
+            docs={},
+            disabled=[],
+            files={},
+            exposures={},
+            selectors={},
+        )
+
+        compiler = dbt.compilation.Compiler(self.config)
+        result = compiler.compile_node(
+            manifest.nodes['model.root.view'],
+            manifest,
+            write=False
+        )
+        self.assertEqualIgnoreWhitespace(
+            result.compiled_sql,
+            ('with __dbt__CTE__inner_ephemeral as ('
+             'select * from source_table'
+             '), '
+             '__dbt__CTE__ephemeral as ('
+             'select * from __dbt__CTE__inner_ephemeral'
+             ') '
+             'select * from __dbt__CTE__ephemeral'))
+        self.assertEqualIgnoreWhitespace(
+            manifest.nodes['model.root.ephemeral'].compiled_sql,
+            ('with __dbt__CTE__inner_ephemeral as ('
+            'select * from source_table'
+            ')  '
+            'select * from __dbt__CTE__inner_ephemeral')
+        )
+