Repository: dbt-labs/dbt-core

Issue:
Incomplete SQL compiled in dbt 0.20.0b1
### Describe the bug
Running `dbt compile` results in partly incomplete sql generated where WITH() clauses are missing for ephemeral models when the table to built is also ephemeral

### Steps To Reproduce
with colors_flat.sql as
```sql
{{
    config(
        materialized='ephemeral'
    )
}}

with colors as (
    select * from {{ ref('base_colors') }}
),

colorclassifications as (
    select * from {{ ref('base_colorclassifications') }} where is_leaf
)

select * from ....
```

gets compiled to

```sql
with colors as (
    select * from __dbt__cte__base_colors
),

colorclassifications as (
    select * from __dbt__cte__base_colorclassifications where is_leaf
)
```

where `base_colors` and  `base_colorclassifications` are ephemeral models as well.

### Expected behavior
The compiled SQL for __dbt__cte__base_colors and __dbt__cte__base_colorclassifications is missing, it should be inlined instead

### Screenshots and log output
If applicable, add screenshots or log output to help explain your problem.

### System information
**Which database are you using dbt with?**
- [ ] postgres
- [ ] redshift
- [ ] bigquery
- [X] snowflake
- [ ] other (specify: ____________)


**The output of `dbt --version`:**
```
installed version: 0.20.0-b1
   latest version: 0.19.1

Your version of dbt is ahead of the latest release!

Plugins:
  - snowflake: 0.20.0b1
```

**The operating system you're using:**
Windows 10

**The output of `python --version`:**
Python 3.9.4

### Additional context
Add any other context about the problem here.



Discussion:

Resolving patch:
diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py
--- a/core/dbt/compilation.py
+++ b/core/dbt/compilation.py
@@ -250,19 +250,19 @@ def _inject_ctes_into_sql(self, sql: str, ctes: List[InjectedCTE]) -> str:
 
         return str(parsed)
 
-    # This method is called by the 'compile_node' method. Starting
-    # from the node that it is passed in, it will recursively call
-    # itself using the 'extra_ctes'.  The 'ephemeral' models do
-    # not produce SQL that is executed directly, instead they
-    # are rolled up into the models that refer to them by
-    # inserting CTEs into the SQL.
     def _recursively_prepend_ctes(
         self,
         model: NonSourceCompiledNode,
         manifest: Manifest,
         extra_context: Optional[Dict[str, Any]],
     ) -> Tuple[NonSourceCompiledNode, List[InjectedCTE]]:
-
+        """This method is called by the 'compile_node' method. Starting
+        from the node that it is passed in, it will recursively call
+        itself using the 'extra_ctes'.  The 'ephemeral' models do
+        not produce SQL that is executed directly, instead they
+        are rolled up into the models that refer to them by
+        inserting CTEs into the SQL.
+        """
         if model.compiled_sql is None:
             raise RuntimeException(
                 'Cannot inject ctes into an unparsed node', model
@@ -320,19 +320,19 @@ def _recursively_prepend_ctes(
             _extend_prepended_ctes(prepended_ctes, new_prepended_ctes)
 
             new_cte_name = self.add_ephemeral_prefix(cte_model.name)
-            sql = f' {new_cte_name} as (\n{cte_model.compiled_sql}\n)'
+            rendered_sql = (
+                cte_model._pre_injected_sql or cte_model.compiled_sql
+            )
+            sql = f' {new_cte_name} as (\n{rendered_sql}\n)'
 
-        _add_prepended_cte(prepended_ctes, InjectedCTE(id=cte.id, sql=sql))
+            _add_prepended_cte(prepended_ctes, InjectedCTE(id=cte.id, sql=sql))
 
-        # We don't save injected_sql into compiled sql for ephemeral models
-        # because it will cause problems with processing of subsequent models.
-        # Ephemeral models do not produce executable SQL of their own.
-        if not model.is_ephemeral_model:
-            injected_sql = self._inject_ctes_into_sql(
-                model.compiled_sql,
-                prepended_ctes,
-            )
-            model.compiled_sql = injected_sql
+        injected_sql = self._inject_ctes_into_sql(
+            model.compiled_sql,
+            prepended_ctes,
+        )
+        model._pre_injected_sql = model.compiled_sql
+        model.compiled_sql = injected_sql
         model.extra_ctes_injected = True
         model.extra_ctes = prepended_ctes
         model.validate(model.to_dict(omit_none=True))
@@ -449,11 +449,6 @@ def _write_node(self, node: NonSourceCompiledNode) -> ManifestNode:
             )
         return node
 
-    # This is the main entry point into this code. It's called by
-    # CompileRunner.compile, GenericRPCRunner.compile, and
-    # RunTask.get_hook_sql. It calls '_compile_node' to convert
-    # the node into a compiled node, and then calls the
-    # recursive method to "prepend" the ctes.
     def compile_node(
         self,
         node: ManifestNode,
@@ -461,6 +456,12 @@ def compile_node(
         extra_context: Optional[Dict[str, Any]] = None,
         write: bool = True,
     ) -> NonSourceCompiledNode:
+        """This is the main entry point into this code. It's called by
+        CompileRunner.compile, GenericRPCRunner.compile, and
+        RunTask.get_hook_sql. It calls '_compile_node' to convert
+        the node into a compiled node, and then calls the
+        recursive method to "prepend" the ctes.
+        """
         node = self._compile_node(node, manifest, extra_context)
 
         node, _ = self._recursively_prepend_ctes(
diff --git a/core/dbt/contracts/graph/compiled.py b/core/dbt/contracts/graph/compiled.py
--- a/core/dbt/contracts/graph/compiled.py
+++ b/core/dbt/contracts/graph/compiled.py
@@ -43,6 +43,7 @@ class CompiledNode(ParsedNode, CompiledNodeMixin):
     extra_ctes_injected: bool = False
     extra_ctes: List[InjectedCTE] = field(default_factory=list)
     relation_name: Optional[str] = None
+    _pre_injected_sql: Optional[str] = None
 
     def set_cte(self, cte_id: str, sql: str):
         """This is the equivalent of what self.extra_ctes[cte_id] = sql would
@@ -55,6 +56,12 @@ def set_cte(self, cte_id: str, sql: str):
         else:
             self.extra_ctes.append(InjectedCTE(id=cte_id, sql=sql))
 
+    def __post_serialize__(self, dct):
+        dct = super().__post_serialize__(dct)
+        if '_pre_injected_sql' in dct:
+            del dct['_pre_injected_sql']
+        return dct
+
 
 @dataclass
 class CompiledAnalysisNode(CompiledNode):


Test patch:
diff --git a/test/unit/test_compiler.py b/test/unit/test_compiler.py
--- a/test/unit/test_compiler.py
+++ b/test/unit/test_compiler.py
@@ -62,22 +62,12 @@ def setUp(self):
 
         self.config = config_from_parts_or_dicts(project_cfg, profile_cfg)
 
-        self._generate_runtime_model_patch = patch.object(dbt.compilation, 'generate_runtime_model')
+        self._generate_runtime_model_patch = patch.object(
+            dbt.compilation, 'generate_runtime_model')
         self.mock_generate_runtime_model = self._generate_runtime_model_patch.start()
 
         inject_adapter(Plugin.adapter(self.config), Plugin)
 
-        # self.mock_adapter = PostgresAdapter MagicMock(type=MagicMock(return_value='postgres'))
-        # self.mock_adapter.Relation =
-        # self.mock_adapter.get_compiler.return_value = dbt.compilation.Compiler
-        # self.mock_plugin = MagicMock(
-        #     adapter=MagicMock(
-        #         credentials=MagicMock(return_value='postgres')
-        #     )
-        # )
-        # inject_adapter(self.mock_adapter, self.mock_plugin)
-        # so we can make an adapter
-
         def mock_generate_runtime_model_context(model, config, manifest):
             def ref(name):
                 result = f'__dbt__cte__{name}'
@@ -98,7 +88,7 @@ def test__prepend_ctes__already_has_cte(self):
         manifest = Manifest(
             macros={},
             nodes={
-                'model.root.view': CompiledModelNode(
+                'model.root.view': ParsedModelNode(
                     name='view',
                     database='dbt',
                     schema='analytics',
@@ -108,23 +98,13 @@ def test__prepend_ctes__already_has_cte(self):
                     fqn=['root', 'view'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(nodes=['model.root.ephemeral']),
                     config=self.model_config,
-                    tags=[],
                     path='view.sql',
                     original_file_path='view.sql',
-                    raw_sql='select * from {{ref("ephemeral")}}',
-                    compiled=True,
-                    extra_ctes_injected=False,
-                    extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')],
-                    compiled_sql=(
-                        'with cte as (select * from something_else) '
-                        'select * from __dbt__cte__ephemeral'),
+                    raw_sql='with cte as (select * from something_else) select * from {{ref("ephemeral")}}',
                     checksum=FileHash.from_contents(''),
                 ),
-                'model.root.ephemeral': CompiledModelNode(
+                'model.root.ephemeral': ParsedModelNode(
                     name='ephemeral',
                     database='dbt',
                     schema='analytics',
@@ -134,18 +114,10 @@ def test__prepend_ctes__already_has_cte(self):
                     fqn=['root', 'ephemeral'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(),
                     config=ephemeral_config,
-                    tags=[],
                     path='ephemeral.sql',
                     original_file_path='ephemeral.sql',
                     raw_sql='select * from source_table',
-                    compiled=True,
-                    compiled_sql='select * from source_table',
-                    extra_ctes_injected=False,
-                    extra_ctes=[],
                     checksum=FileHash.from_contents(''),
                 ),
             },
@@ -158,11 +130,8 @@ def test__prepend_ctes__already_has_cte(self):
         )
 
         compiler = dbt.compilation.Compiler(self.config)
-        result, _ = compiler._recursively_prepend_ctes(
-            manifest.nodes['model.root.view'],
-            manifest,
-            {}
-        )
+        result = compiler.compile_node(
+            manifest.nodes['model.root.view'], manifest, write=False)
 
         self.assertEqual(result, manifest.nodes['model.root.view'])
         self.assertEqual(result.extra_ctes_injected, True)
@@ -172,16 +141,15 @@ def test__prepend_ctes__already_has_cte(self):
              'select * from source_table'
              '), cte as (select * from something_else) '
              'select * from __dbt__cte__ephemeral'))
-
         self.assertEqual(
             manifest.nodes['model.root.ephemeral'].extra_ctes_injected,
-            False)
+            True)
 
     def test__prepend_ctes__no_ctes(self):
         manifest = Manifest(
             macros={},
             nodes={
-                'model.root.view': CompiledModelNode(
+                'model.root.view': ParsedModelNode(
                     name='view',
                     database='dbt',
                     schema='analytics',
@@ -191,23 +159,14 @@ def test__prepend_ctes__no_ctes(self):
                     fqn=['root', 'view'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(),
                     config=self.model_config,
-                    tags=[],
                     path='view.sql',
                     original_file_path='view.sql',
                     raw_sql=('with cte as (select * from something_else) '
                              'select * from source_table'),
-                    compiled=True,
-                    extra_ctes_injected=False,
-                    extra_ctes=[],
-                    compiled_sql=('with cte as (select * from something_else) '
-                                  'select * from source_table'),
                     checksum=FileHash.from_contents(''),
                 ),
-                'model.root.view_no_cte': CompiledModelNode(
+                'model.root.view_no_cte': ParsedModelNode(
                     name='view_no_cte',
                     database='dbt',
                     schema='analytics',
@@ -217,18 +176,10 @@ def test__prepend_ctes__no_ctes(self):
                     fqn=['root', 'view_no_cte'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(),
                     config=self.model_config,
-                    tags=[],
                     path='view.sql',
                     original_file_path='view.sql',
                     raw_sql='select * from source_table',
-                    compiled=True,
-                    extra_ctes_injected=False,
-                    extra_ctes=[],
-                    compiled_sql=('select * from source_table'),
                     checksum=FileHash.from_contents(''),
                 ),
             },
@@ -241,11 +192,8 @@ def test__prepend_ctes__no_ctes(self):
         )
 
         compiler = dbt.compilation.Compiler(self.config)
-        result, _ = compiler._recursively_prepend_ctes(
-            manifest.nodes['model.root.view'],
-            manifest,
-            {}
-        )
+        result = compiler.compile_node(
+            manifest.nodes['model.root.view'], manifest, write=False)
 
         self.assertEqual(
             result,
@@ -253,13 +201,13 @@ def test__prepend_ctes__no_ctes(self):
         self.assertTrue(result.extra_ctes_injected)
         self.assertEqualIgnoreWhitespace(
             result.compiled_sql,
-            manifest.nodes.get('model.root.view').compiled_sql)
+            ('with cte as (select * from something_else) '
+             'select * from source_table')
+        )
 
         compiler = dbt.compilation.Compiler(self.config)
-        result, _ = compiler._recursively_prepend_ctes(
-            manifest.nodes.get('model.root.view_no_cte'),
-            manifest,
-            {})
+        result = compiler.compile_node(
+            manifest.nodes['model.root.view_no_cte'], manifest, write=False)
 
         self.assertEqual(
             result,
@@ -267,7 +215,8 @@ def test__prepend_ctes__no_ctes(self):
         self.assertTrue(result.extra_ctes_injected)
         self.assertEqualIgnoreWhitespace(
             result.compiled_sql,
-            manifest.nodes.get('model.root.view_no_cte').compiled_sql)
+            'select * from source_table'
+        )
 
     def test__prepend_ctes(self):
         ephemeral_config = self.model_config.replace(materialized='ephemeral')
@@ -275,7 +224,7 @@ def test__prepend_ctes(self):
         manifest = Manifest(
             macros={},
             nodes={
-                'model.root.view': CompiledModelNode(
+                'model.root.view': ParsedModelNode(
                     name='view',
                     database='dbt',
                     schema='analytics',
@@ -285,21 +234,13 @@ def test__prepend_ctes(self):
                     fqn=['root', 'view'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(nodes=['model.root.ephemeral']),
                     config=self.model_config,
-                    tags=[],
                     path='view.sql',
                     original_file_path='view.sql',
                     raw_sql='select * from {{ref("ephemeral")}}',
-                    compiled=True,
-                    extra_ctes_injected=False,
-                    extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')],
-                    compiled_sql='select * from __dbt__cte__ephemeral',
                     checksum=FileHash.from_contents(''),
                 ),
-                'model.root.ephemeral': CompiledModelNode(
+                'model.root.ephemeral': ParsedModelNode(
                     name='ephemeral',
                     database='dbt',
                     schema='analytics',
@@ -309,18 +250,10 @@ def test__prepend_ctes(self):
                     fqn=['root', 'ephemeral'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(),
                     config=ephemeral_config,
-                    tags=[],
                     path='ephemeral.sql',
                     original_file_path='ephemeral.sql',
                     raw_sql='select * from source_table',
-                    compiled=True,
-                    extra_ctes_injected=False,
-                    extra_ctes=[],
-                    compiled_sql='select * from source_table',
                     checksum=FileHash.from_contents(''),
                 ),
             },
@@ -333,15 +266,11 @@ def test__prepend_ctes(self):
         )
 
         compiler = dbt.compilation.Compiler(self.config)
-        result, _ = compiler._recursively_prepend_ctes(
+        result = compiler.compile_node(
             manifest.nodes['model.root.view'],
             manifest,
-            {}
+            write=False
         )
-
-        self.assertEqual(result,
-                         manifest.nodes.get('model.root.view'))
-
         self.assertTrue(result.extra_ctes_injected)
         self.assertEqualIgnoreWhitespace(
             result.compiled_sql,
@@ -349,8 +278,8 @@ def test__prepend_ctes(self):
              'select * from source_table'
              ') '
              'select * from __dbt__cte__ephemeral'))
-
-        self.assertFalse(manifest.nodes['model.root.ephemeral'].extra_ctes_injected)
+        self.assertTrue(
+            manifest.nodes['model.root.ephemeral'].extra_ctes_injected)
 
     def test__prepend_ctes__cte_not_compiled(self):
         ephemeral_config = self.model_config.replace(materialized='ephemeral')
@@ -421,7 +350,8 @@ def test__prepend_ctes__cte_not_compiled(self):
                     raw_sql='select * from {{ref("ephemeral")}}',
                     compiled=True,
                     extra_ctes_injected=False,
-                    extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')],
+                    extra_ctes=[InjectedCTE(
+                        id='model.root.ephemeral', sql='select * from source_table')],
                     compiled_sql='select * from __dbt__cte__ephemeral',
                     checksum=FileHash.from_contents(''),
                 ),
@@ -444,7 +374,8 @@ def test__prepend_ctes__cte_not_compiled(self):
                 manifest,
                 {}
             )
-            compile_node.assert_called_once_with(parsed_ephemeral, manifest, {})
+            compile_node.assert_called_once_with(
+                parsed_ephemeral, manifest, {})
 
         self.assertEqual(result,
                          manifest.nodes.get('model.root.view'))
@@ -458,7 +389,8 @@ def test__prepend_ctes__cte_not_compiled(self):
              ') '
              'select * from __dbt__cte__ephemeral'))
 
-        self.assertTrue(manifest.nodes['model.root.ephemeral'].extra_ctes_injected)
+        self.assertTrue(
+            manifest.nodes['model.root.ephemeral'].extra_ctes_injected)
 
     def test__prepend_ctes__multiple_levels(self):
         ephemeral_config = self.model_config.replace(materialized='ephemeral')
@@ -466,7 +398,7 @@ def test__prepend_ctes__multiple_levels(self):
         manifest = Manifest(
             macros={},
             nodes={
-                'model.root.view': CompiledModelNode(
+                'model.root.view': ParsedModelNode(
                     name='view',
                     database='dbt',
                     schema='analytics',
@@ -476,18 +408,10 @@ def test__prepend_ctes__multiple_levels(self):
                     fqn=['root', 'view'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(nodes=['model.root.ephemeral']),
                     config=self.model_config,
-                    tags=[],
                     path='view.sql',
                     original_file_path='view.sql',
                     raw_sql='select * from {{ref("ephemeral")}}',
-                    compiled=True,
-                    extra_ctes_injected=False,
-                    extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql=None)],
-                    compiled_sql='select * from __dbt__cte__ephemeral',
                     checksum=FileHash.from_contents(''),
 
                 ),
@@ -501,11 +425,7 @@ def test__prepend_ctes__multiple_levels(self):
                     fqn=['root', 'ephemeral'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(),
                     config=ephemeral_config,
-                    tags=[],
                     path='ephemeral.sql',
                     original_file_path='ephemeral.sql',
                     raw_sql='select * from {{ref("ephemeral_level_two")}}',
@@ -521,11 +441,7 @@ def test__prepend_ctes__multiple_levels(self):
                     fqn=['root', 'ephemeral_level_two'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(),
                     config=ephemeral_config,
-                    tags=[],
                     path='ephemeral_level_two.sql',
                     original_file_path='ephemeral_level_two.sql',
                     raw_sql='select * from source_table',
@@ -541,11 +457,7 @@ def test__prepend_ctes__multiple_levels(self):
         )
 
         compiler = dbt.compilation.Compiler(self.config)
-        result, _ = compiler._recursively_prepend_ctes(
-            manifest.nodes['model.root.view'],
-            manifest,
-            {}
-        )
+        result = compiler.compile_node(manifest.nodes['model.root.view'], manifest, write=False)
 
         self.assertEqual(result, manifest.nodes['model.root.view'])
         self.assertTrue(result.extra_ctes_injected)
@@ -559,6 +471,97 @@ def test__prepend_ctes__multiple_levels(self):
              'select * from __dbt__cte__ephemeral'))
 
         self.assertTrue(manifest.nodes['model.root.ephemeral'].compiled)
-        self.assertTrue(manifest.nodes['model.root.ephemeral_level_two'].compiled)
-        self.assertTrue(manifest.nodes['model.root.ephemeral'].extra_ctes_injected)
-        self.assertTrue(manifest.nodes['model.root.ephemeral_level_two'].extra_ctes_injected)
+        self.assertTrue(
+            manifest.nodes['model.root.ephemeral_level_two'].compiled)
+        self.assertTrue(
+            manifest.nodes['model.root.ephemeral'].extra_ctes_injected)
+        self.assertTrue(
+            manifest.nodes['model.root.ephemeral_level_two'].extra_ctes_injected)
+
+    def test__prepend_ctes__valid_ephemeral_sql(self):
+        """Assert that the compiled sql for ephemeral models is valid and can be executed on its own"""
+        ephemeral_config = self.model_config.replace(materialized='ephemeral')
+
+        manifest = Manifest(
+            macros={},
+            nodes={
+                'model.root.view': ParsedModelNode(
+                    name='view',
+                    database='dbt',
+                    schema='analytics',
+                    alias='view',
+                    resource_type=NodeType.Model,
+                    unique_id='model.root.view',
+                    fqn=['root', 'view'],
+                    package_name='root',
+                    root_path='/usr/src/app',
+                    config=self.model_config,
+                    path='view.sql',
+                    original_file_path='view.sql',
+                    raw_sql='select * from {{ref("ephemeral")}}',
+                    checksum=FileHash.from_contents(''),
+                ),
+                'model.root.inner_ephemeral': ParsedModelNode(
+                    name='inner_ephemeral',
+                    database='dbt',
+                    schema='analytics',
+                    alias='inner_ephemeral',
+                    resource_type=NodeType.Model,
+                    unique_id='model.root.inner_ephemeral',
+                    fqn=['root', 'inner_ephemeral'],
+                    package_name='root',
+                    root_path='/usr/src/app',
+                    config=ephemeral_config,
+                    path='inner_ephemeral.sql',
+                    original_file_path='inner_ephemeral.sql',
+                    raw_sql='select * from source_table',
+                    checksum=FileHash.from_contents(''),
+                ),
+                 'model.root.ephemeral': ParsedModelNode(
+                    name='ephemeral',
+                    database='dbt',
+                    schema='analytics',
+                    alias='ephemeral',
+                    resource_type=NodeType.Model,
+                    unique_id='model.root.ephemeral',
+                    fqn=['root', 'ephemeral'],
+                    package_name='root',
+                    root_path='/usr/src/app',
+                    config=ephemeral_config,
+                    path='ephemeral.sql',
+                    original_file_path='ephemeral.sql',
+                    raw_sql='select * from {{ ref("inner_ephemeral") }}',
+                    checksum=FileHash.from_contents(''),
+                ),
+            },
+            sources={},
+            docs={},
+            disabled=[],
+            files={},
+            exposures={},
+            selectors={},
+        )
+
+        compiler = dbt.compilation.Compiler(self.config)
+        result = compiler.compile_node(
+            manifest.nodes['model.root.view'],
+            manifest,
+            write=False
+        )
+        self.assertEqualIgnoreWhitespace(
+            result.compiled_sql,
+            ('with __dbt__cte__inner_ephemeral as ('
+             'select * from source_table'
+             '), '
+             '__dbt__cte__ephemeral as ('
+             'select * from __dbt__cte__inner_ephemeral'
+             ') '
+             'select * from __dbt__cte__ephemeral'))
+        self.assertEqualIgnoreWhitespace(
+            manifest.nodes['model.root.ephemeral'].compiled_sql,
+            ('with __dbt__cte__inner_ephemeral as ('
+            'select * from source_table'
+            ')  '
+            'select * from __dbt__cte__inner_ephemeral')
+        )
+