Repository: dbt-labs/dbt-core

Issue:
[Static parser] hook ref missing from sample during comparison
When a model includes a novel `ref()` inside a project-configured hook, the static parser works correctly, but the comparison for sampling/tracking does not.

### Reproduction case
```sql
-- models/model_a.sql
select 1 as id
```
```sql
-- models/subdir/model_b.sql
select * from {{ ref('model_a') }}
```
```sql
-- models/model_c.sql
select 1 as id
```
```sql
-- macros/ref_my_other_model.sql
{% macro ref_my_other_model() %}
  select * from {{ ref('model_c') }}
{% endmacro %}
```
```yml
# dbt_project.yml
models:
  <project_name>:
    subdir:
      +post-hook: "{{ ref_my_other_model() }}"
```

We get the correct results in `node.refs`, both with and without the experimental parser enabled:

```bash
$ dbt --no-partial-parse ls -s model_b --output json --output-keys refs
{"refs": [["model_a"], ["model_c"]]}
```
```bash
$ dbt --use-experimental-parser  --no-partial-parse ls -s model_b --output json --output-keys refs
{"refs": [["model_c"], ["model_a"]]}
```

### Diagnostic

The issue is when the tracking takes place. In order to have an apples-to-apples comparison between the static parser output and the Jinja-rendered output, we'd need some way to do this step _before_ the sample comparison:

https://github.com/dbt-labs/dbt/blob/c009485de223cb86eab2a1db201c02cfea0ceaa2/core/dbt/parser/models.py#L110-L113

Whereas, currently, we perform the comparison using _only_ the output of static extraction:

https://github.com/dbt-labs/dbt/blob/c009485de223cb86eab2a1db201c02cfea0ceaa2/core/dbt/parser/models.py#L216-L220


Discussion:

Resolving patch:
diff --git a/core/dbt/parser/models.py b/core/dbt/parser/models.py
--- a/core/dbt/parser/models.py
+++ b/core/dbt/parser/models.py
@@ -1,3 +1,5 @@
+
+from copy import deepcopy
 from dbt.context.context_config import ContextConfig
 from dbt.contracts.graph.parsed import ParsedModelNode
 import dbt.flags as flags
@@ -11,7 +13,11 @@
 from functools import reduce
 from itertools import chain
 import random
-from typing import Any, Dict, Iterator, List, Optional, Union
+from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
+
+# debug loglines are used for integration testing. If you change
+# the code at the beginning of the debug line, change the tests in
+# test/integration/072_experimental_parser_tests/test_all_experimental_parser.py
 
 
 class ModelParser(SimpleSQLParser[ParsedModelNode]):
@@ -40,16 +46,11 @@ def render_update(
         experimentally_parsed: Optional[Union[str, Dict[str, List[Any]]]] = None
         config_call_dict: Dict[str, Any] = {}
         source_calls: List[List[str]] = []
+        result: List[str] = []
 
         # run the experimental parser if the flag is on or if we're sampling
         if flags.USE_EXPERIMENTAL_PARSER or sample:
             if self._has_banned_macro(node):
-                # this log line is used for integration testing. If you change
-                # the code at the beginning of the line change the tests in
-                # test/integration/072_experimental_parser_tests/test_all_experimental_parser.py
-                logger.debug(
-                    f"1601: parser fallback to jinja because of macro override for {node.path}"
-                )
                 experimentally_parsed = "has_banned_macro"
             else:
                 # run the experimental parser and return the results
@@ -75,69 +76,76 @@ def render_update(
                 source_calls.append([s[0], s[1]])
             experimentally_parsed['sources'] = source_calls
 
-        # normal dbt run
+        # if we're sampling during a normal dbt run, populate an entirely new node to compare
         if not flags.USE_EXPERIMENTAL_PARSER:
-            # normal rendering
-            super().render_update(node, config)
-            # if we're sampling, compare for correctness
-            if sample:
-                result = _get_sample_result(
-                    experimentally_parsed,
-                    config_call_dict,
-                    source_calls,
-                    node,
-                    config
+            if sample and isinstance(experimentally_parsed, dict):
+                # if this will _never_ mutate anything `self` we could avoid these deep copies,
+                # but we can't really guarantee that going forward.
+                model_parser_copy = self.partial_deepcopy()
+                exp_sample_node = deepcopy(node)
+                exp_sample_config = deepcopy(config)
+
+                model_parser_copy.populate(
+                    exp_sample_node,
+                    exp_sample_config,
+                    experimentally_parsed
                 )
-                # fire a tracking event. this fires one event for every sample
-                # so that we have data on a per file basis. Not only can we expect
-                # no false positives or misses, we can expect the number model
-                # files parseable by the experimental parser to match our internal
-                # testing.
-                if tracking.active_user is not None:  # None in some tests
-                    tracking.track_experimental_parser_sample({
-                        "project_id": self.root_project.hashed_name(),
-                        "file_id": utils.get_hash(node),
-                        "status": result
-                    })
+
+            super().render_update(node, config)
 
         # if the --use-experimental-parser flag was set, and the experimental parser succeeded
-        elif isinstance(experimentally_parsed, Dict):
-            # since it doesn't need python jinja, fit the refs, sources, and configs
-            # into the node. Down the line the rest of the node will be updated with
-            # this information. (e.g. depends_on etc.)
-            config._config_call_dict = config_call_dict
-
-            # this uses the updated config to set all the right things in the node.
-            # if there are hooks present, it WILL render jinja. Will need to change
-            # when the experimental parser supports hooks
-            self.update_parsed_node_config(node, config)
-
-            # update the unrendered config with values from the file.
+        elif isinstance(experimentally_parsed, dict):
+            # update the unrendered config with values from the static parser.
             # values from yaml files are in there already
-            node.unrendered_config.update(dict(experimentally_parsed['configs']))
-
-            # set refs and sources on the node object
-            node.refs += experimentally_parsed['refs']
-            node.sources += experimentally_parsed['sources']
-
-            # configs don't need to be merged into the node
-            # setting them in config._config_call_dict is sufficient
+            self.populate(
+                node,
+                config,
+                experimentally_parsed
+            )
 
             self.manifest._parsing_info.static_analysis_parsed_path_count += 1
 
         # the experimental parser didn't run on this model.
         # fall back to python jinja rendering.
-        elif experimentally_parsed in ["has_banned_macro"]:
-            # not logging here since the reason should have been logged above
+        elif isinstance(experimentally_parsed, str):
+            if experimentally_parsed == "cannot_parse":
+                result += ["01_stable_parser_cannot_parse"]
+                logger.debug(
+                    f"1602: parser fallback to jinja for {node.path}"
+                )
+            elif experimentally_parsed == "has_banned_macro":
+                result += ["08_has_banned_macro"]
+                logger.debug(
+                    f"1601: parser fallback to jinja because of macro override for {node.path}"
+                )
+
             super().render_update(node, config)
-        # the experimental parser ran on this model and failed.
-        # fall back to python jinja rendering.
+        # otherwise jinja rendering.
         else:
-            logger.debug(
-                f"1602: parser fallback to jinja because of extractor failure for {node.path}"
-            )
             super().render_update(node, config)
 
+        if sample and isinstance(experimentally_parsed, dict):
+            # now that the sample succeeded, is populated and the current
+            # values are rendered, compare the two and collect the tracking messages
+            result += _get_exp_sample_result(
+                exp_sample_node,
+                exp_sample_config,
+                node,
+                config,
+            )
+
+        # fire a tracking event. this fires one event for every sample
+        # so that we have data on a per file basis. Not only can we expect
+        # no false positives or misses, we can expect the number model
+        # files parseable by the experimental parser to match our internal
+        # testing.
+        if result and tracking.active_user is not None:  # None in some tests
+            tracking.track_experimental_parser_sample({
+                "project_id": self.root_project.hashed_name(),
+                "file_id": utils.get_hash(node),
+                "status": result
+            })
+
     # checks for banned macros
     def _has_banned_macro(
         self, node: ParsedModelNode
@@ -163,64 +171,117 @@ def _has_banned_macro(
             False
         )
 
+    # this method updates the model note rendered and unrendered config as well
+    # as the node object. Used to populate these values when circumventing jinja
+    # rendering like the static parser.
+    def populate(
+        self,
+        node: ParsedModelNode,
+        config: ContextConfig,
+        experimentally_parsed: Dict[str, Any]
+    ):
+        # manually fit configs in
+        config._config_call_dict = _get_config_call_dict(experimentally_parsed)
+
+        # if there are hooks present this, it WILL render jinja. Will need to change
+        # when the experimental parser supports hooks
+        self.update_parsed_node_config(node, config)
+
+        # update the unrendered config with values from the file.
+        # values from yaml files are in there already
+        node.unrendered_config.update(experimentally_parsed['configs'])
+
+        # set refs and sources on the node object
+        node.refs += experimentally_parsed['refs']
+        node.sources += experimentally_parsed['sources']
+
+        # configs don't need to be merged into the node because they
+        # are read from config._config_call_dict
+
+    # the manifest is often huge so this method avoids deepcopying it
+    def partial_deepcopy(self):
+        return ModelParser(
+            deepcopy(self.project),
+            self.manifest,
+            deepcopy(self.root_project)
+        )
+
+
+# pure function. safe to use elsewhere, but unlikely to be useful outside this file.
+def _get_config_call_dict(
+    static_parser_result: Dict[str, List[Any]]
+) -> Dict[str, Any]:
+    config_call_dict: Dict[str, Any] = {}
+
+    for c in static_parser_result['configs']:
+        ContextConfig._add_config_call(config_call_dict, {c[0]: c[1]})
+
+    return config_call_dict
+
 
 # returns a list of string codes to be sent as a tracking event
-def _get_sample_result(
-    sample_output: Optional[Union[str, Dict[str, Any]]],
-    config_call_dict: Dict[str, Any],
-    source_calls: List[List[str]],
+def _get_exp_sample_result(
+    sample_node: ParsedModelNode,
+    sample_config: ContextConfig,
     node: ParsedModelNode,
     config: ContextConfig
 ) -> List[str]:
-    result: List[str] = []
-    # experimental parser didn't run
-    if sample_output is None:
-        result += ["09_experimental_parser_skipped"]
-    # experimental parser couldn't parse
-    elif (isinstance(sample_output, str)):
-        if sample_output == "cannot_parse":
-            result += ["01_experimental_parser_cannot_parse"]
-        elif sample_output == "has_banned_macro":
-            result += ["08_has_banned_macro"]
-    else:
-        # look for false positive configs
-        for k in config_call_dict.keys():
-            if k not in config._config_call_dict:
-                result += ["02_false_positive_config_value"]
-                break
-
-        # look for missed configs
-        for k in config._config_call_dict.keys():
-            if k not in config_call_dict:
-                result += ["03_missed_config_value"]
-                break
-
-        # look for false positive sources
-        for s in sample_output['sources']:
-            if s not in node.sources:
-                result += ["04_false_positive_source_value"]
-                break
-
-        # look for missed sources
-        for s in node.sources:
-            if s not in sample_output['sources']:
-                result += ["05_missed_source_value"]
-                break
-
-        # look for false positive refs
-        for r in sample_output['refs']:
-            if r not in node.refs:
-                result += ["06_false_positive_ref_value"]
-                break
-
-        # look for missed refs
-        for r in node.refs:
-            if r not in sample_output['refs']:
-                result += ["07_missed_ref_value"]
-                break
-
-        # if there are no errors, return a success value
-        if not result:
-            result = ["00_exact_match"]
+    result: List[Tuple[int, str]] = _get_sample_result(sample_node, sample_config, node, config)
+
+    def process(codemsg):
+        code, msg = codemsg
+        return f"0{code}_experimental_{msg}"
+
+    return list(map(process, result))
+
+
+# returns a list of messages and int codes and messages that need a single digit
+# prefix to be prepended before being sent as a tracking event
+def _get_sample_result(
+    sample_node: ParsedModelNode,
+    sample_config: ContextConfig,
+    node: ParsedModelNode,
+    config: ContextConfig
+) -> List[Tuple[int, str]]:
+    result: List[Tuple[int, str]] = []
+    # look for false positive configs
+    for k in sample_config._config_call_dict:
+        if k not in config._config_call_dict:
+            result += [(2, "false_positive_config_value")]
+            break
+
+    # look for missed configs
+    for k in config._config_call_dict.keys():
+        if k not in sample_config._config_call_dict.keys():
+            result += [(3, "missed_config_value")]
+            break
+
+    # look for false positive sources
+    for s in sample_node.sources:
+        if s not in node.sources:
+            result += [(4, "false_positive_source_value")]
+            break
+
+    # look for missed sources
+    for s in node.sources:
+        if s not in sample_node.sources:
+            result += [(5, "missed_source_value")]
+            break
+
+    # look for false positive refs
+    for r in sample_node.refs:
+        if r not in node.refs:
+            result += [(6, "false_positive_ref_value")]
+            break
+
+    # look for missed refs
+    for r in node.refs:
+        if r not in sample_node.refs:
+            result += [(7, "missed_ref_value")]
+            break
+
+    # if there are no errors, return a success value
+    if not result:
+        result = [(0, "exact_match")]
 
     return result


Test patch:
diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py
--- a/test/unit/test_parser.py
+++ b/test/unit/test_parser.py
@@ -5,9 +5,11 @@
 import os
 import yaml
 
+from copy import deepcopy
 import dbt.flags
 import dbt.parser
 from dbt import tracking
+from dbt.context.context_config import ContextConfig
 from dbt.exceptions import CompilationException
 from dbt.parser import (
     ModelParser, MacroParser, DataTestParser, SchemaParser,
@@ -32,6 +34,9 @@
     UnpatchedSourceDefinition
 )
 from dbt.contracts.graph.unparsed import Docs
+from dbt.parser.models import (
+    _get_config_call_dict, _get_exp_sample_result, _get_sample_result
+)
 import itertools
 from .utils import config_from_parts_or_dicts, normalize, generate_name_macros, MockNode, MockSource, MockDocumentation
 
@@ -573,6 +578,144 @@ def test_built_in_macro_override_detection(self):
         assert(self.parser._has_banned_macro(node))
 
 
+class StaticModelParserUnitTest(BaseParserTest):
+
+    def setUp(self):
+        super().setUp()
+        self.parser = ModelParser(
+            project=self.snowplow_project_config,
+            manifest=self.manifest,
+            root_project=self.root_project_config,
+        )
+        self.example_node = ParsedModelNode(
+            alias='model_1',
+            name='model_1',
+            database='test',
+            schema='analytics',
+            resource_type=NodeType.Model,
+            unique_id='model.snowplow.model_1',
+            fqn=['snowplow', 'nested', 'model_1'],
+            package_name='snowplow',
+            original_file_path=normalize('models/nested/model_1.sql'),
+            root_path=get_abs_os_path('./dbt_packages/snowplow'),
+            config=NodeConfig(materialized='table'),
+            path=normalize('nested/model_1.sql'),
+            raw_sql='{{ config(materialized="table") }}select 1 as id',
+            checksum=None,
+            unrendered_config={'materialized': 'table'},
+        )
+        self.example_config = ContextConfig(
+            self.root_project_config,
+            self.example_node.fqn,
+            self.example_node.resource_type,
+            self.snowplow_project_config,
+        )
+
+    def file_block_for(self, data, filename):
+        return super().file_block_for(data, filename, 'models')
+
+    # tests that configs get extracted properly. the function should respect merge behavior,
+    # but becuase it's only reading from one dictionary it won't matter except in edge cases
+    # like this example with tags changing type to a list.
+    def test_config_shifting(self):
+        static_parser_result = {
+            'configs': [
+                ('hello', 'world'),
+                ('flag', True),
+                ('tags', 'tag1'),
+                ('tags', 'tag2')
+            ]
+        }
+        expected = {
+            'hello': 'world',
+            'flag': True,
+            'tags': ['tag1', 'tag2']
+        }
+        got = _get_config_call_dict(static_parser_result)
+        self.assertEqual(expected, got)
+
+    def test_sample_results(self):
+        # --- missed ref --- #
+        node = deepcopy(self.example_node)
+        config = deepcopy(self.example_config)
+        sample_node = deepcopy(self.example_node)
+        sample_config = deepcopy(self.example_config)
+
+        sample_node.refs = []
+        node.refs = ['myref']
+
+        result = _get_sample_result(sample_node, sample_config, node, config)
+        self.assertEqual([(7, "missed_ref_value")], result)
+
+        # --- false positive ref --- #
+        node = deepcopy(self.example_node)
+        config = deepcopy(self.example_config)
+        sample_node = deepcopy(self.example_node)
+        sample_config = deepcopy(self.example_config)
+
+        sample_node.refs = ['myref']
+        node.refs = []
+
+        result = _get_sample_result(sample_node, sample_config, node, config)
+        self.assertEqual([(6, "false_positive_ref_value")], result)
+
+        # --- missed source --- #
+        node = deepcopy(self.example_node)
+        config = deepcopy(self.example_config)
+        sample_node = deepcopy(self.example_node)
+        sample_config = deepcopy(self.example_config)
+
+        sample_node.sources = []
+        node.sources = [['abc', 'def']]
+
+        result = _get_sample_result(sample_node, sample_config, node, config)
+        self.assertEqual([(5, 'missed_source_value')], result)
+
+        # --- false positive source --- #
+        node = deepcopy(self.example_node)
+        config = deepcopy(self.example_config)
+        sample_node = deepcopy(self.example_node)
+        sample_config = deepcopy(self.example_config)
+
+        sample_node.sources = [['abc', 'def']]
+        node.sources = []
+
+        result = _get_sample_result(sample_node, sample_config, node, config)
+        self.assertEqual([(4, 'false_positive_source_value')], result)
+
+        # --- missed config --- #
+        node = deepcopy(self.example_node)
+        config = deepcopy(self.example_config)
+        sample_node = deepcopy(self.example_node)
+        sample_config = deepcopy(self.example_config)
+
+        sample_config._config_call_dict = {}
+        config._config_call_dict = {'key': 'value'}
+
+        result = _get_sample_result(sample_node, sample_config, node, config)
+        self.assertEqual([(3, 'missed_config_value')], result)
+
+        # --- false positive config --- #
+        node = deepcopy(self.example_node)
+        config = deepcopy(self.example_config)
+        sample_node = deepcopy(self.example_node)
+        sample_config = deepcopy(self.example_config)
+
+        sample_config._config_call_dict = {'key': 'value'}
+        config._config_call_dict = {}
+
+        result = _get_sample_result(sample_node, sample_config, node, config)
+        self.assertEqual([(2, "false_positive_config_value")], result)
+
+    def test_exp_sample_results(self):
+        node = deepcopy(self.example_node)
+        config = deepcopy(self.example_config)
+        sample_node = deepcopy(self.example_node)
+        sample_config = deepcopy(self.example_config)
+        result = _get_exp_sample_result(sample_node, sample_config, node, config)
+        self.assertEqual(["00_experimental_exact_match"], result)
+
+
 class SnapshotParserTest(BaseParserTest):
     def setUp(self):
         super().setUp()