Repository: dbt-labs/dbt-core

Issue:
Turn on experimental parser by default
Pre-reqs:
- [ ] Macros that override `ref` and `source`
- [ ] Adding second flag to put this logic behind instead


Discussion:
Broke this out from the over-arching story: https://github.com/dbt-labs/dbt/issues/3502

Resolving patch:
diff --git a/core/dbt/parser/models.py b/core/dbt/parser/models.py
--- a/core/dbt/parser/models.py
+++ b/core/dbt/parser/models.py
@@ -1,15 +1,17 @@
 from dbt.context.context_config import ContextConfig
 from dbt.contracts.graph.parsed import ParsedModelNode
 import dbt.flags as flags
-import dbt.tracking
+from dbt.logger import GLOBAL_LOGGER as logger
 from dbt.node_types import NodeType
 from dbt.parser.base import SimpleSQLParser
 from dbt.parser.search import FileBlock
 import dbt.tracking as tracking
 from dbt import utils
 from dbt_extractor import ExtractionError, py_extract_from_source  # type: ignore
+from functools import reduce
+from itertools import chain
 import random
-from typing import Any, Dict, List
+from typing import Any, Dict, Iterator, List, Optional, Union
 
 
 class ModelParser(SimpleSQLParser[ParsedModelNode]):
@@ -26,32 +28,51 @@ def resource_type(self) -> NodeType:
     def get_compiled_path(cls, block: FileBlock):
         return block.path.relative_path
 
+    # TODO when this is turned on by default, simplify the nasty if/else tree inside this method.
     def render_update(
         self, node: ParsedModelNode, config: ContextConfig
     ) -> None:
-        self.manifest._parsing_info.static_analysis_path_count += 1
-
         # `True` roughly 1/100 times this function is called
         sample: bool = random.randint(1, 101) == 100
 
+        # top-level declaration of variables
+        experimentally_parsed: Optional[Union[str, Dict[str, List[Any]]]] = None
+        config_call_dict: Dict[str, Any] = {}
+        source_calls: List[List[str]] = []
+
         # run the experimental parser if the flag is on or if we're sampling
         if flags.USE_EXPERIMENTAL_PARSER or sample:
-            try:
-                experimentally_parsed: Dict[str, List[Any]] = py_extract_from_source(node.raw_sql)
-
-                # second config format
-                config_call_dict: Dict[str, Any] = {}
-                for c in experimentally_parsed['configs']:
-                    ContextConfig._add_config_call(config_call_dict, {c[0]: c[1]})
-
-                # format sources TODO change extractor to match this type
-                source_calls: List[List[str]] = []
-                for s in experimentally_parsed['sources']:
-                    source_calls.append([s[0], s[1]])
-                experimentally_parsed['sources'] = source_calls
-
-            except ExtractionError as e:
-                experimentally_parsed = e
+            if self._has_banned_macro(node):
+                # this log line is used for integration testing. If you change
+                # the code at the beginning of the line change the tests in
+                # test/integration/072_experimental_parser_tests/test_all_experimental_parser.py
+                logger.debug(
+                    f"1601: parser fallback to jinja because of macro override for {node.path}"
+                )
+                experimentally_parsed = "has_banned_macro"
+            else:
+                # run the experimental parser and return the results
+                try:
+                    experimentally_parsed = py_extract_from_source(
+                        node.raw_sql
+                    )
+                    logger.debug(f"1699: statically parsed {node.path}")
+                # if we want information on what features are barring the experimental
+                # parser from reading model files, this is where we would add that
+                # since that information is stored in the `ExtractionError`.
+                except ExtractionError:
+                    experimentally_parsed = "cannot_parse"
+
+        # if the parser succeeded, extract some data in easy-to-compare formats
+        if isinstance(experimentally_parsed, dict):
+            # create second config format
+            for c in experimentally_parsed['configs']:
+                ContextConfig._add_config_call(config_call_dict, {c[0]: c[1]})
+
+            # format sources TODO change extractor to match this type
+            for s in experimentally_parsed['sources']:
+                source_calls.append([s[0], s[1]])
+            experimentally_parsed['sources'] = source_calls
 
         # normal dbt run
         if not flags.USE_EXPERIMENTAL_PARSER:
@@ -59,57 +80,19 @@ def render_update(
             super().render_update(node, config)
             # if we're sampling, compare for correctness
             if sample:
-                result: List[str] = []
-                # experimental parser couldn't parse
-                if isinstance(experimentally_parsed, Exception):
-                    result += ["01_experimental_parser_cannot_parse"]
-                else:
-                    # look for false positive configs
-                    for k in config_call_dict.keys():
-                        if k not in config._config_call_dict:
-                            result += ["02_false_positive_config_value"]
-                            break
-
-                    # look for missed configs
-                    for k in config._config_call_dict.keys():
-                        if k not in config_call_dict:
-                            result += ["03_missed_config_value"]
-                            break
-
-                    # look for false positive sources
-                    for s in experimentally_parsed['sources']:
-                        if s not in node.sources:
-                            result += ["04_false_positive_source_value"]
-                            break
-
-                    # look for missed sources
-                    for s in node.sources:
-                        if s not in experimentally_parsed['sources']:
-                            result += ["05_missed_source_value"]
-                            break
-
-                    # look for false positive refs
-                    for r in experimentally_parsed['refs']:
-                        if r not in node.refs:
-                            result += ["06_false_positive_ref_value"]
-                            break
-
-                    # look for missed refs
-                    for r in node.refs:
-                        if r not in experimentally_parsed['refs']:
-                            result += ["07_missed_ref_value"]
-                            break
-
-                    # if there are no errors, return a success value
-                    if not result:
-                        result = ["00_exact_match"]
-
+                result = _get_sample_result(
+                    experimentally_parsed,
+                    config_call_dict,
+                    source_calls,
+                    node,
+                    config
+                )
                 # fire a tracking event. this fires one event for every sample
                 # so that we have data on a per file basis. Not only can we expect
                 # no false positives or misses, we can expect the number model
                 # files parseable by the experimental parser to match our internal
                 # testing.
-                if dbt.tracking.active_user is not None:  # None in some tests
+                if tracking.active_user is not None:  # None in some tests
                     tracking.track_experimental_parser_sample({
                         "project_id": self.root_project.hashed_name(),
                         "file_id": utils.get_hash(node),
@@ -117,7 +100,7 @@ def render_update(
                     })
 
         # if the --use-experimental-parser flag was set, and the experimental parser succeeded
-        elif not isinstance(experimentally_parsed, Exception):
+        elif isinstance(experimentally_parsed, Dict):
             # since it doesn't need python jinja, fit the refs, sources, and configs
             # into the node. Down the line the rest of the node will be updated with
             # this information. (e.g. depends_on etc.)
@@ -141,7 +124,102 @@ def render_update(
 
             self.manifest._parsing_info.static_analysis_parsed_path_count += 1
 
-        # the experimental parser tried and failed on this model.
+        # the experimental parser didn't run on this model.
+        # fall back to python jinja rendering.
+        elif experimentally_parsed in ["has_banned_macro"]:
+            # not logging here since the reason should have been logged above
+            super().render_update(node, config)
+        # the experimental parser ran on this model and failed.
         # fall back to python jinja rendering.
         else:
+            logger.debug(
+                f"1602: parser fallback to jinja because of extractor failure for {node.path}"
+            )
             super().render_update(node, config)
+
+    # checks for banned macros
+    def _has_banned_macro(
+        self, node: ParsedModelNode
+    ) -> bool:
+        # first check if there is a banned macro defined in scope for this model file
+        root_project_name = self.root_project.project_name
+        project_name = node.package_name
+        banned_macros = ['ref', 'source', 'config']
+
+        all_banned_macro_keys: Iterator[str] = chain.from_iterable(
+            map(
+                lambda name: [
+                    f"macro.{project_name}.{name}",
+                    f"macro.{root_project_name}.{name}"
+                ],
+                banned_macros
+            )
+        )
+
+        return reduce(
+            lambda z, key: z or (key in self.manifest.macros),
+            all_banned_macro_keys,
+            False
+        )
+
+
+# returns a list of string codes to be sent as a tracking event
+def _get_sample_result(
+    sample_output: Optional[Union[str, Dict[str, Any]]],
+    config_call_dict: Dict[str, Any],
+    source_calls: List[List[str]],
+    node: ParsedModelNode,
+    config: ContextConfig
+) -> List[str]:
+    result: List[str] = []
+    # experimental parser didn't run
+    if sample_output is None:
+        result += ["09_experimental_parser_skipped"]
+    # experimental parser couldn't parse
+    elif (isinstance(sample_output, str)):
+        if sample_output == "cannot_parse":
+            result += ["01_experimental_parser_cannot_parse"]
+        elif sample_output == "has_banned_macro":
+            result += ["08_has_banned_macro"]
+    else:
+        # look for false positive configs
+        for k in config_call_dict.keys():
+            if k not in config._config_call_dict:
+                result += ["02_false_positive_config_value"]
+                break
+
+        # look for missed configs
+        for k in config._config_call_dict.keys():
+            if k not in config_call_dict:
+                result += ["03_missed_config_value"]
+                break
+
+        # look for false positive sources
+        for s in sample_output['sources']:
+            if s not in node.sources:
+                result += ["04_false_positive_source_value"]
+                break
+
+        # look for missed sources
+        for s in node.sources:
+            if s not in sample_output['sources']:
+                result += ["05_missed_source_value"]
+                break
+
+        # look for false positive refs
+        for r in sample_output['refs']:
+            if r not in node.refs:
+                result += ["06_false_positive_ref_value"]
+                break
+
+        # look for missed refs
+        for r in node.refs:
+            if r not in sample_output['refs']:
+                result += ["07_missed_ref_value"]
+                break
+
+        # if there are no errors, return a success value
+        if not result:
+            result = ["00_exact_match"]
+
+    return result


Test patch:
diff --git a/test/integration/072_experimental_parser_tests/models/model_a.sql b/test/integration/072_experimental_parser_tests/basic/model_a.sql
similarity index 100%
rename from test/integration/072_experimental_parser_tests/models/model_a.sql
rename to test/integration/072_experimental_parser_tests/basic/model_a.sql
diff --git a/test/integration/072_experimental_parser_tests/models/schema.yml b/test/integration/072_experimental_parser_tests/basic/schema.yml
similarity index 100%
rename from test/integration/072_experimental_parser_tests/models/schema.yml
rename to test/integration/072_experimental_parser_tests/basic/schema.yml
diff --git a/test/integration/072_experimental_parser_tests/config_macro/macros/config.sql b/test/integration/072_experimental_parser_tests/config_macro/macros/config.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/072_experimental_parser_tests/config_macro/macros/config.sql
@@ -0,0 +1,3 @@
+{% macro config() %}
+
+{% endmacro %}
\ No newline at end of file
diff --git a/test/integration/072_experimental_parser_tests/config_macro/models/model_a.sql b/test/integration/072_experimental_parser_tests/config_macro/models/model_a.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/072_experimental_parser_tests/config_macro/models/model_a.sql
@@ -0,0 +1 @@
+select 1 as id
diff --git a/test/integration/072_experimental_parser_tests/config_macro/schema.yml b/test/integration/072_experimental_parser_tests/config_macro/schema.yml
new file mode 100644
--- /dev/null
+++ b/test/integration/072_experimental_parser_tests/config_macro/schema.yml
@@ -0,0 +1 @@
+version: 2
diff --git a/test/integration/072_experimental_parser_tests/ref_macro/macros/ref.sql b/test/integration/072_experimental_parser_tests/ref_macro/macros/ref.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/072_experimental_parser_tests/ref_macro/macros/ref.sql
@@ -0,0 +1,3 @@
+{% macro ref(model_name) %}
+
+{% endmacro %}
\ No newline at end of file
diff --git a/test/integration/072_experimental_parser_tests/ref_macro/models/model_a.sql b/test/integration/072_experimental_parser_tests/ref_macro/models/model_a.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/072_experimental_parser_tests/ref_macro/models/model_a.sql
@@ -0,0 +1 @@
+select 1 as id
diff --git a/test/integration/072_experimental_parser_tests/ref_macro/schema.yml b/test/integration/072_experimental_parser_tests/ref_macro/schema.yml
new file mode 100644
--- /dev/null
+++ b/test/integration/072_experimental_parser_tests/ref_macro/schema.yml
@@ -0,0 +1 @@
+version: 2
diff --git a/test/integration/072_experimental_parser_tests/source_macro/macros/source.sql b/test/integration/072_experimental_parser_tests/source_macro/macros/source.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/072_experimental_parser_tests/source_macro/macros/source.sql
@@ -0,0 +1,3 @@
+{% macro source(source_name, table_name) %}
+
+{% endmacro %}
\ No newline at end of file
diff --git a/test/integration/072_experimental_parser_tests/source_macro/models/model_a.sql b/test/integration/072_experimental_parser_tests/source_macro/models/model_a.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/072_experimental_parser_tests/source_macro/models/model_a.sql
@@ -0,0 +1 @@
+select 1 as id
diff --git a/test/integration/072_experimental_parser_tests/source_macro/schema.yml b/test/integration/072_experimental_parser_tests/source_macro/schema.yml
new file mode 100644
--- /dev/null
+++ b/test/integration/072_experimental_parser_tests/source_macro/schema.yml
@@ -0,0 +1 @@
+version: 2
diff --git a/test/integration/072_experimental_parser_tests/test_all_experimental_parser.py b/test/integration/072_experimental_parser_tests/test_all_experimental_parser.py
--- a/test/integration/072_experimental_parser_tests/test_all_experimental_parser.py
+++ b/test/integration/072_experimental_parser_tests/test_all_experimental_parser.py
@@ -14,17 +14,18 @@ def get_manifest():
         return None
 
 
-class TestAllExperimentalParser(DBTIntegrationTest):
+class TestBasicExperimentalParser(DBTIntegrationTest):
     @property
     def schema(self):
-        return "072_experimental_parser"
+        return "072_basic"
 
     @property
     def models(self):
-        return "models"
+        return "basic"
 
+    # test that the experimental parser extracts some basic ref, source, and config calls.
     @use_profile('postgres')
-    def test_postgres_experimental_parser(self):
+    def test_postgres_experimental_parser_basic(self):
         results = self.run_dbt(['--use-experimental-parser', 'parse'])
         manifest = get_manifest()
         node = manifest.nodes['model.test.model_a']
@@ -32,4 +33,93 @@ def test_postgres_experimental_parser(self):
         self.assertEqual(node.sources, [['my_src', 'my_tbl']])
         self.assertEqual(node.config._extra, {'x': True})
         self.assertEqual(node.config.tags, ['hello', 'world'])
-        
\ No newline at end of file
+
+class TestRefOverrideExperimentalParser(DBTIntegrationTest):
+    @property
+    def schema(self):
+        return "072_ref_macro"
+
+    @property
+    def models(self):
+        return "ref_macro/models"
+
+    @property
+    def project_config(self):
+        return {
+            'config-version': 2,
+            'macro-paths': ['source_macro', 'macros'],
+        }
+
+    # test that the experimental parser doesn't run if the ref built-in is overriden with a macro
+    @use_profile('postgres')
+    def test_postgres_experimental_parser_ref_override(self):
+        _, log_output = self.run_dbt_and_capture(['--debug', '--use-experimental-parser', 'parse'])
+        
+        print(log_output)
+
+        # successful static parsing
+        self.assertFalse("1699: " in log_output)
+        # ran static parser but failed
+        self.assertFalse("1602: " in log_output)
+        # didn't run static parser because dbt detected a built-in macro override
+        self.assertTrue("1601: " in log_output)
+
+class TestSourceOverrideExperimentalParser(DBTIntegrationTest):
+    @property
+    def schema(self):
+        return "072_source_macro"
+
+    @property
+    def models(self):
+        return "source_macro/models"
+
+    @property
+    def project_config(self):
+        return {
+            'config-version': 2,
+            'macro-paths': ['source_macro', 'macros'],
+        }
+
+    # test that the experimental parser doesn't run if the source built-in is overriden with a macro
+    @use_profile('postgres')
+    def test_postgres_experimental_parser_source_override(self):
+        _, log_output = self.run_dbt_and_capture(['--debug', '--use-experimental-parser', 'parse'])
+        
+        print(log_output)
+
+        # successful static parsing
+        self.assertFalse("1699: " in log_output)
+        # ran static parser but failed
+        self.assertFalse("1602: " in log_output)
+        # didn't run static parser because dbt detected a built-in macro override
+        self.assertTrue("1601: " in log_output)
+
+class TestConfigOverrideExperimentalParser(DBTIntegrationTest):
+    @property
+    def schema(self):
+        return "072_config_macro"
+
+    @property
+    def models(self):
+        return "config_macro/models"
+
+    @property
+    def project_config(self):
+        return {
+            'config-version': 2,
+            'macro-paths': ['config_macro', 'macros'],
+        }
+
+    # test that the experimental parser doesn't run if the config built-in is overriden with a macro
+    @use_profile('postgres')
+    def test_postgres_experimental_parser_config_override(self):
+        _, log_output = self.run_dbt_and_capture(['--debug', '--use-experimental-parser', 'parse'])
+        
+        print(log_output)
+
+        # successful static parsing
+        self.assertFalse("1699: " in log_output)
+        # ran static parser but failed
+        self.assertFalse("1602: " in log_output)
+        # didn't run static parser because dbt detected a built-in macro override
+        self.assertTrue("1601: " in log_output)
diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py
--- a/test/unit/test_parser.py
+++ b/test/unit/test_parser.py
@@ -32,7 +32,7 @@
     UnpatchedSourceDefinition
 )
 from dbt.contracts.graph.unparsed import Docs
-
+import itertools
 from .utils import config_from_parts_or_dicts, normalize, generate_name_macros, MockNode, MockSource, MockDocumentation
 
 
@@ -521,6 +521,57 @@ def test_parse_error(self):
             self.parser.parse_file(block)
 
 
+class StaticModelParserTest(BaseParserTest):
+    def setUp(self):
+        super().setUp()
+        self.parser = ModelParser(
+            project=self.snowplow_project_config,
+            manifest=self.manifest,
+            root_project=self.root_project_config,
+        )
+
+    def file_block_for(self, data, filename):
+        return super().file_block_for(data, filename, 'models')
+
+    # tests that when the ref built-in is overriden with a macro definition
+    # that the ModelParser can detect it. This does not test that the static
+    # parser does not run in this case. That test is in integration test suite 072
+    def test_built_in_macro_override_detection(self):
+        macro_unique_id = 'macro.root.ref'
+        self.parser.manifest.macros[macro_unique_id] = ParsedMacro(
+            name='ref',
+            resource_type=NodeType.Macro,
+            unique_id=macro_unique_id,
+            package_name='root',
+            original_file_path=normalize('macros/macro.sql'),
+            root_path=get_abs_os_path('./dbt_modules/root'),
+            path=normalize('macros/macro.sql'),
+            macro_sql='{% macro ref(model_name) %}{% set x = raise("boom") %}{% endmacro %}',
+        )
+
+        raw_sql = '{{ config(materialized="table") }}select 1 as id'
+        block = self.file_block_for(raw_sql, 'nested/model_1.sql')
+        node = ParsedModelNode(
+            alias='model_1',
+            name='model_1',
+            database='test',
+            schema='analytics',
+            resource_type=NodeType.Model,
+            unique_id='model.snowplow.model_1',
+            fqn=['snowplow', 'nested', 'model_1'],
+            package_name='snowplow',
+            original_file_path=normalize('models/nested/model_1.sql'),
+            root_path=get_abs_os_path('./dbt_modules/snowplow'),
+            config=NodeConfig(materialized='table'),
+            path=normalize('nested/model_1.sql'),
+            raw_sql=raw_sql,
+            checksum=block.file.checksum,
+            unrendered_config={'materialized': 'table'},
+        )
+
+        assert(self.parser._has_banned_macro(node))
+
+
 class SnapshotParserTest(BaseParserTest):
     def setUp(self):
         super().setUp()