Repository: dbt-labs/dbt-core

Issue:
Trigger full reparse if changes to widely used built-in macros
There is a special category of macro that:
- dbt uses at parse time, across almost all resources in the project
- is not explicitly called within project code, such that its dependency is not grabbed during static analysis; or is excluded from `depends_on.macros` because of being so widely used

Those macros are:
- `ref`
- `source`
- `config`
- `execute` (?)
- `generate_schema_name`
- `generate_database_name`
- `generate_alias_name`

The [builtins](https://docs.getdbt.com/reference/dbt-jinja-functions/builtins) are fairly self-explanatory (though `execute` is a weird inclusion in those docs).

As for `generate_x_name` macros: The code that uses these macros is _incredibly_ thorny, pretty old, and I think it has to do with a method named `generate_generate_component_name_macro`. I don't mind hard-coding that these seven need special treatment, and that any detected changes to them should be grounds for triggering a full re-parse (with logging + tracking to show for it).

Opened as a bug because this currently yields incorrect results :)


Discussion:
> any detected changes to them should be grounds for triggering a full re-parse

Edit: any detected changes to these macros, or _any macros that these macros depend on_ (via `depends_on.macros`)

Resolving patch:
diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py
--- a/core/dbt/parser/manifest.py
+++ b/core/dbt/parser/manifest.py
@@ -29,7 +29,7 @@
 from dbt.context.providers import ParseProvider
 from dbt.contracts.files import FileHash, ParseFileType, SchemaSourceFile
 from dbt.parser.read_files import read_files, load_source_file
-from dbt.parser.partial import PartialParsing
+from dbt.parser.partial import PartialParsing, special_override_macros
 from dbt.contracts.graph.compiled import ManifestNode
 from dbt.contracts.graph.manifest import (
     Manifest, Disabled, MacroManifest, ManifestStateCheck, ParsingInfo
@@ -130,7 +130,9 @@ def __init__(
         self.root_project: RuntimeConfig = root_project
         self.all_projects: Mapping[str, Project] = all_projects
         self.manifest: Manifest = Manifest()
+        self.new_manifest = self.manifest
         self.manifest.metadata = root_project.get_metadata()
+        self.macro_resolver = None  # built after macros are loaded
         self.started_at = int(time.time())
         # This is a MacroQueryStringSetter callable, which is called
         # later after we set the MacroManifest in the adapter. It sets
@@ -149,6 +151,7 @@ def __init__(
         # We need to know if we're actually partially parsing. It could
         # have been enabled, but not happening because of some issue.
         self.partially_parsing = False
+        self.partial_parser = None
 
         # This is a saved manifest from a previous run that's used for partial parsing
         self.saved_manifest: Optional[Manifest] = self.read_manifest_for_partial_parse()
@@ -207,13 +210,14 @@ def load(self):
             saved_files = self.saved_manifest.files
         for project in self.all_projects.values():
             read_files(project, self.manifest.files, project_parser_files, saved_files)
+        orig_project_parser_files = project_parser_files
         self._perf_info.path_count = len(self.manifest.files)
         self._perf_info.read_files_elapsed = (time.perf_counter() - start_read_files)
 
         skip_parsing = False
         if self.saved_manifest is not None:
-            partial_parsing = PartialParsing(self.saved_manifest, self.manifest.files)
-            skip_parsing = partial_parsing.skip_parsing()
+            self.partial_parser = PartialParsing(self.saved_manifest, self.manifest.files)
+            skip_parsing = self.partial_parser.skip_parsing()
             if skip_parsing:
                 # nothing changed, so we don't need to generate project_parser_files
                 self.manifest = self.saved_manifest
@@ -223,7 +227,7 @@ def load(self):
                 # files are different, we need to create a new set of
                 # project_parser_files.
                 try:
-                    project_parser_files = partial_parsing.get_parsing_files()
+                    project_parser_files = self.partial_parser.get_parsing_files()
                     self.partially_parsing = True
                     self.manifest = self.saved_manifest
                 except Exception:
@@ -245,7 +249,7 @@ def load(self):
 
                     # get file info for local logs
                     parse_file_type = None
-                    file_id = partial_parsing.processing_file
+                    file_id = self.partial_parser.processing_file
                     if file_id and file_id in self.manifest.files:
                         old_file = self.manifest.files[file_id]
                         parse_file_type = old_file.parse_file_type
@@ -270,20 +274,19 @@ def load(self):
             # We need to parse the macros first, so they're resolvable when
             # the other files are loaded
             start_load_macros = time.perf_counter()
-            for project in self.all_projects.values():
-                if project.project_name not in project_parser_files:
-                    continue
-                parser_files = project_parser_files[project.project_name]
-                if 'MacroParser' not in parser_files:
-                    continue
-                parser = MacroParser(project, self.manifest)
-                for file_id in parser_files['MacroParser']:
-                    block = FileBlock(self.manifest.files[file_id])
-                    parser.parse_file(block)
-                    # increment parsed path count for performance tracking
-                    self._perf_info.parsed_path_count = self._perf_info.parsed_path_count + 1
-            # Look at changed macros and update the macro.depends_on.macros
-            self.macro_depends_on()
+            self.load_and_parse_macros(project_parser_files)
+
+            # If we're partially parsing check that certain macros have not been changed
+            if self.partially_parsing and self.skip_partial_parsing_because_of_macros():
+                logger.info(
+                    "Change detected to override macro used during parsing. Starting full parse."
+                )
+                # Get new Manifest with original file records and move over the macros
+                self.manifest = self.new_manifest  # contains newly read files
+                project_parser_files = orig_project_parser_files
+                self.partially_parsing = False
+                self.load_and_parse_macros(project_parser_files)
+
             self._perf_info.load_macros_elapsed = (time.perf_counter() - start_load_macros)
 
             # Now that the macros are parsed, parse the rest of the files.
@@ -370,6 +373,24 @@ def load(self):
 
         return self.manifest
 
+    def load_and_parse_macros(self, project_parser_files):
+        for project in self.all_projects.values():
+            if project.project_name not in project_parser_files:
+                continue
+            parser_files = project_parser_files[project.project_name]
+            if 'MacroParser' not in parser_files:
+                continue
+            parser = MacroParser(project, self.manifest)
+            for file_id in parser_files['MacroParser']:
+                block = FileBlock(self.manifest.files[file_id])
+                parser.parse_file(block)
+                # increment parsed path count for performance tracking
+                self._perf_info.parsed_path_count = self._perf_info.parsed_path_count + 1
+
+        self.build_macro_resolver()
+        # Look at changed macros and update the macro.depends_on.macros
+        self.macro_depends_on()
+
     # Parse the files in the 'parser_files' dictionary, for parsers listed in
     # 'parser_types'
     def parse_project(
@@ -440,20 +461,23 @@ def parse_project(
             self._perf_info.parsed_path_count + total_parsed_path_count
         )
 
-    # Loop through macros in the manifest and statically parse
-    # the 'macro_sql' to find depends_on.macros
-    def macro_depends_on(self):
+    # This should only be called after the macros have been loaded
+    def build_macro_resolver(self):
         internal_package_names = get_adapter_package_names(
             self.root_project.credentials.type
         )
-        macro_resolver = MacroResolver(
+        self.macro_resolver = MacroResolver(
             self.manifest.macros,
             self.root_project.project_name,
             internal_package_names
         )
+
+    # Loop through macros in the manifest and statically parse
+    # the 'macro_sql' to find depends_on.macros
+    def macro_depends_on(self):
         macro_ctx = generate_macro_context(self.root_project)
         macro_namespace = TestMacroNamespace(
-            macro_resolver, {}, None, MacroStack(), []
+            self.macro_resolver, {}, None, MacroStack(), []
         )
         adapter = get_adapter(self.root_project)
         db_wrapper = ParseProvider().DatabaseWrapper(
@@ -472,7 +496,7 @@ def macro_depends_on(self):
                 package_name = macro.package_name
                 if '.' in macro_name:
                     package_name, macro_name = macro_name.split('.')
-                dep_macro_id = macro_resolver.get_macro_id(package_name, macro_name)
+                dep_macro_id = self.macro_resolver.get_macro_id(package_name, macro_name)
                 if dep_macro_id:
                     macro.depends_on.add_macro(dep_macro_id)  # will check for dupes
 
@@ -538,6 +562,21 @@ def is_partial_parsable(self, manifest: Manifest) -> Tuple[bool, Optional[str]]:
                     reparse_reason = ReparseReason.project_config_changed
         return valid, reparse_reason
 
+    def skip_partial_parsing_because_of_macros(self):
+        if not self.partial_parser:
+            return False
+        if self.partial_parser.deleted_special_override_macro:
+            return True
+        # Check for custom versions of these special macros
+        for macro_name in special_override_macros:
+            macro = self.macro_resolver.get_macro(None, macro_name)
+            if macro and macro.package_name != 'dbt':
+                if (macro.file_id in self.partial_parser.file_diff['changed'] or
+                        macro.file_id in self.partial_parser.file_diff['added']):
+                    # The file with the macro in it has changed
+                    return True
+        return False
+
     def read_manifest_for_partial_parse(self) -> Optional[Manifest]:
         if not flags.PARTIAL_PARSE:
             logger.debug('Partial parsing not enabled')
diff --git a/core/dbt/parser/partial.py b/core/dbt/parser/partial.py
--- a/core/dbt/parser/partial.py
+++ b/core/dbt/parser/partial.py
@@ -1,3 +1,4 @@
+from copy import deepcopy
 from typing import MutableMapping, Dict, List
 from dbt.contracts.graph.manifest import Manifest
 from dbt.contracts.files import (
@@ -32,10 +33,21 @@
 }
 
 
+# These macro names have special treatment in the ManifestLoader and
+# partial parsing. If they have changed we will skip partial parsing
+special_override_macros = [
+    'ref', 'source', 'config', 'generate_schema_name',
+    'generate_database_name', 'generate_alias_name'
+]
+
+
 # Partial parsing. Create a diff of files from saved manifest and current
 # files and produce a project_parser_file dictionary to drive parsing of
 # only the necessary changes.
 # Will produce a 'skip_parsing' method, and a project_parser_file dictionary
+# All file objects from the new manifest are deepcopied, because we need
+# to preserve an unchanged file object in case we need to drop back to a
+# a full parse (such as for certain macro changes)
 class PartialParsing:
     def __init__(self, saved_manifest: Manifest, new_files: MutableMapping[str, AnySourceFile]):
         self.saved_manifest = saved_manifest
@@ -47,6 +59,7 @@ def __init__(self, saved_manifest: Manifest, new_files: MutableMapping[str, AnyS
         self.macro_child_map: Dict[str, List[str]] = {}
         self.build_file_diff()
         self.processing_file = None
+        self.deleted_special_override_macro = False
 
     def skip_parsing(self):
         return (
@@ -168,7 +181,7 @@ def already_scheduled_for_parsing(self, source_file):
     # Add new files, including schema files
     def add_to_saved(self, file_id):
         # add file object to saved manifest.files
-        source_file = self.new_files[file_id]
+        source_file = deepcopy(self.new_files[file_id])
         if source_file.parse_file_type == ParseFileType.Schema:
             self.handle_added_schema_file(source_file)
         self.saved_files[file_id] = source_file
@@ -210,7 +223,7 @@ def delete_from_saved(self, file_id):
 
     # Updates for non-schema files
     def update_in_saved(self, file_id):
-        new_source_file = self.new_files[file_id]
+        new_source_file = deepcopy(self.new_files[file_id])
         old_source_file = self.saved_files[file_id]
 
         if new_source_file.parse_file_type in mssat_files:
@@ -238,7 +251,7 @@ def update_mssat_in_saved(self, new_source_file, old_source_file):
         # replace source_file in saved and add to parsing list
         file_id = new_source_file.file_id
         self.deleted_manifest.files[file_id] = old_source_file
-        self.saved_files[file_id] = new_source_file
+        self.saved_files[file_id] = deepcopy(new_source_file)
         self.add_to_pp_files(new_source_file)
         self.remove_node_in_saved(new_source_file, unique_id)
 
@@ -276,14 +289,14 @@ def update_macro_in_saved(self, new_source_file, old_source_file):
             return
         self.handle_macro_file_links(old_source_file, follow_references=True)
         file_id = new_source_file.file_id
-        self.saved_files[file_id] = new_source_file
+        self.saved_files[file_id] = deepcopy(new_source_file)
         self.add_to_pp_files(new_source_file)
 
     def update_doc_in_saved(self, new_source_file, old_source_file):
         if self.already_scheduled_for_parsing(old_source_file):
             return
         self.delete_doc_node(old_source_file)
-        self.saved_files[new_source_file.file_id] = new_source_file
+        self.saved_files[new_source_file.file_id] = deepcopy(new_source_file)
         self.add_to_pp_files(new_source_file)
 
     def remove_mssat_file(self, source_file):
@@ -313,7 +326,7 @@ def schedule_nodes_for_parsing(self, unique_ids):
                     source_file = self.saved_files[file_id]
                     self.remove_mssat_file(source_file)
                     # content of non-schema files is only in new files
-                    self.saved_files[file_id] = self.new_files[file_id]
+                    self.saved_files[file_id] = deepcopy(self.new_files[file_id])
                     self.add_to_pp_files(self.saved_files[file_id])
             elif unique_id in self.saved_manifest.sources:
                 source = self.saved_manifest.sources[unique_id]
@@ -346,14 +359,25 @@ def schedule_nodes_for_parsing(self, unique_ids):
                 if file_id in self.saved_files and file_id not in self.file_diff['deleted']:
                     source_file = self.saved_files[file_id]
                     self.delete_macro_file(source_file)
-                    self.saved_files[file_id] = self.new_files[file_id]
+                    self.saved_files[file_id] = deepcopy(self.new_files[file_id])
                     self.add_to_pp_files(self.saved_files[file_id])
 
     def delete_macro_file(self, source_file, follow_references=False):
+        self.check_for_special_deleted_macros(source_file)
         self.handle_macro_file_links(source_file, follow_references)
         file_id = source_file.file_id
         self.deleted_manifest.files[file_id] = self.saved_files.pop(file_id)
 
+    def check_for_special_deleted_macros(self, source_file):
+        for unique_id in source_file.macros:
+            if unique_id in self.saved_manifest.macros:
+                package_name = unique_id.split('.')[1]
+                if package_name == 'dbt':
+                    continue
+                macro = self.saved_manifest.macros[unique_id]
+                if macro.name in special_override_macros:
+                    self.deleted_special_override_macro = True
+
     def recursively_gather_macro_references(self, macro_unique_id, referencing_nodes):
         for unique_id in self.macro_child_map[macro_unique_id]:
             if unique_id in referencing_nodes:
@@ -434,7 +458,7 @@ def schedule_macro_nodes_for_parsing(self, unique_ids):
                         source_file = self.saved_files[file_id]
                         self.remove_mssat_file(source_file)
                         # content of non-schema files is only in new files
-                        self.saved_files[file_id] = self.new_files[file_id]
+                        self.saved_files[file_id] = deepcopy(self.new_files[file_id])
                         self.add_to_pp_files(self.saved_files[file_id])
             elif unique_id in self.saved_manifest.macros:
                 macro = self.saved_manifest.macros[unique_id]
@@ -442,7 +466,7 @@ def schedule_macro_nodes_for_parsing(self, unique_ids):
                 if file_id in self.saved_files and file_id not in self.file_diff['deleted']:
                     source_file = self.saved_files[file_id]
                     self.delete_macro_file(source_file)
-                    self.saved_files[file_id] = self.new_files[file_id]
+                    self.saved_files[file_id] = deepcopy(self.new_files[file_id])
                     self.add_to_pp_files(self.saved_files[file_id])
 
     def delete_doc_node(self, source_file):
@@ -460,7 +484,7 @@ def delete_doc_node(self, source_file):
     # Changed schema files
     def change_schema_file(self, file_id):
         saved_schema_file = self.saved_files[file_id]
-        new_schema_file = self.new_files[file_id]
+        new_schema_file = deepcopy(self.new_files[file_id])
         saved_yaml_dict = saved_schema_file.dict_from_yaml
         new_yaml_dict = new_schema_file.dict_from_yaml
         if 'version' in new_yaml_dict:
@@ -639,7 +663,7 @@ def delete_schema_mssa_links(self, schema_file, dict_key, elem):
                 file_id = node.file_id
                 # need to copy new file to saved files in order to get content
                 if file_id in self.new_files:
-                    self.saved_files[file_id] = self.new_files[file_id]
+                    self.saved_files[file_id] = deepcopy(self.new_files[file_id])
                 if self.saved_files[file_id]:
                     source_file = self.saved_files[file_id]
                     self.add_to_pp_files(source_file)
@@ -687,7 +711,7 @@ def delete_schema_macro_patch(self, schema_file, macro):
             self.deleted_manifest.macros[macro_unique_id] = macro
             macro_file_id = macro.file_id
             if macro_file_id in self.new_files:
-                self.saved_files[macro_file_id] = self.new_files[macro_file_id]
+                self.saved_files[macro_file_id] = deepcopy(self.new_files[macro_file_id])
                 self.add_to_pp_files(self.saved_files[macro_file_id])
 
     # exposures are created only from schema files, so just delete


Test patch:
diff --git a/test/integration/068_partial_parsing_tests/extra-files/gsm_override.sql b/test/integration/068_partial_parsing_tests/extra-files/gsm_override.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/068_partial_parsing_tests/extra-files/gsm_override.sql
@@ -0,0 +1,6 @@
+- custom macro
+{% macro generate_schema_name(schema_name, node) %}
+
+    {{ schema_name }}_{{ target.schema }}
+
+{% endmacro %}
diff --git a/test/integration/068_partial_parsing_tests/extra-files/gsm_override2.sql b/test/integration/068_partial_parsing_tests/extra-files/gsm_override2.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/068_partial_parsing_tests/extra-files/gsm_override2.sql
@@ -0,0 +1,6 @@
+- custom macro xxxx
+{% macro generate_schema_name(schema_name, node) %}
+
+    {{ schema_name }}_{{ target.schema }}
+
+{% endmacro %}
diff --git a/test/integration/068_partial_parsing_tests/extra-files/ref_override.sql b/test/integration/068_partial_parsing_tests/extra-files/ref_override.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/068_partial_parsing_tests/extra-files/ref_override.sql
@@ -0,0 +1,4 @@
+- Macro to override ref
+{% macro ref(modelname) %}
+{% do return(builtins.ref(modelname)) %}
+{% endmacro %}
diff --git a/test/integration/068_partial_parsing_tests/extra-files/ref_override2.sql b/test/integration/068_partial_parsing_tests/extra-files/ref_override2.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/068_partial_parsing_tests/extra-files/ref_override2.sql
@@ -0,0 +1,4 @@
+- Macro to override ref xxxx
+{% macro ref(modelname) %}
+{% do return(builtins.ref(modelname)) %}
+{% endmacro %}
diff --git a/test/integration/068_partial_parsing_tests/test_partial_parsing.py b/test/integration/068_partial_parsing_tests/test_partial_parsing.py
--- a/test/integration/068_partial_parsing_tests/test_partial_parsing.py
+++ b/test/integration/068_partial_parsing_tests/test_partial_parsing.py
@@ -2,6 +2,7 @@
 from dbt.contracts.graph.manifest import Manifest
 from dbt.contracts.files import ParseFileType
 from dbt.contracts.results import TestStatus
+from dbt.parser.partial import special_override_macros
 from test.integration.base import DBTIntegrationTest, use_profile, normalize
 import shutil
 import os
@@ -436,6 +437,10 @@ def project_config(self):
     def tearDown(self):
         if os.path.exists(normalize('macros-macros/custom_schema_tests.sql')):
             os.remove(normalize('macros-macros/custom_schema_tests.sql'))
+        if os.path.exists(normalize('macros-macros/ref_override.sql')):
+            os.remove(normalize('macros-macros/ref_override.sql'))
+        if os.path.exists(normalize('macros-macros/gsm_override.sql')):
+            os.remove(normalize('macros-macros/gsm_override.sql'))
 
     @use_profile('postgres')
     def test_postgres_nested_macros(self):
@@ -468,3 +473,38 @@ def test_postgres_nested_macros(self):
         self.assertEqual(results[1].status, TestStatus.Fail)
         self.assertEqual(results[1].node.config.severity, 'ERROR')
 
+    @use_profile('postgres')
+    def test_postgres_skip_macros(self):
+        expected_special_override_macros = [
+            'ref', 'source', 'config', 'generate_schema_name',
+            'generate_database_name', 'generate_alias_name'
+        ]
+        self.assertEqual(special_override_macros, expected_special_override_macros)
+
+        # initial run so we have a msgpack file
+        results = self.run_dbt()
+
+        # add a new ref override macro
+        shutil.copyfile('extra-files/ref_override.sql', 'macros-macros/ref_override.sql')
+        results, log_output = self.run_dbt_and_capture(['--partial-parse', 'run'])
+        self.assertTrue('Starting full parse.' in log_output)
+
+        # modify a ref override macro
+        shutil.copyfile('extra-files/ref_override2.sql', 'macros-macros/ref_override.sql')
+        results, log_output = self.run_dbt_and_capture(['--partial-parse', 'run'])
+        self.assertTrue('Starting full parse.' in log_output)
+
+        # remove a ref override macro
+        os.remove(normalize('macros-macros/ref_override.sql'))
+        results, log_output = self.run_dbt_and_capture(['--partial-parse', 'run'])
+        self.assertTrue('Starting full parse.' in log_output)
+
+        # custom generate_schema_name macro
+        shutil.copyfile('extra-files/gsm_override.sql', 'macros-macros/gsm_override.sql')
+        results, log_output = self.run_dbt_and_capture(['--partial-parse', 'run'])
+        self.assertTrue('Starting full parse.' in log_output)
+
+        # change generate_schema_name macro
+        shutil.copyfile('extra-files/gsm_override2.sql', 'macros-macros/gsm_override.sql')
+        results, log_output = self.run_dbt_and_capture(['--partial-parse', 'run'])
+        self.assertTrue('Starting full parse.' in log_output)