Repository: dbt-labs/dbt-core

Issue:
[CORE-19] Partial parsing: check for generic or singular test node
Error encountered from logged events:

AttributeError: 'SourceFile' object has no attribute 'get_key_and_name_for_test'

On line:
    (key, name) = schema_file.get_key_and_name_for_test(node.unique_id)


Discussion:

Resolving patch:
diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py
--- a/core/dbt/contracts/graph/parsed.py
+++ b/core/dbt/contracts/graph/parsed.py
@@ -431,6 +431,10 @@ class ParsedSingularTestNode(ParsedNode):
     # refactor the various configs.
     config: TestConfig = field(default_factory=TestConfig)  # type: ignore
 
+    @property
+    def test_node_type(self):
+        return 'singular'
+
 
 @dataclass
 class ParsedGenericTestNode(ParsedNode, HasTestMetadata):
@@ -452,6 +456,10 @@ def same_contents(self, other) -> bool:
             True
         )
 
+    @property
+    def test_node_type(self):
+        return 'generic'
+
 
 @dataclass
 class IntermediateSnapshotNode(ParsedNode):
diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py
--- a/core/dbt/parser/manifest.py
+++ b/core/dbt/parser/manifest.py
@@ -246,7 +246,7 @@ def load(self):
                     project_parser_files = self.partial_parser.get_parsing_files()
                     self.partially_parsing = True
                     self.manifest = self.saved_manifest
-                except Exception:
+                except Exception as exc:
                     # pp_files should still be the full set and manifest is new manifest,
                     # since get_parsing_files failed
                     fire_event(PartialParsingFullReparseBecauseOfError())
@@ -284,6 +284,9 @@ def load(self):
                         exc_info['full_reparse_reason'] = ReparseReason.exception
                         dbt.tracking.track_partial_parser(exc_info)
 
+                    if os.environ.get('DBT_PP_TEST'):
+                        raise exc
+
         if self.manifest._parsing_info is None:
             self.manifest._parsing_info = ParsingInfo()
 
diff --git a/core/dbt/parser/partial.py b/core/dbt/parser/partial.py
--- a/core/dbt/parser/partial.py
+++ b/core/dbt/parser/partial.py
@@ -315,7 +315,7 @@ def remove_node_in_saved(self, source_file, unique_id):
         if node.patch_path:
             file_id = node.patch_path
             # it might be changed...  then what?
-            if file_id not in self.file_diff['deleted']:
+            if file_id not in self.file_diff['deleted'] and file_id in self.saved_files:
                 # schema_files should already be updated
                 schema_file = self.saved_files[file_id]
                 dict_key = parse_file_type_to_key[source_file.parse_file_type]
@@ -375,7 +375,7 @@ def schedule_nodes_for_parsing(self, unique_ids):
         for unique_id in unique_ids:
             if unique_id in self.saved_manifest.nodes:
                 node = self.saved_manifest.nodes[unique_id]
-                if node.resource_type == NodeType.Test:
+                if node.resource_type == NodeType.Test and node.test_node_type == 'generic':
                     # test nodes are handled separately. Must be removed from schema file
                     continue
                 file_id = node.file_id
@@ -435,7 +435,9 @@ def delete_macro_file(self, source_file, follow_references=False):
         self.check_for_special_deleted_macros(source_file)
         self.handle_macro_file_links(source_file, follow_references)
         file_id = source_file.file_id
-        self.deleted_manifest.files[file_id] = self.saved_files.pop(file_id)
+        # It's not clear when this file_id would not exist in saved_files
+        if file_id in self.saved_files:
+            self.deleted_manifest.files[file_id] = self.saved_files.pop(file_id)
 
     def check_for_special_deleted_macros(self, source_file):
         for unique_id in source_file.macros:
@@ -498,7 +500,9 @@ def schedule_macro_nodes_for_parsing(self, unique_ids):
         for unique_id in unique_ids:
             if unique_id in self.saved_manifest.nodes:
                 node = self.saved_manifest.nodes[unique_id]
-                if node.resource_type == NodeType.Test:
+                # Both generic tests from yaml files and singular tests have NodeType.Test
+                # so check for generic test.
+                if node.resource_type == NodeType.Test and node.test_node_type == 'generic':
                     schema_file_id = node.file_id
                     schema_file = self.saved_manifest.files[schema_file_id]
                     (key, name) = schema_file.get_key_and_name_for_test(node.unique_id)
@@ -670,8 +674,8 @@ def handle_schema_file_changes(self, schema_file, saved_yaml_dict, new_yaml_dict
                     continue
                 elem = self.get_schema_element(new_yaml_dict[dict_key], name)
                 if elem:
-                    self.delete_schema_macro_patch(schema_file, macro)
-                    self.merge_patch(schema_file, dict_key, macro)
+                    self.delete_schema_macro_patch(schema_file, elem)
+                    self.merge_patch(schema_file, dict_key, elem)
 
         # exposures
         dict_key = 'exposures'


Test patch:
diff --git a/test/integration/068_partial_parsing_tests/test-files/my_test.sql b/test/integration/068_partial_parsing_tests/test-files/my_test.sql
--- a/test/integration/068_partial_parsing_tests/test-files/my_test.sql
+++ b/test/integration/068_partial_parsing_tests/test-files/my_test.sql
@@ -1,2 +1,2 @@
 select
-   * from {{ ref('customers') }} where customer_id > 100
+   * from {{ ref('customers') }} where first_name = '{{ macro_something() }}'
diff --git a/test/integration/068_partial_parsing_tests/test-files/test-macro.sql b/test/integration/068_partial_parsing_tests/test-files/test-macro.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/068_partial_parsing_tests/test-files/test-macro.sql
@@ -0,0 +1,5 @@
+{% macro macro_something() %}
+
+    {% do return('macro_something') %}
+
+{% endmacro %}
diff --git a/test/integration/068_partial_parsing_tests/test-files/test-macro2.sql b/test/integration/068_partial_parsing_tests/test-files/test-macro2.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/068_partial_parsing_tests/test-files/test-macro2.sql
@@ -0,0 +1,5 @@
+{% macro macro_something() %}
+
+    {% do return('some_name') %}
+
+{% endmacro %}
diff --git a/test/integration/068_partial_parsing_tests/test_partial_parsing.py b/test/integration/068_partial_parsing_tests/test_partial_parsing.py
--- a/test/integration/068_partial_parsing_tests/test_partial_parsing.py
+++ b/test/integration/068_partial_parsing_tests/test_partial_parsing.py
@@ -46,6 +46,7 @@ def setup_directories(self):
         os.mkdir(os.path.join(self.test_root_dir, 'macros'))
         os.mkdir(os.path.join(self.test_root_dir, 'analyses'))
         os.mkdir(os.path.join(self.test_root_dir, 'snapshots'))
+        os.environ['DBT_PP_TEST'] = 'true'
 
 
 
@@ -332,6 +333,7 @@ def test_postgres_pp_sources(self):
         results = self.run_dbt(["--partial-parse", "run"])
 
         # Add a data test
+        self.copy_file('test-files/test-macro.sql', 'macros/test-macro.sql')
         self.copy_file('test-files/my_test.sql', 'tests/my_test.sql')
         results = self.run_dbt(["--partial-parse", "test"])
         manifest = get_manifest()
@@ -339,6 +341,11 @@ def test_postgres_pp_sources(self):
         test_id = 'test.test.my_test'
         self.assertIn(test_id, manifest.nodes)
 
+        # Change macro that data test depends on
+        self.copy_file('test-files/test-macro2.sql', 'macros/test-macro.sql')
+        results = self.run_dbt(["--partial-parse", "test"])
+        manifest = get_manifest()
+
         # Add an analysis
         self.copy_file('test-files/my_analysis.sql', 'analyses/my_analysis.sql')
         results = self.run_dbt(["--partial-parse", "run"])
diff --git a/test/integration/068_partial_parsing_tests/test_pp_docs.py b/test/integration/068_partial_parsing_tests/test_pp_docs.py
--- a/test/integration/068_partial_parsing_tests/test_pp_docs.py
+++ b/test/integration/068_partial_parsing_tests/test_pp_docs.py
@@ -37,6 +37,7 @@ def setup_directories(self):
         os.mkdir(os.path.join(self.test_root_dir, 'macros'))
         os.mkdir(os.path.join(self.test_root_dir, 'analyses'))
         os.mkdir(os.path.join(self.test_root_dir, 'snapshots'))
+        os.environ['DBT_PP_TEST'] = 'true'
 
 
     @use_profile('postgres')
diff --git a/test/integration/068_partial_parsing_tests/test_pp_metrics.py b/test/integration/068_partial_parsing_tests/test_pp_metrics.py
--- a/test/integration/068_partial_parsing_tests/test_pp_metrics.py
+++ b/test/integration/068_partial_parsing_tests/test_pp_metrics.py
@@ -45,6 +45,7 @@ def setup_directories(self):
         os.mkdir(os.path.join(self.test_root_dir, 'macros'))
         os.mkdir(os.path.join(self.test_root_dir, 'analyses'))
         os.mkdir(os.path.join(self.test_root_dir, 'snapshots'))
+        os.environ['DBT_PP_TEST'] = 'true'
 
 
 
diff --git a/test/integration/068_partial_parsing_tests/test_pp_vars.py b/test/integration/068_partial_parsing_tests/test_pp_vars.py
--- a/test/integration/068_partial_parsing_tests/test_pp_vars.py
+++ b/test/integration/068_partial_parsing_tests/test_pp_vars.py
@@ -41,7 +41,7 @@ def setup_directories(self):
         os.mkdir(os.path.join(self.test_root_dir, 'tests'))
         os.mkdir(os.path.join(self.test_root_dir, 'macros'))
         os.mkdir(os.path.join(self.test_root_dir, 'seeds'))
-
+        os.environ['DBT_PP_TEST'] = 'true'
 
 
 class EnvVarTest(BasePPTest):