Repository: dbt-labs/dbt-core

Issue:
[CT-2337] [Bug] [Partial parsing] Error in `merge_patch` when `version: 2` missing from yaml file
I think there might be a bug with partial parsing if a yaml ("schema") file is missing `version: 2`, per https://github.com/dbt-labs/dbt-core/pull/7060!

```yaml
models:
  - name: any_model
    description: My favorite model
```
```sql
-- models/any_model.sql
select 1 as id
```
```
$ dbt parse
00:32:51  Running with dbt=1.5.0-b4
00:32:51  Unable to do partial parsing because an error occurred. Switching to full reparse.
00:32:52  Performance info: target/perf_info.json
```
In `logs/dbt.log`:
```
PP exception info: {'code': '    schema_file.pp_dict = {"version": schema_file.dict_from_yaml["version"]}', 'parse_file_type': 'model', 'exception': "KeyError: 'version'", 'location': 'line 810 in merge_patch', 'traceback': 'Traceback (most recent call last):
  File "/Users/jerco/dev/product/dbt-core/core/dbt/parser/manifest.py", line 293, in load
    project_parser_files = self.partial_parser.get_parsing_files()
  File "/Users/jerco/dev/product/dbt-core/core/dbt/parser/partial.py", line 188, in get_parsing_files
    self.update_in_saved(file_id)
  File "/Users/jerco/dev/product/dbt-core/core/dbt/parser/partial.py", line 288, in update_in_saved
    self.update_mssat_in_saved(new_source_file, old_source_file)
  File "/Users/jerco/dev/product/dbt-core/core/dbt/parser/partial.py", line 316, in update_mssat_in_saved
    self.remove_node_in_saved(new_source_file, unique_id)
  File "/Users/jerco/dev/product/dbt-core/core/dbt/parser/partial.py", line 350, in remove_node_in_saved
    self.merge_patch(schema_file, dict_key, elem_patch)
  File "/Users/jerco/dev/product/dbt-core/core/dbt/parser/partial.py", line 810, in merge_patch
    schema_file.pp_dict = {"version": schema_file.dict_from_yaml["version"]}
KeyError: \'version\'
'}
```


Discussion:

Resolving patch:
diff --git a/core/dbt/parser/partial.py b/core/dbt/parser/partial.py
--- a/core/dbt/parser/partial.py
+++ b/core/dbt/parser/partial.py
@@ -577,13 +577,7 @@ def change_schema_file(self, file_id):
         new_schema_file = deepcopy(self.new_files[file_id])
         saved_yaml_dict = saved_schema_file.dict_from_yaml
         new_yaml_dict = new_schema_file.dict_from_yaml
-        if "version" in new_yaml_dict:
-            # despite the fact that this goes in the saved_schema_file, it
-            # should represent the new yaml dictionary, and should produce
-            # an error if the updated yaml file doesn't have a version
-            saved_schema_file.pp_dict = {"version": new_yaml_dict["version"]}
-        else:
-            saved_schema_file.pp_dict = {}
+        saved_schema_file.pp_dict = {}
         self.handle_schema_file_changes(saved_schema_file, saved_yaml_dict, new_yaml_dict)
 
         # copy from new schema_file to saved_schema_file to preserve references
@@ -806,8 +800,8 @@ def get_diff_for(self, key, saved_yaml_dict, new_yaml_dict):
 
     # Merge a patch file into the pp_dict in a schema file
     def merge_patch(self, schema_file, key, patch):
-        if not schema_file.pp_dict:
-            schema_file.pp_dict = {"version": schema_file.dict_from_yaml["version"]}
+        if schema_file.pp_dict is None:
+            schema_file.pp_dict = {}
         pp_dict = schema_file.pp_dict
         if key not in pp_dict:
             pp_dict[key] = [patch]
diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py
--- a/core/dbt/parser/schemas.py
+++ b/core/dbt/parser/schemas.py
@@ -502,9 +502,9 @@ def parse_tests(self, block: TestBlock) -> None:
 
     def parse_file(self, block: FileBlock, dct: Dict = None) -> None:
         assert isinstance(block.file, SchemaSourceFile)
-        if not dct:
-            dct = yaml_from_file(block.file)
 
+        # If partially parsing, dct should be from pp_dict, otherwise
+        # dict_from_yaml
         if dct:
             # contains the FileBlock and the data (dictionary)
             yaml_block = YamlBlock.from_file_block(block, dct)


Test patch:
diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py
--- a/test/unit/test_parser.py
+++ b/test/unit/test_parser.py
@@ -45,10 +45,15 @@
     SourceParser,
     AnalysisPatchParser,
     MacroPatchParser,
+    yaml_from_file,
 )
 from dbt.parser.search import FileBlock
 from dbt.parser.sources import SourcePatcher
 from .utils import config_from_parts_or_dicts, normalize, generate_name_macros, MockNode
+from dbt.flags import set_from_args
+from argparse import Namespace
+
+set_from_args(Namespace(WARN_ERROR=False), None)
 
 
 def get_abs_os_path(unix_path):
@@ -210,7 +215,6 @@ def assertEqualNodes(node_one, node_two):
 
 
 SINGLE_TABLE_SOURCE = """
-version: 2
 sources:
     - name: my_source
       tables:
@@ -218,7 +222,6 @@ def assertEqualNodes(node_one, node_two):
 """
 
 SINGLE_TABLE_SOURCE_TESTS = """
-version: 2
 sources:
     - name: my_source
       tables:
@@ -234,7 +237,6 @@ def assertEqualNodes(node_one, node_two):
 """
 
 SINGLE_TABLE_MODEL_TESTS = """
-version: 2
 models:
     - name: my_model
       description: A description of my model
@@ -251,7 +253,6 @@ def assertEqualNodes(node_one, node_two):
 """
 
 SINGLE_TABLE_SOURCE_PATCH = """
-version: 2
 sources:
   - name: my_source
     overrides: snowplow
@@ -310,7 +311,8 @@ def test__read_basic_source(self):
 
     def test__parse_basic_source(self):
         block = self.file_block_for(SINGLE_TABLE_SOURCE, "test_one.yml")
-        self.parser.parse_file(block)
+        dct = yaml_from_file(block.file)
+        self.parser.parse_file(block, dct)
         self.assert_has_manifest_lengths(self.parser.manifest, sources=1)
         src = list(self.parser.manifest.sources.values())[0]
         assert isinstance(src, UnpatchedSourceDefinition)
@@ -342,7 +344,8 @@ def test__read_basic_source_tests(self):
     def test__parse_basic_source_tests(self):
         block = self.file_block_for(SINGLE_TABLE_SOURCE_TESTS, "test_one.yml")
         self.parser.manifest.files[block.file.file_id] = block.file
-        self.parser.parse_file(block)
+        dct = yaml_from_file(block.file)
+        self.parser.parse_file(block, dct)
         self.assertEqual(len(self.parser.manifest.nodes), 0)
         self.assertEqual(len(self.parser.manifest.sources), 1)
         src = list(self.parser.manifest.sources.values())[0]
@@ -423,14 +426,16 @@ def setUp(self):
 
     def test__read_basic_model_tests(self):
         block = self.yaml_block_for(SINGLE_TABLE_MODEL_TESTS, "test_one.yml")
-        self.parser.parse_file(block)
+        dct = yaml_from_file(block.file)
+        self.parser.parse_file(block, dct)
         self.assertEqual(len(list(self.parser.manifest.sources)), 0)
         self.assertEqual(len(list(self.parser.manifest.nodes)), 4)
 
     def test__parse_basic_model_tests(self):
         block = self.file_block_for(SINGLE_TABLE_MODEL_TESTS, "test_one.yml")
         self.parser.manifest.files[block.file.file_id] = block.file
-        self.parser.parse_file(block)
+        dct = yaml_from_file(block.file)
+        self.parser.parse_file(block, dct)
         self.assert_has_manifest_lengths(self.parser.manifest, nodes=4)
 
         all_nodes = sorted(self.parser.manifest.nodes.values(), key=lambda n: n.unique_id)
diff --git a/tests/functional/partial_parsing/fixtures.py b/tests/functional/partial_parsing/fixtures.py
--- a/tests/functional/partial_parsing/fixtures.py
+++ b/tests/functional/partial_parsing/fixtures.py
@@ -26,7 +26,6 @@
 """
 
 local_dependency__models__schema_yml = """
-version: 2
 sources:
   - name: seed_source
     schema: "{{ var('schema_override', target.schema) }}"
@@ -56,12 +55,10 @@
 """
 
 empty_schema_with_version_yml = """
-version: 2
 
 """
 
 schema_sources5_yml = """
-version: 2
 
 sources:
   - name: seed_sources
@@ -135,7 +132,6 @@
 """
 
 schema_sources4_yml = """
-version: 2
 
 sources:
   - name: seed_sources
@@ -169,7 +165,6 @@
 """
 
 env_var_schema_yml = """
-version: 2
 
 models:
     - name: model_one
@@ -189,7 +184,6 @@
 """
 
 schema_models_c_yml = """
-version: 2
 
 sources:
   - name: seed_source
@@ -207,7 +201,6 @@
 """
 
 env_var_sources_yml = """
-version: 2
 sources:
   - name: seed_sources
     schema: "{{ target.schema }}"
@@ -258,7 +251,6 @@
 """
 
 schema_sources1_yml = """
-version: 2
 sources:
   - name: seed_sources
     schema: "{{ target.schema }}"
@@ -279,7 +271,6 @@
 """
 
 schema_sources3_yml = """
-version: 2
 
 sources:
   - name: seed_sources
@@ -316,7 +307,6 @@
 """
 
 schema_sources2_yml = """
-version: 2
 
 sources:
   - name: seed_sources
@@ -354,7 +344,6 @@
 """
 
 my_metric_yml = """
-version: 2
 metrics:
   - name: new_customers
     label: New Customers
@@ -381,7 +370,6 @@
 """
 
 env_var_schema2_yml = """
-version: 2
 
 models:
     - name: model_one
@@ -416,7 +404,6 @@
 """
 
 env_var_schema3_yml = """
-version: 2
 
 models:
     - name: model_one
@@ -441,7 +428,6 @@
 """
 
 env_var_metrics_yml = """
-version: 2
 
 metrics:
 
@@ -598,7 +584,6 @@
 """
 
 people_metrics_yml = """
-version: 2
 
 metrics:
 
@@ -665,7 +650,6 @@
 """
 
 models_schema2b_yml = """
-version: 2
 
 models:
     - name: model_one
@@ -680,7 +664,6 @@
 """
 
 env_var_macros_yml = """
-version: 2
 macros:
     - name: do_something
       description: "This is a test macro"
@@ -691,7 +674,6 @@
 """
 
 models_schema4_yml = """
-version: 2
 
 models:
     - name: model_one
@@ -713,7 +695,6 @@
 """
 
 generic_test_schema_yml = """
-version: 2
 
 models:
   - name: orders
@@ -754,7 +735,6 @@
 """
 
 macros_yml = """
-version: 2
 macros:
     - name: do_something
       description: "This is a test macro"
@@ -773,7 +753,6 @@
 """
 
 models_schema2_yml = """
-version: 2
 
 models:
     - name: model_one
@@ -798,7 +777,6 @@
 """
 
 models_schema3_yml = """
-version: 2
 
 models:
     - name: model_one
@@ -843,7 +821,6 @@
 """
 
 env_var_model_test_yml = """
-version: 2
 models:
   - name: model_color
     columns:
@@ -879,7 +856,6 @@
 """
 
 models_schema1_yml = """
-version: 2
 
 models:
     - name: model_one
@@ -889,7 +865,6 @@
 
 macros_schema_yml = """
 
-version: 2
 
 models:
     - name: model_a
@@ -944,7 +919,6 @@
 """
 
 models_schema4b_yml = """
-version: 2
 
 models:
     - name: model_one
@@ -970,7 +944,6 @@
 """
 
 people_metrics2_yml = """
-version: 2
 
 metrics:
 
@@ -1004,7 +977,6 @@
 """
 
 generic_schema_yml = """
-version: 2
 
 models:
   - name: orders
@@ -1018,7 +990,6 @@
 
 
 groups_schema_yml_one_group = """
-version: 2
 
 groups:
   - name: test_group
@@ -1032,7 +1003,6 @@
 
 
 groups_schema_yml_two_groups = """
-version: 2
 
 groups:
   - name: test_group
@@ -1048,7 +1018,6 @@
 """
 
 groups_schema_yml_one_group_model_in_group2 = """
-version: 2
 
 groups:
   - name: test_group
@@ -1063,7 +1032,6 @@
 """
 
 groups_schema_yml_two_groups_edited = """
-version: 2
 
 groups:
   - name: test_group
@@ -1127,7 +1095,6 @@
 """
 
 people_metrics3_yml = """
-version: 2
 
 metrics: