Repository: dbt-labs/dbt-core

Issue:
[LMA regression] Support custom snapshot strategies
A regression was introduced in `dev/louisa-may-alcott` which prevents custom snapshot strategies from working. I think this has to do with overzealous validation for the snapshot config.

```
Running with dbt=0.15.0-a1
Encountered an error:
Compilation Error in snapshot my_snaphshot_2 (snapshots/dir/my_snapshot.sql)
  at path ['config']['strategy']: 'check_fixed' is not one of ['timestamp', 'check']
```

This field probably should only be validated as a string -- we have a specific runtime error for missing/invalid strategies.

Nice looking error message though - pretty informative! :D 


Discussion:

Resolving patch:
diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py
--- a/core/dbt/contracts/graph/parsed.py
+++ b/core/dbt/contracts/graph/parsed.py
@@ -288,7 +288,7 @@ def __init__(
 
 @dataclass(init=False)
 class GenericSnapshotConfig(_SnapshotConfig):
-    strategy: SnapshotStrategy
+    strategy: str
 
     def __init__(self, strategy: str, **kwargs) -> None:
         self.strategy = strategy
@@ -297,13 +297,13 @@ def __init__(self, strategy: str, **kwargs) -> None:
 
 @dataclass(init=False)
 class TimestampSnapshotConfig(_SnapshotConfig):
-    strategy: SnapshotStrategy = field(metadata={
-        'restrict': [SnapshotStrategy.Timestamp]
+    strategy: str = field(metadata={
+        'restrict': [str(SnapshotStrategy.Timestamp)]
     })
     updated_at: str
 
     def __init__(
-        self, strategy: SnapshotStrategy, updated_at: str, **kwargs
+        self, strategy: str, updated_at: str, **kwargs
     ) -> None:
         self.strategy = strategy
         self.updated_at = updated_at
@@ -312,8 +312,8 @@ def __init__(
 
 @dataclass(init=False)
 class CheckSnapshotConfig(_SnapshotConfig):
-    strategy: SnapshotStrategy = field(metadata={
-        'restrict': [SnapshotStrategy.Check]
+    strategy: str = field(metadata={
+        'restrict': [str(SnapshotStrategy.Check)]
     })
     # TODO: is there a way to get this to accept tuples of strings? Adding
     # `Tuple[str, ...]` to the list of types results in this:
@@ -325,7 +325,7 @@ class CheckSnapshotConfig(_SnapshotConfig):
     check_cols: Union[All, List[str]]
 
     def __init__(
-        self, strategy: SnapshotStrategy, check_cols: Union[All, List[str]],
+        self, strategy: str, check_cols: Union[All, List[str]],
         **kwargs
     ) -> None:
         self.strategy = strategy


Test patch:
diff --git a/test/integration/004_simple_snapshot_test/custom-snapshot-macros/custom.sql b/test/integration/004_simple_snapshot_test/custom-snapshot-macros/custom.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/004_simple_snapshot_test/custom-snapshot-macros/custom.sql
@@ -0,0 +1,18 @@
+{# A "custom" strategy that's really just the timestamp one #}
+{% macro snapshot_custom_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}
+    {% set primary_key = config['unique_key'] %}
+    {% set updated_at = config['updated_at'] %}
+
+    {% set row_changed_expr -%}
+        ({{ snapshotted_rel }}.{{ updated_at }} < {{ current_rel }}.{{ updated_at }})
+    {%- endset %}
+
+    {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}
+
+    {% do return({
+        "unique_key": primary_key,
+        "updated_at": updated_at,
+        "row_changed": row_changed_expr,
+        "scd_id": scd_id_expr
+    }) %}
+{% endmacro %}
diff --git a/test/integration/004_simple_snapshot_test/test-snapshots-pg-custom-invalid/snapshot.sql b/test/integration/004_simple_snapshot_test/test-snapshots-pg-custom-invalid/snapshot.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/004_simple_snapshot_test/test-snapshots-pg-custom-invalid/snapshot.sql
@@ -0,0 +1,14 @@
+{% snapshot snapshot_actual %}
+    {# this custom strategy doesn't exist  in the 'dbt' package #}
+    {{
+        config(
+            target_database=var('target_database', database),
+            target_schema=var('target_schema', schema),
+            unique_key='id || ' ~ "'-'" ~ ' || first_name',
+            strategy='dbt.custom',
+            updated_at='updated_at',
+        )
+    }}
+    select * from {{target.database}}.{{target.schema}}.seed
+
+{% endsnapshot %}
diff --git a/test/integration/004_simple_snapshot_test/test-snapshots-pg-custom-namespaced/snapshot.sql b/test/integration/004_simple_snapshot_test/test-snapshots-pg-custom-namespaced/snapshot.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/004_simple_snapshot_test/test-snapshots-pg-custom-namespaced/snapshot.sql
@@ -0,0 +1,14 @@
+{% snapshot snapshot_actual %}
+
+    {{
+        config(
+            target_database=var('target_database', database),
+            target_schema=var('target_schema', schema),
+            unique_key='id || ' ~ "'-'" ~ ' || first_name',
+            strategy='test.custom',
+            updated_at='updated_at',
+        )
+    }}
+    select * from {{target.database}}.{{target.schema}}.seed
+
+{% endsnapshot %}
diff --git a/test/integration/004_simple_snapshot_test/test-snapshots-pg-custom/snapshot.sql b/test/integration/004_simple_snapshot_test/test-snapshots-pg-custom/snapshot.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/004_simple_snapshot_test/test-snapshots-pg-custom/snapshot.sql
@@ -0,0 +1,14 @@
+{% snapshot snapshot_actual %}
+
+    {{
+        config(
+            target_database=var('target_database', database),
+            target_schema=var('target_schema', schema),
+            unique_key='id || ' ~ "'-'" ~ ' || first_name',
+            strategy='custom',
+            updated_at='updated_at',
+        )
+    }}
+    select * from {{target.database}}.{{target.schema}}.seed
+
+{% endsnapshot %}
diff --git a/test/integration/004_simple_snapshot_test/test_simple_snapshot.py b/test/integration/004_simple_snapshot_test/test_simple_snapshot.py
--- a/test/integration/004_simple_snapshot_test/test_simple_snapshot.py
+++ b/test/integration/004_simple_snapshot_test/test_simple_snapshot.py
@@ -2,7 +2,7 @@
 import dbt.exceptions
 
 
-class TestSimpleSnapshotFiles(DBTIntegrationTest):
+class BaseSimpleSnapshotTest(DBTIntegrationTest):
     NUM_SNAPSHOT_MODELS = 1
 
     @property
@@ -16,19 +16,6 @@ def models(self):
     def run_snapshot(self):
         return self.run_dbt(['snapshot'])
 
-    @property
-    def project_config(self):
-        return {
-            "data-paths": ['data'],
-            "snapshot-paths": ['test-snapshots-pg'],
-        }
-
-    @use_profile('postgres')
-    def test__postgres_ref_snapshot(self):
-        self.dbt_run_seed_snapshot()
-        results = self.run_dbt(['run'])
-        self.assertEqual(len(results), 1)
-
     def dbt_run_seed_snapshot(self):
         if self.adapter_type == 'postgres':
             self.run_sql_file('seed_pg.sql')
@@ -48,6 +35,21 @@ def assert_case_tables_equal(self, actual, expected):
     def assert_expected(self):
         self.assert_case_tables_equal('snapshot_actual', 'snapshot_expected')
 
+
+class TestSimpleSnapshotFiles(BaseSimpleSnapshotTest):
+    @property
+    def project_config(self):
+        return {
+            "data-paths": ['data'],
+            "snapshot-paths": ['test-snapshots-pg'],
+        }
+
+    @use_profile('postgres')
+    def test__postgres_ref_snapshot(self):
+        self.dbt_run_seed_snapshot()
+        results = self.run_dbt(['run'])
+        self.assertEqual(len(results), 1)
+
     @use_profile('postgres')
     def test__postgres__simple_snapshot(self):
         self.dbt_run_seed_snapshot()
@@ -100,6 +102,77 @@ def test__presto__simple_snapshot_disabled(self):
         self.assertIn('not implemented for presto', results[0].error)
 
 
+class TestCustomSnapshotFiles(BaseSimpleSnapshotTest):
+    @property
+    def project_config(self):
+        return {
+            'data-paths': ['data'],
+            'macro-paths': ['custom-snapshot-macros'],
+            'snapshot-paths': ['test-snapshots-pg-custom'],
+        }
+
+    @use_profile('postgres')
+    def test__postgres_ref_snapshot(self):
+        self.dbt_run_seed_snapshot()
+        results = self.run_dbt(['run'])
+        self.assertEqual(len(results), 1)
+
+    @use_profile('postgres')
+    def test__postgres__simple_custom_snapshot(self):
+        self.dbt_run_seed_snapshot()
+
+        self.assert_expected()
+
+        self.run_sql_file("invalidate_postgres.sql")
+        self.run_sql_file("update.sql")
+
+        results = self.run_snapshot()
+        self.assertEqual(len(results),  self.NUM_SNAPSHOT_MODELS)
+
+        self.assert_expected()
+
+
+class TestNamespacedCustomSnapshotFiles(BaseSimpleSnapshotTest):
+    @property
+    def project_config(self):
+        return {
+            'data-paths': ['data'],
+            'macro-paths': ['custom-snapshot-macros'],
+            'snapshot-paths': ['test-snapshots-pg-custom-namespaced'],
+        }
+
+    @use_profile('postgres')
+    def test__postgres__simple_custom_snapshot_namespaced(self):
+        self.dbt_run_seed_snapshot()
+
+        self.assert_expected()
+
+        self.run_sql_file("invalidate_postgres.sql")
+        self.run_sql_file("update.sql")
+
+        results = self.run_snapshot()
+        self.assertEqual(len(results),  self.NUM_SNAPSHOT_MODELS)
+
+        self.assert_expected()
+
+
+class TestInvalidNamespacedCustomSnapshotFiles(BaseSimpleSnapshotTest):
+    @property
+    def project_config(self):
+        return {
+            'data-paths': ['data'],
+            'macro-paths': ['custom-snapshot-macros'],
+            'snapshot-paths': ['test-snapshots-pg-custom-invalid'],
+        }
+
+    def run_snapshot(self):
+        return self.run_dbt(['snapshot'], expect_pass=False)
+
+    @use_profile('postgres')
+    def test__postgres__simple_custom_snapshot_invalid_namespace(self):
+        self.dbt_run_seed_snapshot()
+
+
 class TestSimpleSnapshotFileSelects(DBTIntegrationTest):
     @property
     def schema(self):