Repository: dbt-labs/dbt-core

Issue:
dbt build: Change the way we handle tests (if parent test fails then don't run non-test children)
This work was finished, however it had to be removed from the PR due to difficulties with how some (many) of our tests operate.  Because the tests don't build a correct graph failures occurred when running those tests.  The obvious fix is to re-write those tests but it was too many to do as part of the `dbt build` effort.

References:
- [Branch with operational blocking feature and broken tests](https://github.com/dbt-labs/dbt/tree/aaac5ff2e67f4f6f9482b1601855036671678c36).
- [Discussion about why the tests caused it to hang](https://dbt-labs.slack.com/archives/C0131TY7EEA/p1625772168426200)




Discussion:
It looks like the particular commits we'd want to cherry-pick and revive are https://github.com/dbt-labs/dbt/commit/4dc29630b561aebe8c02f49cf1223476c9204950 + https://github.com/dbt-labs/dbt/commit/aaac5ff2e67f4f6f9482b1601855036671678c36
Exit criteria via example:
- `model_a` has two children: a `unique` test and `model_b`
- The `unique` test should run before `model_b`
- If the `unique` test fails, `model_b` should skip running
- The overall `dbt build` command _should keep running_

```
18:49:43 | Concurrency: 1 threads (target='dev')
18:49:43 |
18:49:43 | 1 of 7 START view model dbt_jcohen.model_a........................ [RUN]
18:49:43 | 1 of 7 OK created model dbt_jcohen.model_a........................ [INSERT 2 in 0.09s]
18:49:43 | 2 of 7 START test unique_model_a_id............................... [RUN]
18:49:43 | 2 of 7 FAIL test unique_model_a_id................................ [FAIL in 0.12s]
18:49:43 | 3 of 7 SKIP relation model_b...................................... [SKIP]
18:49:43 | 4 of 7 START relation another_model............................... [START]
... and so on ...
```

Resolving patch:
diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py
--- a/core/dbt/compilation.py
+++ b/core/dbt/compilation.py
@@ -10,7 +10,7 @@
 from dbt.clients import jinja
 from dbt.clients.system import make_directory
 from dbt.context.providers import generate_runtime_model
-from dbt.contracts.graph.manifest import Manifest
+from dbt.contracts.graph.manifest import Manifest, UniqueID
 from dbt.contracts.graph.compiled import (
     COMPILED_TYPES,
     CompiledSchemaTestNode,
@@ -107,6 +107,18 @@ def _extend_prepended_ctes(prepended_ctes, new_prepended_ctes):
         _add_prepended_cte(prepended_ctes, new_cte)
 
 
+def _get_tests_for_node(manifest: Manifest, unique_id: UniqueID) -> List[UniqueID]:
+    """ Get a list of tests that depend on the node with the
+    provided unique id """
+
+    return [
+        node.unique_id
+        for _, node in manifest.nodes.items()
+        if node.resource_type == NodeType.Test and
+        unique_id in node.depends_on_nodes
+    ]
+
+
 class Linker:
     def __init__(self, data=None):
         if data is None:
@@ -142,7 +154,7 @@ def write_graph(self, outfile: str, manifest: Manifest):
         include all nodes in their corresponding graph entries.
         """
         out_graph = self.graph.copy()
-        for node_id in self.graph.nodes():
+        for node_id in self.graph:
             data = manifest.expect(node_id).to_dict(omit_none=True)
             out_graph.add_node(node_id, **data)
         nx.write_gpickle(out_graph, outfile)
@@ -412,13 +424,80 @@ def link_graph(self, linker: Linker, manifest: Manifest):
             self.link_node(linker, node, manifest)
         for exposure in manifest.exposures.values():
             self.link_node(linker, exposure, manifest)
-            # linker.add_node(exposure.unique_id)
 
         cycle = linker.find_cycles()
 
         if cycle:
             raise RuntimeError("Found a cycle: {}".format(cycle))
 
+        self.resolve_graph(linker, manifest)
+
+    def resolve_graph(self, linker: Linker, manifest: Manifest) -> None:
+        """ This method adds additional edges to the DAG. For a given non-test
+        executable node, add an edge from an upstream test to the given node if
+        the set of nodes the test depends on is a proper/strict subset of the
+        upstream nodes for the given node. """
+
+        # Given a graph:
+        # model1 --> model2 --> model3
+        #   |         |
+        #   |        \/
+        #  \/      test 2
+        # test1
+        #
+        # Produce the following graph:
+        # model1 --> model2 --> model3
+        #   |         |         /\ /\
+        #   |        \/         |  |
+        #  \/      test2 -------   |
+        # test1 -------------------
+
+        for node_id in linker.graph:
+            # If node is executable (in manifest.nodes) and does _not_
+            # represent a test, continue.
+            if (
+                node_id in manifest.nodes and
+                manifest.nodes[node_id].resource_type != NodeType.Test
+            ):
+                # Get *everything* upstream of the node
+                all_upstream_nodes = nx.traversal.bfs_tree(
+                    linker.graph, node_id, reverse=True
+                )
+                # Get the set of upstream nodes not including the current node.
+                upstream_nodes = set([
+                    n for n in all_upstream_nodes if n != node_id
+                ])
+
+                # Get all tests that depend on any upstream nodes.
+                upstream_tests = []
+                for upstream_node in upstream_nodes:
+                    upstream_tests += _get_tests_for_node(
+                        manifest,
+                        upstream_node
+                    )
+
+                for upstream_test in upstream_tests:
+                    # Get the set of all nodes that the test depends on
+                    # including the upstream_node itself. This is necessary
+                    # because tests can depend on multiple nodes (ex:
+                    # relationship tests). Test nodes do not distinguish
+                    # between what node the test is "testing" and what
+                    # node(s) it depends on.
+                    test_depends_on = set(
+                        manifest.nodes[upstream_test].depends_on_nodes
+                    )
+
+                    # If the set of nodes that an upstream test depends on
+                    # is a proper (or strict) subset of all upstream nodes of
+                    # the current node, add an edge from the upstream test
+                    # to the current node. Must be a proper/strict subset to
+                    # avoid adding a circular dependency to the graph.
+                    if (test_depends_on < upstream_nodes):
+                        linker.graph.add_edge(
+                            upstream_test,
+                            node_id
+                        )
+
     def compile(self, manifest: Manifest, write=True) -> Graph:
         self.initialize()
         linker = Linker()
diff --git a/core/dbt/task/build.py b/core/dbt/task/build.py
--- a/core/dbt/task/build.py
+++ b/core/dbt/task/build.py
@@ -3,18 +3,19 @@
 from .seed import SeedRunner as seed_runner
 from .test import TestRunner as test_runner
 
+from dbt.contracts.results import NodeStatus
 from dbt.graph import ResourceTypeSelector
 from dbt.exceptions import InternalException
 from dbt.node_types import NodeType
 
 
 class BuildTask(RunTask):
-    """The Build task processes all assets of a given process and attempts to 'build'
-    them in an opinionated fashion.  Every resource type outlined in RUNNER_MAP
-    will be processed by the mapped runner class.
+    """The Build task processes all assets of a given process and
+    attempts to 'build' them in an opinionated fashion. Every resource
+    type outlined in RUNNER_MAP will be processed by the mapped runner class.
 
-    I.E. a resource of type Model is handled by the ModelRunner which is imported
-    as run_model_runner.
+    I.E. a resource of type Model is handled by the ModelRunner which is
+    imported as run_model_runner.
     """
 
     RUNNER_MAP = {
@@ -24,6 +25,8 @@ class BuildTask(RunTask):
         NodeType.Test: test_runner,
     }
 
+    MARK_DEPENDENT_ERRORS_STATUSES = [NodeStatus.Error, NodeStatus.Fail]
+
     def get_node_selector(self) -> ResourceTypeSelector:
         if self.manifest is None or self.graph is None:
             raise InternalException(
diff --git a/core/dbt/task/runnable.py b/core/dbt/task/runnable.py
--- a/core/dbt/task/runnable.py
+++ b/core/dbt/task/runnable.py
@@ -83,6 +83,9 @@ def _runtime_initialize(self):
 
 
 class GraphRunnableTask(ManifestTask):
+
+    MARK_DEPENDENT_ERRORS_STATUSES = [NodeStatus.Error]
+
     def __init__(self, args, config):
         super().__init__(args, config)
         self.job_queue: Optional[GraphQueue] = None
@@ -289,7 +292,7 @@ def _handle_result(self, result):
         else:
             self.manifest.update_node(node)
 
-        if result.status == NodeStatus.Error:
+        if result.status in self.MARK_DEPENDENT_ERRORS_STATUSES:
             if is_ephemeral:
                 cause = result
             else:


Test patch:
diff --git a/test/integration/069_build_test/models-circular-relationship/model_0.sql b/test/integration/069_build_test/models-circular-relationship/model_0.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/069_build_test/models-circular-relationship/model_0.sql
@@ -0,0 +1,3 @@
+{{ config(materialized='table') }}
+
+select * from {{ ref('countries') }}
\ No newline at end of file
diff --git a/test/integration/069_build_test/models-circular-relationship/model_1.sql b/test/integration/069_build_test/models-circular-relationship/model_1.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/069_build_test/models-circular-relationship/model_1.sql
@@ -0,0 +1,3 @@
+{{ config(materialized='table') }}
+
+select * from {{ ref('model_0') }}
diff --git a/test/integration/069_build_test/models-circular-relationship/model_99.sql b/test/integration/069_build_test/models-circular-relationship/model_99.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/069_build_test/models-circular-relationship/model_99.sql
@@ -0,0 +1,4 @@
+{{ config(materialized='table') }}
+
+select '1' as "num"
+
diff --git a/test/integration/069_build_test/models-circular-relationship/test.yml b/test/integration/069_build_test/models-circular-relationship/test.yml
new file mode 100644
--- /dev/null
+++ b/test/integration/069_build_test/models-circular-relationship/test.yml
@@ -0,0 +1,18 @@
+version: 2
+
+models:
+  - name: model_0
+    columns:
+      - name: iso3
+        tests:
+          - relationships:
+              to: ref('model_1')
+              field: iso3
+
+  - name: model_1
+    columns:
+      - name: iso3
+        tests:
+          - relationships:
+              to: ref('model_0')
+              field: iso3
diff --git a/test/integration/069_build_test/models-failing/model_3.sql b/test/integration/069_build_test/models-failing/model_3.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/069_build_test/models-failing/model_3.sql
@@ -0,0 +1,3 @@
+{{ config(materialized='table') }}
+
+select * from {{ ref('model_1') }}
diff --git a/test/integration/069_build_test/test_build.py b/test/integration/069_build_test/test_build.py
--- a/test/integration/069_build_test/test_build.py
+++ b/test/integration/069_build_test/test_build.py
@@ -2,15 +2,11 @@
 import yaml
 
 
-class TestBuild(DBTIntegrationTest):
+class TestBuildBase(DBTIntegrationTest):
     @property
     def schema(self):
         return "build_test_069"
 
-    @property
-    def models(self):
-        return "models"
-
     @property
     def project_config(self):
         return {
@@ -31,24 +27,55 @@ def build(self, expect_pass=True, extra_args=None, **kwargs):
 
         return self.run_dbt(args, expect_pass=expect_pass)
 
+
+class TestPassingBuild(TestBuildBase):
+    @property
+    def models(self):
+        return "models"
+
     @use_profile("postgres")
     def test__postgres_build_happy_path(self):
         self.build()
 
 
-class TestFailingBuild(TestBuild):
-    @property
-    def schema(self):
-        return "build_test_069"
-
+class TestFailingBuild(TestBuildBase):
     @property
     def models(self):
         return "models-failing"
-        
+
     @use_profile("postgres")
     def test__postgres_build_happy_path(self):
         results = self.build(expect_pass=False)
-        self.assertEqual(len(results), 12)
+        self.assertEqual(len(results), 13)
+        actual = [r.status for r in results]
+        expected = ['error']*1 + ['skipped']*5 + ['pass']*2 + ['success']*5
+        self.assertEqual(sorted(actual), sorted(expected))
+
+
+class TestFailingTestsBuild(TestBuildBase):
+    @property
+    def models(self):
+        return "tests-failing"
+
+    @use_profile("postgres")
+    def test__postgres_failing_test_skips_downstream(self):
+        results = self.build(expect_pass=False)
+        self.assertEqual(len(results), 13)
+        actual = [str(r.status) for r in results]
+        expected = ['fail'] + ['skipped']*6 + ['pass']*2 + ['success']*4
+        self.assertEqual(sorted(actual), sorted(expected))
+
+
+class TestCircularRelationshipTestsBuild(TestBuildBase):
+    @property
+    def models(self):
+        return "models-circular-relationship"
+
+    @use_profile("postgres")
+    def test__postgres_circular_relationship_test_success(self):
+        """ Ensure that tests that refer to each other's model don't create
+        a circular dependency. """
+        results = self.build()
         actual = [r.status for r in results]
-        expected = ['error']*1 + ['skipped']*4 + ['pass']*2 + ['success']*5
+        expected = ['success']*7 + ['pass']*2
         self.assertEqual(sorted(actual), sorted(expected))
diff --git a/test/integration/069_build_test/tests-failing/model_0.sql b/test/integration/069_build_test/tests-failing/model_0.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/069_build_test/tests-failing/model_0.sql
@@ -0,0 +1,3 @@
+{{ config(materialized='table') }}
+
+select * from {{ ref('countries') }}
\ No newline at end of file
diff --git a/test/integration/069_build_test/tests-failing/model_1.sql b/test/integration/069_build_test/tests-failing/model_1.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/069_build_test/tests-failing/model_1.sql
@@ -0,0 +1,3 @@
+{{ config(materialized='table') }}
+
+select * from {{ ref('snap_0') }}
diff --git a/test/integration/069_build_test/tests-failing/model_2.sql b/test/integration/069_build_test/tests-failing/model_2.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/069_build_test/tests-failing/model_2.sql
@@ -0,0 +1,3 @@
+{{ config(materialized='table') }}
+
+select * from {{ ref('snap_1') }}
\ No newline at end of file
diff --git a/test/integration/069_build_test/tests-failing/model_99.sql b/test/integration/069_build_test/tests-failing/model_99.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/069_build_test/tests-failing/model_99.sql
@@ -0,0 +1,3 @@
+{{ config(materialized='table') }}
+
+select '1' as "num"
\ No newline at end of file
diff --git a/test/integration/069_build_test/tests-failing/test.yml b/test/integration/069_build_test/tests-failing/test.yml
new file mode 100644
--- /dev/null
+++ b/test/integration/069_build_test/tests-failing/test.yml
@@ -0,0 +1,18 @@
+version: 2
+
+models:
+  - name: model_0
+    columns:
+      - name: iso3
+        tests:
+          - unique
+          - not_null
+      - name: historical_iso_numeric
+        tests:
+          - not_null
+  - name: model_2
+    columns:
+      - name: iso3
+        tests:
+          - unique
+          - not_null
diff --git a/test/rpc/test_build.py b/test/rpc/test_build.py
--- a/test/rpc/test_build.py
+++ b/test/rpc/test_build.py
@@ -26,6 +26,7 @@
 {% endsnapshot %}
 '''
 
+
 @pytest.mark.supported('postgres')
 def test_rpc_build_threads(
     project_root, profiles_root, dbt_profile, unique_schema