Repository: dbt-labs/dbt-core

Issue:
[CT-2537] Include "compiled" node attributes in run results
### Proposed change

Let's include attributes that can vary at compile/runtime. Those attributes are:
- `compiled: bool = True` (always `False` after parsing)
- `compiled_code` (always empty after parsing) - this is **most** important
- `relation_name` - this is set after parsing, but it can be updated for tests that have `--store-failures` enabled
- `depends_on.macros` (https://github.com/dbt-labs/dbt-core/issues/5079) - since additional macro dependencies are required/registered while a model is being materialized, versus just compiled (e.g. `create_table_as`)

([Here](https://github.com/dbt-labs/dbt-core/blob/f1dddaa6e96b743498440925a55e66a7d5cdacfa/core/dbt/compilation.py#L361-L407) is where we set many of those attributes, during compilation)

The `run_results.json` produced by `compile`, `docs generate`, `run`/`build`/`test`/etc should include these fields (and only these fields) in addition to the node's `unique_id`. All other fields can be accessed from the manifest, and represent logical state at a point in time.

### Background

- Way back in v0.19, we removed the full node entry from `run_results.json`, and moved the `compiled_code` attribute into the manifest, so as to power `dbt-docs` from _only_ manifest + catalog.
- Starting in v1.4 (https://github.com/dbt-labs/dbt-core/pull/6384), we narrowed down the set of fields that differ between "uncompiled" and "compiled" nodes.

### Rationale

- The manifest represents "logical" state (parsed from project code + configs)
- Run results + catalog represent "applied" state (as materialized by dbt, as the objects exist in the data warehouse)

Compiled code can be a form of "applied" state. If a model's SQL depends on the results of an introspective query, it can vary given different inputs from the data warehouse.

### Acceptance Criteria

- The `run_results.json` produced by `compile`, `docs generate`, `run`, `build`, and `test` should include the `compiled`, `compiled_code`, `relation_name`, and `depends_on.macros` fields (and only these fields) in addition to the node's `unique_id`.
- The same information should also be included in the results returned by dbtRunner when is used to invoke dbt as a library.




Discussion:
@jtcohen6 per BLG: We're estimating this as written but have some concerns and would like to understand the use-case better-- Maybe a situation that would call for a new runtime artifact, `compile_results.json`?

Resolving patch:
diff --git a/core/dbt/contracts/results.py b/core/dbt/contracts/results.py
--- a/core/dbt/contracts/results.py
+++ b/core/dbt/contracts/results.py
@@ -1,7 +1,7 @@
 import threading
 
 from dbt.contracts.graph.unparsed import FreshnessThreshold
-from dbt.contracts.graph.nodes import SourceDefinition, ResultNode
+from dbt.contracts.graph.nodes import CompiledNode, SourceDefinition, ResultNode
 from dbt.contracts.util import (
     BaseArtifactMetadata,
     ArtifactMixin,
@@ -203,9 +203,15 @@ class RunResultsMetadata(BaseArtifactMetadata):
 @dataclass
 class RunResultOutput(BaseResult):
     unique_id: str
+    compiled: Optional[bool]
+    compiled_code: Optional[str]
+    relation_name: Optional[str]
 
 
 def process_run_result(result: RunResult) -> RunResultOutput:
+
+    compiled = isinstance(result.node, CompiledNode)
+
     return RunResultOutput(
         unique_id=result.node.unique_id,
         status=result.status,
@@ -215,6 +221,9 @@ def process_run_result(result: RunResult) -> RunResultOutput:
         message=result.message,
         adapter_response=result.adapter_response,
         failures=result.failures,
+        compiled=result.node.compiled if compiled else None,  # type:ignore
+        compiled_code=result.node.compiled_code if compiled else None,  # type:ignore
+        relation_name=result.node.relation_name if compiled else None,  # type:ignore
     )
 
 
@@ -237,7 +246,7 @@ def write(self, path: str):
 
 
 @dataclass
-@schema_version("run-results", 4)
+@schema_version("run-results", 5)
 class RunResultsArtifact(ExecutionResult, ArtifactMixin):
     results: Sequence[RunResultOutput]
     args: Dict[str, Any] = field(default_factory=dict)


Test patch:
diff --git a/core/dbt/tests/util.py b/core/dbt/tests/util.py
--- a/core/dbt/tests/util.py
+++ b/core/dbt/tests/util.py
@@ -5,7 +5,7 @@
 import json
 import warnings
 from datetime import datetime
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 from contextlib import contextmanager
 from dbt.adapters.factory import Adapter
 
@@ -160,6 +160,16 @@ def get_manifest(project_root) -> Optional[Manifest]:
         return None
 
 
+# Used in test cases to get the run_results.json file.
+def get_run_results(project_root) -> Any:
+    path = os.path.join(project_root, "target", "run_results.json")
+    if os.path.exists(path):
+        with open(path) as run_result_text:
+            return json.load(run_result_text)
+    else:
+        return None
+
+
 # Used in tests to copy a file, usually from a data directory to the project directory
 def copy_file(src_path, src, dest_path, dest) -> None:
     # dest is a list, so that we can provide nested directories, like 'models' etc.
diff --git a/tests/functional/artifacts/expected_run_results.py b/tests/functional/artifacts/expected_run_results.py
--- a/tests/functional/artifacts/expected_run_results.py
+++ b/tests/functional/artifacts/expected_run_results.py
@@ -17,6 +17,9 @@ def expected_run_results():
             "thread_id": ANY,
             "timing": [ANY, ANY],
             "failures": ANY,
+            "compiled": True,
+            "compiled_code": ANY,
+            "relation_name": ANY,
         },
         {
             "status": "success",
@@ -27,6 +30,9 @@ def expected_run_results():
             "thread_id": ANY,
             "timing": [ANY, ANY],
             "failures": ANY,
+            "compiled": True,
+            "compiled_code": ANY,
+            "relation_name": ANY,
         },
         {
             "status": "success",
@@ -37,6 +43,9 @@ def expected_run_results():
             "thread_id": ANY,
             "timing": [ANY, ANY],
             "failures": ANY,
+            "compiled": None,
+            "compiled_code": ANY,
+            "relation_name": None,
         },
         {
             "status": "success",
@@ -47,6 +56,9 @@ def expected_run_results():
             "thread_id": ANY,
             "timing": [ANY, ANY],
             "failures": ANY,
+            "compiled": True,
+            "compiled_code": ANY,
+            "relation_name": ANY,
         },
         {
             "status": "success",
@@ -57,6 +69,9 @@ def expected_run_results():
             "thread_id": ANY,
             "timing": [ANY, ANY],
             "failures": ANY,
+            "compiled": True,
+            "compiled_code": ANY,
+            "relation_name": None,
         },
         {
             "status": "success",
@@ -67,6 +82,9 @@ def expected_run_results():
             "thread_id": ANY,
             "timing": [ANY, ANY],
             "failures": ANY,
+            "compiled": True,
+            "compiled_code": ANY,
+            "relation_name": None,
         },
         {
             "status": "success",
@@ -77,6 +95,9 @@ def expected_run_results():
             "thread_id": ANY,
             "timing": [ANY, ANY],
             "failures": ANY,
+            "compiled": True,
+            "compiled_code": ANY,
+            "relation_name": None,
         },
     ]
 
@@ -92,6 +113,9 @@ def expected_references_run_results():
             "thread_id": ANY,
             "timing": [ANY, ANY],
             "failures": ANY,
+            "compiled": True,
+            "compiled_code": ANY,
+            "relation_name": ANY,
         },
         {
             "status": "success",
@@ -102,6 +126,9 @@ def expected_references_run_results():
             "thread_id": ANY,
             "timing": [ANY, ANY],
             "failures": ANY,
+            "compiled": True,
+            "compiled_code": ANY,
+            "relation_name": ANY,
         },
         {
             "status": "success",
@@ -112,6 +139,9 @@ def expected_references_run_results():
             "thread_id": ANY,
             "timing": [ANY, ANY],
             "failures": ANY,
+            "compiled": None,
+            "compiled_code": ANY,
+            "relation_name": ANY,
         },
         {
             "status": "success",
@@ -122,6 +152,9 @@ def expected_references_run_results():
             "thread_id": ANY,
             "timing": [ANY, ANY],
             "failures": ANY,
+            "compiled": True,
+            "compiled_code": ANY,
+            "relation_name": ANY,
         },
     ]
 
@@ -137,6 +170,9 @@ def expected_versions_run_results():
             "thread_id": ANY,
             "timing": [ANY, ANY],
             "failures": ANY,
+            "compiled": True,
+            "compiled_code": ANY,
+            "relation_name": ANY,
         },
         {
             "status": "success",
@@ -147,6 +183,9 @@ def expected_versions_run_results():
             "thread_id": ANY,
             "timing": [ANY, ANY],
             "failures": ANY,
+            "compiled": True,
+            "compiled_code": ANY,
+            "relation_name": ANY,
         },
         {
             "status": "success",
@@ -157,6 +196,9 @@ def expected_versions_run_results():
             "thread_id": ANY,
             "timing": [ANY, ANY],
             "failures": ANY,
+            "compiled": True,
+            "compiled_code": ANY,
+            "relation_name": ANY,
         },
         {
             "status": "success",
@@ -167,6 +209,9 @@ def expected_versions_run_results():
             "thread_id": ANY,
             "timing": [ANY, ANY],
             "failures": ANY,
+            "compiled": True,
+            "compiled_code": ANY,
+            "relation_name": ANY,
         },
         {
             "status": "success",
@@ -177,6 +222,9 @@ def expected_versions_run_results():
             "thread_id": ANY,
             "timing": [ANY, ANY],
             "failures": ANY,
+            "compiled": True,
+            "compiled_code": ANY,
+            "relation_name": ANY,
         },
         {
             "status": "success",
@@ -187,5 +235,8 @@ def expected_versions_run_results():
             "thread_id": ANY,
             "timing": [ANY, ANY],
             "failures": ANY,
+            "compiled": True,
+            "compiled_code": ANY,
+            "relation_name": ANY,
         },
     ]
diff --git a/tests/functional/assertions/test_runner.py b/tests/functional/assertions/test_runner.py
new file mode 100644
--- /dev/null
+++ b/tests/functional/assertions/test_runner.py
@@ -0,0 +1,43 @@
+import os
+from typing import Callable, List, Optional
+
+from dbt.cli.main import dbtRunner, dbtRunnerResult
+from dbt.contracts.graph.manifest import Manifest
+from dbt.events.base_types import EventMsg
+from dbt.tests.util import get_run_results
+
+
+def assert_run_results_have_compiled_node_attributes(
+    args: List[str], result: dbtRunnerResult
+) -> None:
+    commands_with_run_results = ["build", "compile", "docs", "run", "test"]
+    if not [a for a in args if a in commands_with_run_results] or not result.success:
+        return
+
+    run_results = get_run_results(os.getcwd())
+    for r in run_results["results"]:
+        if r["unique_id"].startswith("model") and r["status"] == "success":
+            assert "compiled_code" in r
+            assert "compiled" in r
+
+
+_STANDARD_ASSERTIONS = [assert_run_results_have_compiled_node_attributes]
+
+
+class dbtTestRunner(dbtRunner):
+    def __init__(
+        self,
+        manifest: Optional[Manifest] = None,
+        callbacks: Optional[List[Callable[[EventMsg], None]]] = None,
+        exit_assertions: Optional[List[Callable[[List[str], dbtRunnerResult], None]]] = None,
+    ):
+        self.exit_assertions = exit_assertions if exit_assertions else _STANDARD_ASSERTIONS
+        super().__init__(manifest, callbacks)
+
+    def invoke(self, args: List[str], **kwargs) -> dbtRunnerResult:
+        result = super().invoke(args, **kwargs)
+
+        for assertion in self.exit_assertions:
+            assertion(args, result)
+
+        return result
diff --git a/tests/functional/compile/test_compile.py b/tests/functional/compile/test_compile.py
--- a/tests/functional/compile/test_compile.py
+++ b/tests/functional/compile/test_compile.py
@@ -3,7 +3,6 @@
 import pytest
 import re
 
-from dbt.cli.main import dbtRunner
 from dbt.exceptions import DbtRuntimeError, Exception as DbtException
 from dbt.tests.util import run_dbt, run_dbt_and_capture, read_file
 from tests.functional.compile.fixtures import (
@@ -16,6 +15,7 @@
     schema_yml,
     model_multiline_jinja,
 )
+from tests.functional.assertions.test_runner import dbtTestRunner
 
 
 def norm_whitespace(string):
@@ -189,11 +189,11 @@ def test_output_json_inline(self, project):
         assert '"compiled"' in log_output
 
     def test_compile_inline_not_add_node(self, project):
-        dbt = dbtRunner()
+        dbt = dbtTestRunner()
         parse_result = dbt.invoke(["parse"])
         manifest = parse_result.result
         assert len(manifest.nodes) == 4
-        dbt = dbtRunner(manifest=manifest)
+        dbt = dbtTestRunner(manifest=manifest)
         dbt.invoke(
             ["compile", "--inline", "select * from {{ ref('second_model') }}"],
             populate_cache=False,
@@ -218,7 +218,7 @@ def test_graph_summary_output(self, project):
         """Ensure that the compile command generates a file named graph_summary.json
         in the target directory, that the file contains valid json, and that the
         json has the high level structure it should."""
-        dbtRunner().invoke(["compile"])
+        dbtTestRunner().invoke(["compile"])
         summary_path = pathlib.Path(project.project_root, "target/graph_summary.json")
         with open(summary_path, "r") as summary_file:
             summary = json.load(summary_file)
diff --git a/tests/functional/semantic_models/test_semantic_model_parsing.py b/tests/functional/semantic_models/test_semantic_model_parsing.py
--- a/tests/functional/semantic_models/test_semantic_model_parsing.py
+++ b/tests/functional/semantic_models/test_semantic_model_parsing.py
@@ -4,11 +4,10 @@
 
 from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity
 
-from dbt.cli.main import dbtRunner
 from dbt.contracts.graph.manifest import Manifest
 from dbt.events.base_types import BaseEvent
 from dbt.tests.util import write_file
-
+from tests.functional.assertions.test_runner import dbtTestRunner
 
 schema_yml = """models:
   - name: fct_revenue
@@ -119,7 +118,7 @@ def models(self):
         }
 
     def test_semantic_model_parsing(self, project):
-        runner = dbtRunner()
+        runner = dbtTestRunner()
         result = runner.invoke(["parse"])
         assert result.success
         assert isinstance(result.result, Manifest)
@@ -142,7 +141,7 @@ def test_semantic_model_error(self, project):
         error_schema_yml = schema_yml.replace("sum_of_things", "has_revenue")
         write_file(error_schema_yml, project.project_root, "models", "schema.yml")
         events: List[BaseEvent] = []
-        runner = dbtRunner(callbacks=[events.append])
+        runner = dbtTestRunner(callbacks=[events.append])
         result = runner.invoke(["parse"])
         assert not result.success
 
@@ -162,7 +161,7 @@ def models(self):
     def test_semantic_model_changed_partial_parsing(self, project):
         # First, use the default schema.yml to define our semantic model, and
         # run the dbt parse command
-        runner = dbtRunner()
+        runner = dbtTestRunner()
         result = runner.invoke(["parse"])
         assert result.success
 
@@ -183,7 +182,7 @@ def test_semantic_model_changed_partial_parsing(self, project):
     def test_semantic_model_deleted_partial_parsing(self, project):
         # First, use the default schema.yml to define our semantic model, and
         # run the dbt parse command
-        runner = dbtRunner()
+        runner = dbtTestRunner()
         result = runner.invoke(["parse"])
         assert result.success
         assert "semantic_model.test.revenue" in result.result.semantic_models
@@ -203,7 +202,7 @@ def test_semantic_model_flipping_create_metric_partial_parsing(self, project):
         # First, use the default schema.yml to define our semantic model, and
         # run the dbt parse command
         write_file(schema_yml, project.project_root, "models", "schema.yml")
-        runner = dbtRunner()
+        runner = dbtTestRunner()
         result = runner.invoke(["parse"])
         assert result.success