Repository: dbt-labs/dbt-core

Issue:
[CT-39] Bug: ensure that dump_graph is not run for logging events in cachy.py unless cache logging is turned on
We skip actual logging of core/dbt/adapters/cache.py in core/dbt/events/functions.py fire_event, but that doesn't prevent the actual 'dump_graph' call from executing, which is a performance problem.

In 0.21 we used a 'lazy_log' function to skip those events. We should re-implement something like that to prevent the overhead of these calls.


Discussion:

Resolving patch:
diff --git a/core/dbt/adapters/cache.py b/core/dbt/adapters/cache.py
--- a/core/dbt/adapters/cache.py
+++ b/core/dbt/adapters/cache.py
@@ -4,7 +4,7 @@
 
 from dbt.adapters.reference_keys import _make_key, _ReferenceKey
 import dbt.exceptions
-from dbt.events.functions import fire_event
+from dbt.events.functions import fire_event, fire_event_if
 from dbt.events.types import (
     AddLink,
     AddRelation,
@@ -20,6 +20,7 @@
     UncachedRelation,
     UpdateReference
 )
+import dbt.flags as flags
 from dbt.utils import lowercase
 
 
@@ -323,11 +324,11 @@ def add(self, relation):
         """
         cached = _CachedRelation(relation)
         fire_event(AddRelation(relation=_make_key(cached)))
-        fire_event(DumpBeforeAddGraph(dump=self.dump_graph()))
+        fire_event_if(flags.LOG_CACHE_EVENTS, lambda: DumpBeforeAddGraph(dump=self.dump_graph()))
 
         with self.lock:
             self._setdefault(cached)
-        fire_event(DumpAfterAddGraph(dump=self.dump_graph()))
+        fire_event_if(flags.LOG_CACHE_EVENTS, lambda: DumpAfterAddGraph(dump=self.dump_graph()))
 
     def _remove_refs(self, keys):
         """Removes all references to all entries in keys. This does not
@@ -440,8 +441,10 @@ def rename(self, old, new):
         old_key = _make_key(old)
         new_key = _make_key(new)
         fire_event(RenameSchema(old_key=old_key, new_key=new_key))
-
-        fire_event(DumpBeforeRenameSchema(dump=self.dump_graph()))
+        fire_event_if(
+            flags.LOG_CACHE_EVENTS,
+            lambda: DumpBeforeRenameSchema(dump=self.dump_graph())
+        )
 
         with self.lock:
             if self._check_rename_constraints(old_key, new_key):
@@ -449,7 +452,10 @@ def rename(self, old, new):
             else:
                 self._setdefault(_CachedRelation(new))
 
-        fire_event(DumpAfterRenameSchema(dump=self.dump_graph()))
+        fire_event_if(
+            flags.LOG_CACHE_EVENTS,
+            lambda: DumpAfterRenameSchema(dump=self.dump_graph())
+        )
 
     def get_relations(
         self, database: Optional[str], schema: Optional[str]
diff --git a/core/dbt/events/functions.py b/core/dbt/events/functions.py
--- a/core/dbt/events/functions.py
+++ b/core/dbt/events/functions.py
@@ -306,6 +306,13 @@ def send_exc_to_logger(
         )
 
 
+# an alternative to fire_event which only creates and logs the event value
+# if the condition is met. Does nothing otherwise.
+def fire_event_if(conditional: bool, lazy_e: Callable[[], Event]) -> None:
+    if conditional:
+        fire_event(lazy_e())
+
+
 # top-level method for accessing the new eventing system
 # this is where all the side effects happen branched by event type
 # (i.e. - mutating the event history, printing to stdout, logging
diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py
--- a/core/dbt/events/types.py
+++ b/core/dbt/events/types.py
@@ -699,7 +699,7 @@ def message(self) -> str:
 
 @dataclass
 class DumpBeforeAddGraph(DebugLevel, Cli, File, Cache):
-    # large value. delay not necessary since every debug level message is logged anyway.
+    # large value. delay creation with fire_event_if.
     dump: Dict[str, List[str]]
     code: str = "E031"
 
@@ -709,7 +709,7 @@ def message(self) -> str:
 
 @dataclass
 class DumpAfterAddGraph(DebugLevel, Cli, File, Cache):
-    # large value. delay not necessary since every debug level message is logged anyway.
+    # large value. delay creation with fire_event_if.
     dump: Dict[str, List[str]]
     code: str = "E032"
 
@@ -719,7 +719,7 @@ def message(self) -> str:
 
 @dataclass
 class DumpBeforeRenameSchema(DebugLevel, Cli, File, Cache):
-    # large value. delay not necessary since every debug level message is logged anyway.
+    # large value. delay creation with fire_event_if.
     dump: Dict[str, List[str]]
     code: str = "E033"
 
@@ -729,7 +729,7 @@ def message(self) -> str:
 
 @dataclass
 class DumpAfterRenameSchema(DebugLevel, Cli, File, Cache):
-    # large value. delay not necessary since every debug level message is logged anyway.
+    # large value. delay creation with fire_event_if.
     dump: Dict[str, List[str]]
     code: str = "E034"
 


Test patch:
diff --git a/test/unit/test_events.py b/test/unit/test_events.py
--- a/test/unit/test_events.py
+++ b/test/unit/test_events.py
@@ -428,3 +428,34 @@ def test_all_serializable(self):
             except TypeError as e:
                 raise Exception(f"{event} is not serializable to json. Originating exception: {e}")
                 
+
+class TestConditionallyFiringEvents(TestCase):
+
+    # tests that fire_event_if only creates the event if the condition is met
+    def test_fire_event_if(self):
+
+        # this class exists to count how many times the stub is called
+        @dataclass
+        class DumpStub():
+            counter = 0
+
+            def dump_graph(self):
+                self.counter = self.counter + 1
+                return dict()
+
+        # initialize to start the counter at 0
+        dump_stub = DumpStub()
+
+        # becaue the condition is false, the event should never be created,
+        # and the stub function should never be called
+        event_funcs.fire_event_if(False, lambda: DumpAfterAddGraph(dump=dump_stub.dump_graph()))
+
+        # assert the stub function was never called
+        self.assertEqual(0, dump_stub.counter)
+
+        # becuase the condition is true, the event should be created and 
+        # the stub function will be called
+        event_funcs.fire_event_if(True, lambda: DumpAfterAddGraph(dump=dump_stub.dump_graph()))
+        
+        # assert that the stub function was called
+        self.assertEqual(1, dump_stub.counter)