Repository: dbt-labs/dbt-core

Issue:
Scalable in-memory event history
Does something graceful when it runs out of memory (small machines shouldn't explode because of logging), is queryable by a potential future server-wrapper.


Discussion:
@jtcohen6 Do you have thoughts on what it should do when it reaches it's max capacity?  My current thought is that it's simply a FIFO queue that logs a warning stating that the event buffer is full when it hits it's limit and drops the earliest messages.
@iknox-fa Yes, that sounds right to me for the first cut of this. When @nathaniel-may and I chatted about this (at a very high level) last week, he was thinking the initial implementation could just be a good old fashioned `deque`
I'm thinking that even though Python class values are pretty bloated, we'll still want to fill the structure with them instead of rendered log-lines because we want to preserve the ability to choose rendering options (text vs json for example) after the fact.

Resolving patch:
diff --git a/core/dbt/events/functions.py b/core/dbt/events/functions.py
--- a/core/dbt/events/functions.py
+++ b/core/dbt/events/functions.py
@@ -3,7 +3,7 @@
 from datetime import datetime
 import dbt.events.functions as this  # don't worry I hate it too.
 from dbt.events.base_types import Cli, Event, File, ShowException
-from dbt.events.types import T_Event
+from dbt.events.types import EventBufferFull, T_Event
 import dbt.flags as flags
 # TODO this will need to move eventually
 from dbt.logger import SECRET_ENV_PREFIX, make_log_dir_if_missing, GLOBAL_LOGGER
@@ -18,7 +18,13 @@
 import uuid
 from typing import Any, Callable, Dict, List, Optional, Union
 from dataclasses import _FIELD_BASE  # type: ignore[attr-defined]
+from collections import deque
 
+# create the global event history buffer with a max size of 100k records
+# python 3.7 doesn't support type hints on globals, but mypy requires them. hence the ignore.
+# TODO: make the maxlen something configurable from the command line via args(?)
+global EVENT_HISTORY
+EVENT_HISTORY = deque(maxlen=100000)  # type: ignore
 
 # create the global file logger with no configuration
 global FILE_LOG
@@ -248,7 +254,14 @@ def send_exc_to_logger(
 # (i.e. - mutating the event history, printing to stdout, logging
 # to files, etc.)
 def fire_event(e: Event) -> None:
-    # TODO manage history in phase 2:  EVENT_HISTORY.append(e)
+
+    # if and only if the event history deque will be completely filled by this event
+    # fire warning that old events are now being dropped
+    global EVENT_HISTORY
+    if len(EVENT_HISTORY) == ((EVENT_HISTORY.maxlen or 100000) - 1):
+        fire_event(EventBufferFull())
+
+    EVENT_HISTORY.append(e)
 
     # backwards compatibility for plugins that require old logger (dbt-rpc)
     if flags.ENABLE_LEGACY_LOGGER:
diff --git a/core/dbt/events/history.py b/core/dbt/events/history.py
deleted file mode 100644
--- a/core/dbt/events/history.py
+++ /dev/null
@@ -1,7 +0,0 @@
-from dbt.events.base_types import Event
-from typing import List
-
-
-# the global history of events for this session
-# TODO this is naive and the memory footprint is likely far too large.
-EVENT_HISTORY: List[Event] = []
diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py
--- a/core/dbt/events/types.py
+++ b/core/dbt/events/types.py
@@ -2595,6 +2595,14 @@ def fields_to_json(self, val: Any) -> Any:
         return val
 
 
+@dataclass
+class EventBufferFull(WarnLevel, Cli, File):
+    code: str = "Z048"
+
+    def message(self) -> str:
+        return "Internal event buffer full. Earliest events will be dropped (FIFO)."
+
+
 # since mypy doesn't run on every file we need to suggest to mypy that every
 # class gets instantiated. But we don't actually want to run this code.
 # making the conditional `if False` causes mypy to skip it as dead code so
@@ -2840,3 +2848,4 @@ def dump_callable():
     RetryExternalCall(attempt=0, max=0)
     GeneralWarningMsg(msg='', log_fmt='')
     GeneralWarningException(exc=Exception(''), log_fmt='')
+    EventBufferFull()


Test patch:
diff --git a/core/dbt/events/test_types.py b/core/dbt/events/test_types.py
--- a/core/dbt/events/test_types.py
+++ b/core/dbt/events/test_types.py
@@ -57,6 +57,15 @@ def message(self) -> str:
         return f"Integration Test: {self.msg}"
 
 
+@dataclass
+class UnitTestInfo(InfoLevel, Cli):
+    msg: str
+    code: str = "T006"
+
+    def message(self) -> str:
+        return f"Unit Test: {self.msg}"
+
+
 # since mypy doesn't run on every file we need to suggest to mypy that every
 # class gets instantiated. But we don't actually want to run this code.
 # making the conditional `if False` causes mypy to skip it as dead code so
@@ -69,3 +78,4 @@ def message(self) -> str:
     IntegrationTestWarn(msg='')
     IntegrationTestError(msg='')
     IntegrationTestException(msg='')
+    UnitTestInfo(msg='')
diff --git a/test/unit/test_events.py b/test/unit/test_events.py
--- a/test/unit/test_events.py
+++ b/test/unit/test_events.py
@@ -1,7 +1,11 @@
+from dbt import events
+from dbt.events.functions import EVENT_HISTORY, fire_event
+from dbt.events.test_types import UnitTestInfo
 from argparse import Namespace
 from dbt.events import AdapterLogger
 from dbt.events.functions import event_to_serializable_dict
 from dbt.events.types import *
+from dbt.events.test_types import *
 from dbt.events.base_types import Event
 from dbt.events.stubs import _CachedRelation, BaseRelation, _ReferenceKey
 import inspect
@@ -78,6 +82,28 @@ def test_event_codes(self):
                 all_codes.add(event.code)
 
 
+class TestEventBuffer(TestCase):
+
+    # ensure events are populated to the buffer exactly once
+    def test_buffer_populates(self):
+        fire_event(UnitTestInfo(msg="Test Event 1"))
+        fire_event(UnitTestInfo(msg="Test Event 2"))
+        self.assertTrue(
+            EVENT_HISTORY.count(UnitTestInfo(msg='Test Event 1', code='T006')) == 1
+        )
+
+    # ensure events drop from the front of the buffer when buffer maxsize is reached
+    # TODO commenting out till we can make this not spit out 100k log lines.
+    # def test_buffer_FIFOs(self):
+    #     for n in range(0,100001):
+    #         fire_event(UnitTestInfo(msg=f"Test Event {n}"))
+    #     self.assertTrue(
+    #         EVENT_HISTORY.count(EventBufferFull(code='Z048')) == 1
+    #     )
+    #     self.assertTrue(
+    #         EVENT_HISTORY.count(UnitTestInfo(msg='Test Event 1', code='T006')) == 0
+    #     )
+
 def dump_callable():
     return dict()
 
@@ -333,7 +359,14 @@ def dump_callable():
     SQlRunnerException(Exception('')),
     DropRelation(''),
     PartialParsingProjectEnvVarsChanged(),
-    RegistryProgressGETResponse('', '')
+    RegistryProgressGETResponse('', ''),
+    IntegrationTestDebug(''),
+    IntegrationTestInfo(''),
+    IntegrationTestWarn(''),
+    IntegrationTestError(''),
+    IntegrationTestException(''),
+    EventBufferFull(),
+    UnitTestInfo('')
 ]
 
 
@@ -359,4 +392,4 @@ def test_all_serializable(self):
                 json.dumps(d)
             except TypeError as e:
                 raise Exception(f"{event} is not serializable to json. Originating exception: {e}")
-                
\ No newline at end of file
+