Repository: dbt-labs/dbt-core

Issue:
[CT-2139] Don't exclude `Formatting` events from JSON-formatted logs
We have some events that are only for formatting / aesthetic display in the terminal. (Previously, this was called `EmptyLine`; in v1.5 we're renaming it to `Formatting`.) Historically, we've intentionally _excluded_ these events from JSON-formatted structured logs. This makes sense in one way, but it's actively unhelpful for someone who wants _both_ the full structure of JSON logs _and_ the ability to produce their own text-formatted logs at parity with `dbt-core`'s.

_copying from slack_
> @jtcohen6
> While it's great to be opinionated about what & how we display `text`, it does feel weird to me that we're depriving the json log consumer of some detail/flexibility.
> @peterallenwebb:
> On balance, I think we should include the EmptyLine/Formatting events. I infer that the original logic behind excluding them was that they would be confusing/unwanted by consumers, but the existence of this use case suggests otherwise.

### Proposed change

Remove the line from `_stdout_filter` that filters out `Formatting` if we're emitting JSON-formatted logs:
https://github.com/dbt-labs/dbt-core/blob/c952d44ec5c2506995fbad75320acbae49125d3d/core/dbt/events/functions.py#L89

[`TextOnly`](https://github.com/dbt-labs/dbt-core/search?q=TextOnly) achieves the same purpose, for legacy logging _only_. We could remove it now, or remove it when we deprecate legacy logging in ~v1.6~ v1.7.


Discussion:
Add a test looking for formatting to ensure it's getting through.
Let's check with Orchestration & Experience (IDE) before backporting this change

Resolving patch:
diff --git a/core/dbt/events/functions.py b/core/dbt/events/functions.py
--- a/core/dbt/events/functions.py
+++ b/core/dbt/events/functions.py
@@ -2,7 +2,7 @@
 from dbt.events.base_types import BaseEvent, EventLevel, EventMsg
 from dbt.events.eventmgr import EventManager, LoggerConfig, LineFormat, NoFilter, IEventManager
 from dbt.events.helpers import env_secrets, scrub_secrets
-from dbt.events.types import Formatting, Note
+from dbt.events.types import Note
 from dbt.flags import get_flags, ENABLE_LEGACY_LOGGER
 from dbt.logger import GLOBAL_LOGGER, make_log_dir_if_missing
 from functools import partial
@@ -115,9 +115,7 @@ def _stdout_filter(
     line_format: LineFormat,
     msg: EventMsg,
 ) -> bool:
-    return (msg.info.name not in ["CacheAction", "CacheDumpGraph"] or log_cache_events) and not (
-        line_format == LineFormat.Json and type(msg.data) == Formatting
-    )
+    return msg.info.name not in ["CacheAction", "CacheDumpGraph"] or log_cache_events
 
 
 def _get_logfile_config(
@@ -140,10 +138,8 @@ def _get_logfile_config(
 
 
 def _logfile_filter(log_cache_events: bool, line_format: LineFormat, msg: EventMsg) -> bool:
-    return (
-        msg.info.code not in nofile_codes
-        and not (msg.info.name in ["CacheAction", "CacheDumpGraph"] and not log_cache_events)
-        and not (line_format == LineFormat.Json and type(msg.data) == Formatting)
+    return msg.info.code not in nofile_codes and not (
+        msg.info.name in ["CacheAction", "CacheDumpGraph"] and not log_cache_events
     )
 
 


Test patch:
diff --git a/tests/functional/logging/test_logging.py b/tests/functional/logging/test_logging.py
--- a/tests/functional/logging/test_logging.py
+++ b/tests/functional/logging/test_logging.py
@@ -68,6 +68,26 @@ def test_basic(project, logs_dir):
             assert "orig_conn_name" in data and data["orig_conn_name"]
 
 
+def test_formatted_logs(project, logs_dir):
+    # a basic run of dbt with a single model should have 5 `Formatting` events in the json logs
+    results = run_dbt(["--log-format=json", "run"])
+    assert len(results) == 1
+
+    # get log file
+    json_log_file = read_file(logs_dir, "dbt.log")
+    formatted_json_lines = 0
+    for log_line in json_log_file.split("\n"):
+        # skip the empty line at the end
+        if len(log_line) == 0:
+            continue
+        log_dct = json.loads(log_line)
+        log_event = log_dct["info"]["name"]
+        if log_event == "Formatting":
+            formatted_json_lines += 1
+
+    assert formatted_json_lines == 5
+
+
 def test_invalid_event_value(project, logs_dir):
     results = run_dbt(["--log-format=json", "run"])
     assert len(results) == 1