Repository: dbt-labs/dbt-core
Pull request: Add Snowplow tracking for behavior flags
resolves #10552

### Problem

We need to track deprecated states of behavior flags in Snowplow so that we get telemetry for both Cloud and Core users.

### Solution

Handle the event that is already firing for logging with a callback that tracks the event in Snowplow.

### Checklist

- [x] I have read [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md) and understand what's expected of me.
- [ ] I have run this code in development, and it appears to resolve the stated issue.
- [x] This PR includes tests, or tests are not required or relevant for this PR.
- [x] This PR has no interface changes (e.g., macros, CLI, logs, JSON artifacts, config files, adapter interface, etc.) or this PR has already received feedback and approval from Product or DX.
- [x] This PR includes [type annotations](https://docs.python.org/3/library/typing.html) for new and modified functions.


Problem statement:
[Behavior Flags] Track behavior flags in Snowplow
### Housekeeping

- [X] I am a maintainer of dbt-core

## Short description

As a maintainer, I need to track what proportion of users are using legacy behavior, so that I know when it's safe to retire that behavior. We already fire a warning event when the flag evaluates to `False`. We need to track how often this occurs for each flag as a count of invocation id.

```[tasklist]
## Acceptance criteria
- [x] Any time a behavior flag evaluates to `False`, this gets tracked in our Snowplow instance
- [x] No events for behavior flags are tracked if the user has turned off anonymous tracking (this is existing behavior for all other tracking and should continue to be observed)
```

```[tasklist]
## Suggested testing
- [x] Create a scenario where the flag evaluates to `False`, verify that the callback is fired
- [x] Create a scenario where the flag evaluates to `True`, verify that the callback is not fired
- [x] Fire an event other than a `BehaviorDeprecationEvent` and verify that the callback is not fired
```

### Impact to other teams

There's no immediate impact to any team. However, it would make sense for existing behavior flags in `dbt-core` to migrate to the new framework at some point.

### Will backports be required?

No backports are required; this is new functionality.

Hints from issue discussion:

Commit message: update tests for new callback

Patch to review:
diff --git a/.changes/unreleased/Under the Hood-20240911-162730.yaml b/.changes/unreleased/Under the Hood-20240911-162730.yaml
new file mode 100644
index 00000000000..0d35aeb5262
--- /dev/null
+++ b/.changes/unreleased/Under the Hood-20240911-162730.yaml	
@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Add Snowplow tracking for behavior flag deprecations
+time: 2024-09-11T16:27:30.293832-04:00
+custom:
+  Author: mikealfare
+  Issue: "10552"
diff --git a/core/dbt/events/logging.py b/core/dbt/events/logging.py
index f0bef3ae497..e12ff2d5e4e 100644
--- a/core/dbt/events/logging.py
+++ b/core/dbt/events/logging.py
@@ -2,8 +2,10 @@
 from functools import partial
 from typing import Callable, List
 
+from dbt.tracking import track_behavior_deprecation_warn
 from dbt_common.events.base_types import EventLevel, EventMsg
 from dbt_common.events.event_manager_client import (
+    add_callback_to_manager,
     add_logger_to_manager,
     cleanup_event_logger,
     get_event_manager,
@@ -68,6 +70,7 @@ def setup_event_logger(flags, callbacks: List[Callable[[EventMsg], None]] = [])
     make_log_dir_if_missing(flags.LOG_PATH)
     event_manager = get_event_manager()
     event_manager.callbacks = callbacks.copy()
+    add_callback_to_manager(track_behavior_deprecation_warn)
 
     if flags.LOG_LEVEL != "none":
         line_format = _line_format_from_str(flags.LOG_FORMAT, LineFormat.PlainText)
diff --git a/core/dbt/tracking.py b/core/dbt/tracking.py
index 880243e4d6e..c2a53bd4b46 100644
--- a/core/dbt/tracking.py
+++ b/core/dbt/tracking.py
@@ -25,7 +25,8 @@
     SendingEvent,
     TrackingInitializeFailure,
 )
-from dbt_common.events.functions import fire_event, get_invocation_id
+from dbt_common.events.base_types import EventMsg
+from dbt_common.events.functions import fire_event, get_invocation_id, msg_to_dict
 from dbt_common.exceptions import NotImplementedError
 
 sp_logger.setLevel(100)
@@ -36,6 +37,7 @@
 
 ADAPTER_INFO_SPEC = "iglu:com.dbt/adapter_info/jsonschema/1-0-1"
 DEPRECATION_WARN_SPEC = "iglu:com.dbt/deprecation_warn/jsonschema/1-0-0"
+BEHAVIOR_DEPRECATION_WARN_SPEC = "iglu:com.dbt/behavior_deprecation_warn/jsonschema/1-0-0"
 EXPERIMENTAL_PARSER = "iglu:com.dbt/experimental_parser/jsonschema/1-0-0"
 INVOCATION_ENV_SPEC = "iglu:com.dbt/invocation_env/jsonschema/1-0-0"
 INVOCATION_SPEC = "iglu:com.dbt/invocation/jsonschema/1-0-2"
@@ -364,6 +366,20 @@ def track_deprecation_warn(options):
     )
 
 
+def track_behavior_deprecation_warn(msg: EventMsg) -> None:
+    if msg.info.name != "BehaviorDeprecationEvent" or active_user is None:
+        return
+
+    context = [SelfDescribingJson(BEHAVIOR_DEPRECATION_WARN_SPEC, msg_to_dict(msg))]
+    track(
+        active_user,
+        category="dbt",
+        action=msg.info.name,
+        label=get_invocation_id(),
+        context=context,
+    )
+
+
 def track_invocation_end(invocation_context, result_type=None):
     data = {"progress": "end", "result_type": result_type, "result": None}
     data.update(invocation_context)
diff --git a/tests/unit/events/test_logging.py b/tests/unit/events/test_logging.py
index 00284ecab78..b2077b64793 100644
--- a/tests/unit/events/test_logging.py
+++ b/tests/unit/events/test_logging.py
@@ -23,7 +23,7 @@ def test_clears_preexisting_event_manager_state(self) -> None:
 
         setup_event_logger(get_flags())
         assert len(manager.loggers) == 0
-        assert len(manager.callbacks) == 0
+        assert len(manager.callbacks) == 1  # snowplow tracker for behavior flags
 
     def test_specify_max_bytes(
         self,
diff --git a/tests/unit/test_behavior_flags.py b/tests/unit/test_behavior_flags.py
new file mode 100644
index 00000000000..0188480ce02
--- /dev/null
+++ b/tests/unit/test_behavior_flags.py
@@ -0,0 +1,59 @@
+import pytest
+
+from dbt.tracking import (
+    disable_tracking,
+    initialize_from_flags,
+    track_behavior_deprecation_warn,
+)
+from dbt_common.behavior_flags import Behavior
+from dbt_common.events.event_manager_client import (
+    add_callback_to_manager,
+    cleanup_event_logger,
+)
+
+
+@pytest.fixture
+def snowplow_tracker(mocker):
+    # initialize `active_user` without writing the cookie to disk
+    initialize_from_flags(True, "")
+    mocker.patch("dbt.tracking.User.set_cookie").return_value = {"id": 42}
+
+    # add the relevant callback to the event manager
+    add_callback_to_manager(track_behavior_deprecation_warn)
+
+    # don't make a call, catch the request
+    snowplow_tracker = mocker.patch("dbt.tracking.tracker.track_struct_event")
+
+    yield snowplow_tracker
+
+    # teardown
+    cleanup_event_logger()
+    disable_tracking()
+
+
+def test_false_evaluation_triggers_snowplow_tracking(snowplow_tracker):
+    behavior = Behavior([{"name": "my_flag", "default": False}], {})
+    if behavior.my_flag:
+        # trigger a False evaluation
+        assert False, "This flag should evaluate to false and skip this line"
+    assert snowplow_tracker.called
+
+
+def test_true_evaluation_does_not_trigger_snowplow_tracking(snowplow_tracker):
+    behavior = Behavior([{"name": "my_flag", "default": True}], {})
+    if behavior.my_flag:
+        pass
+    else:
+        # trigger a True evaluation
+        assert False, "This flag should evaluate to false and skip this line"
+    assert not snowplow_tracker.called
+
+
+def test_false_evaluation_does_not_trigger_snowplow_tracking_when_disabled(snowplow_tracker):
+    disable_tracking()
+
+    behavior = Behavior([{"name": "my_flag", "default": False}], {})
+    if behavior.my_flag:
+        # trigger a False evaluation
+        assert False, "This flag should evaluate to false and skip this line"
+    assert not snowplow_tracker.called


Code review comments:
diff_hunk: @@ -68,6 +70,7 @@ def setup_event_logger(flags, callbacks: List[Callable[[EventMsg], None]] = [])
     make_log_dir_if_missing(flags.LOG_PATH)
     event_manager = get_event_manager()
     event_manager.callbacks = callbacks.copy()
+    add_callback_to_manager(track_behavior_deprecation_warn)
path: core/dbt/events/logging.py
text: @author:
I think we want snowplow tracking for behavior flags regardless of the log setting. It should only be turned off when the user turns off tracking. However, this is handled outside of the event manager. The downside is that tests need to be updated for this, though it appears to only be one test.