Repository: DataDog/datadog-agent
Pull request #35280: Unset'ing a setting removes it from AllSettings
Author: login: dustmop
Labels: name: changelog/no-changelog
description: No changelog entry needed
name: qa/done
description: QA done before merge and regressions are covered by tests
name: medium review
description: PR review might take time
name: team/agent-configuration
Created: 2025-03-19T18:15:49Z | Merged: 2025-03-25T16:32:42Z

### What does this PR do?

Unset'ing a setting removes it from AllSettings
    
For settings that have no default, but then get assigned a value later, calling Unset will remove that setting entirely. The same is true for SetWithoutSource when passed a nil value.

### Motivation

Match behavior with Viper.

### Describe how you validated your changes

Unit test added to viper and nodetreemodel to validate the behavior works in both.

### Possible Drawbacks / Trade-offs

### Additional Notes



Diff:
diff --git a/pkg/config/nodetreemodel/config.go b/pkg/config/nodetreemodel/config.go
index 368c11c9e4a7..71df3f417f67 100644
--- a/pkg/config/nodetreemodel/config.go
+++ b/pkg/config/nodetreemodel/config.go
@@ -282,16 +282,20 @@ func (c *ntmConfig) UnsetForSource(key string, source model.Source) {
 	}
 
 	// Find what the previous value used to be, based upon the previous source
-	prevNode, err := c.findPreviousSourceNode(key, source)
-	if err != nil {
-		return
-	}
+	prevNode, findPreviousSourceError := c.findPreviousSourceNode(key, source)
 
 	// Get the parent node of the leaf we're unsetting
 	parentNode, childName, err = c.parentOfNode(c.root, key)
 	if err != nil {
 		return
 	}
+
+	// If there was no previous source with a node of this name, simply remove it from the parent
+	if findPreviousSourceError != nil {
+		parentNode.RemoveChild(childName)
+		return
+	}
+
 	// Replace the child with the node from the previous layer
 	parentNode.InsertChildNode(childName, prevNode.Clone())
 }
diff --git a/pkg/config/nodetreemodel/config_test.go b/pkg/config/nodetreemodel/config_test.go
index ffdaaea1a7fe..018f86e13eae 100644
--- a/pkg/config/nodetreemodel/config_test.go
+++ b/pkg/config/nodetreemodel/config_test.go
@@ -619,6 +619,45 @@ func TestUnsetForSource(t *testing.T) {
 	assert.Equal(t, expect, txt)
 }
 
+func TestUnsetForSourceRemoveIfNotPrevious(t *testing.T) {
+	cfg := NewConfig("test", "TEST", strings.NewReplacer(".", "_"))
+	cfg.BindEnv("api_key")
+	cfg.BuildSchema()
+
+	// api_key is not in the config (does not have a default value)
+	assert.Equal(t, "", cfg.GetString("api_key"))
+	_, found := cfg.AllSettings()["api_key"]
+	assert.False(t, found)
+
+	cfg.Set("api_key", "0123456789abcdef", model.SourceAgentRuntime)
+
+	// api_key is set
+	assert.Equal(t, "0123456789abcdef", cfg.GetString("api_key"))
+	_, found = cfg.AllSettings()["api_key"]
+	assert.True(t, found)
+
+	cfg.UnsetForSource("api_key", model.SourceAgentRuntime)
+
+	// api_key is unset, which means its not listed in AllSettings
+	assert.Equal(t, "", cfg.GetString("api_key"))
+	_, found = cfg.AllSettings()["api_key"]
+	assert.False(t, found)
+
+	cfg.SetWithoutSource("api_key", "0123456789abcdef")
+
+	// api_key is set
+	assert.Equal(t, "0123456789abcdef", cfg.GetString("api_key"))
+	_, found = cfg.AllSettings()["api_key"]
+	assert.True(t, found)
+
+	cfg.UnsetForSource("api_key", model.SourceUnknown)
+
+	// api_key is unset again, should not appear in AllSettings
+	assert.Equal(t, "", cfg.GetString("api_key"))
+	_, found = cfg.AllSettings()["api_key"]
+	assert.False(t, found)
+}
+
 func TestMergeFleetPolicy(t *testing.T) {
 	config := NewConfig("test", "TEST", strings.NewReplacer(".", "_")) // nolint: forbidigo
 	config.SetConfigType("yaml")
diff --git a/pkg/config/setup/config.go b/pkg/config/setup/config.go
index 1f7d4ec512f4..031e341e64e8 100644
--- a/pkg/config/setup/config.go
+++ b/pkg/config/setup/config.go
@@ -263,22 +263,18 @@ func init() {
 	//                nodetreemodel internals
 	// - other:       Use viper for the config
 	if envvar == "enable" {
-		datadog = nodetreemodel.NewConfig("datadog", "DD", strings.NewReplacer(".", "_")) // nolint: forbidigo // legit use case
+		datadog = nodetreemodel.NewConfig("datadog", "DD", strings.NewReplacer(".", "_"))          // nolint: forbidigo // legit use case
+		systemProbe = nodetreemodel.NewConfig("system-probe", "DD", strings.NewReplacer(".", "_")) // nolint: forbidigo // legit use case
 	} else if envvar == "tee" {
 		viperConfig := viperconfig.NewConfig("datadog", "DD", strings.NewReplacer(".", "_"))      // nolint: forbidigo // legit use case
 		nodetreeConfig := nodetreemodel.NewConfig("datadog", "DD", strings.NewReplacer(".", "_")) // nolint: forbidigo // legit use case
 		datadog = teeconfig.NewTeeConfig(viperConfig, nodetreeConfig)
-	} else {
-		datadog = viperconfig.NewConfig("datadog", "DD", strings.NewReplacer(".", "_")) // nolint: forbidigo // legit use case
-	}
 
-	if envvar == "enable" {
-		systemProbe = nodetreemodel.NewConfig("system-probe", "DD", strings.NewReplacer(".", "_")) // nolint: forbidigo // legit use case
-	} else if envvar == "tee" {
-		viperConfig := viperconfig.NewConfig("system-probe", "DD", strings.NewReplacer(".", "_"))      // nolint: forbidigo // legit use case
-		nodetreeConfig := nodetreemodel.NewConfig("system-probe", "DD", strings.NewReplacer(".", "_")) // nolint: forbidigo // legit use case
+		viperConfig = viperconfig.NewConfig("system-probe", "DD", strings.NewReplacer(".", "_"))      // nolint: forbidigo // legit use case
+		nodetreeConfig = nodetreemodel.NewConfig("system-probe", "DD", strings.NewReplacer(".", "_")) // nolint: forbidigo // legit use case
 		systemProbe = teeconfig.NewTeeConfig(viperConfig, nodetreeConfig)
 	} else {
+		datadog = viperconfig.NewConfig("datadog", "DD", strings.NewReplacer(".", "_"))          // nolint: forbidigo // legit use case
 		systemProbe = viperconfig.NewConfig("system-probe", "DD", strings.NewReplacer(".", "_")) // nolint: forbidigo // legit use case
 	}
 
diff --git a/pkg/config/setup/config_change_checker_test.go b/pkg/config/setup/config_change_checker_test.go
index 8fd8f2cf7035..fac6dda6d489 100644
--- a/pkg/config/setup/config_change_checker_test.go
+++ b/pkg/config/setup/config_change_checker_test.go
@@ -8,6 +8,7 @@ package setup
 import (
 	"testing"
 
+	"github.com/DataDog/datadog-agent/pkg/config/model"
 	"github.com/stretchr/testify/require"
 )
 
@@ -27,6 +28,6 @@ func TestChangeChecker(t *testing.T) {
 func assertConfigChangeDetected(r *require.Assertions, checker *ChangeChecker, key string, value interface{}) {
 	Datadog().SetWithoutSource(key, value)
 	r.True(checker.HasChanged())
-	Datadog().SetWithoutSource(key, nil)
+	Datadog().UnsetForSource(key, model.SourceUnknown)
 	r.False(checker.HasChanged())
 }
diff --git a/pkg/config/viperconfig/viper_test.go b/pkg/config/viperconfig/viper_test.go
index b109437851e5..fc8506b20aca 100644
--- a/pkg/config/viperconfig/viper_test.go
+++ b/pkg/config/viperconfig/viper_test.go
@@ -383,3 +383,42 @@ func TestListenersUnsetForSource(t *testing.T) {
 
 	assert.Equal(t, []string{"info", "debug", "info"}, logLevels)
 }
+
+func TestUnsetForSourceRemoveIfNotPrevious(t *testing.T) {
+	cfg := NewConfig("test", "TEST", strings.NewReplacer(".", "_"))
+	cfg.BindEnv("api_key")
+	cfg.BuildSchema()
+
+	// api_key is not in the config (does not have a default value)
+	assert.Equal(t, "", cfg.GetString("api_key"))
+	_, found := cfg.AllSettings()["api_key"]
+	assert.False(t, found)
+
+	cfg.Set("api_key", "0123456789abcdef", model.SourceAgentRuntime)
+
+	// api_key is set
+	assert.Equal(t, "0123456789abcdef", cfg.GetString("api_key"))
+	_, found = cfg.AllSettings()["api_key"]
+	assert.True(t, found)
+
+	cfg.UnsetForSource("api_key", model.SourceAgentRuntime)
+
+	// api_key is unset, which means its not listed in AllSettings
+	assert.Equal(t, "", cfg.GetString("api_key"))
+	_, found = cfg.AllSettings()["api_key"]
+	assert.False(t, found)
+
+	cfg.SetWithoutSource("api_key", "0123456789abcdef")
+
+	// api_key is set
+	assert.Equal(t, "0123456789abcdef", cfg.GetString("api_key"))
+	_, found = cfg.AllSettings()["api_key"]
+	assert.True(t, found)
+
+	cfg.UnsetForSource("api_key", model.SourceUnknown)
+
+	// api_key is unset again, should not appear in AllSettings
+	assert.Equal(t, "", cfg.GetString("api_key"))
+	_, found = cfg.AllSettings()["api_key"]
+	assert.False(t, found)
+}


Reviews:
user: login: hush-hush
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-20T10:24:56Z
user: login: dustmop
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-21T14:18:17Z
user: login: dustmop
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-21T14:19:40Z
user: login: hush-hush
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-25T10:27:50Z

Review comments:
diff_hunk: @@ -272,7 +272,15 @@ func init() {
 		datadog = viperconfig.NewConfig("datadog", "DD", strings.NewReplacer(".", "_")) // nolint: forbidigo // legit use case
 	}
 
-	systemProbe = viperconfig.NewConfig("system-probe", "DD", strings.NewReplacer(".", "_")) // nolint: forbidigo // legit use case
+	if envvar == "enable" {
+		systemProbe = nodetreemodel.NewConfig("system-probe", "DD", strings.NewReplacer(".", "_")) // nolint: forbidigo // legit use case
+	} else if envvar == "tee" {
+		viperConfig := viperconfig.NewConfig("system-probe", "DD", strings.NewReplacer(".", "_"))      // nolint: forbidigo // legit use case
+		nodetreeConfig := nodetreemodel.NewConfig("system-probe", "DD", strings.NewReplacer(".", "_")) // nolint: forbidigo // legit use case
+		systemProbe = teeconfig.NewTeeConfig(viperConfig, nodetreeConfig)
+	} else {
+		systemProbe = viperconfig.NewConfig("system-probe", "DD", strings.NewReplacer(".", "_")) // nolint: forbidigo // legit use case
+	}
path: pkg/config/setup/config.go
user: login: hush-hush
body: Why not reuse the if/else above ?
created_at: 2025-03-20T10:21:37Z
author_association: MEMBER
diff_hunk: @@ -215,6 +215,10 @@ func (c *ntmConfig) Set(key string, newValue interface{}, source model.Source) {
 
 // SetWithoutSource assigns the value to the given key using source Unknown
 func (c *ntmConfig) SetWithoutSource(key string, value interface{}) {
+	if value == nil {
+		c.UnsetForSource(key, model.SourceUnknown)
+		return
+	}
path: pkg/config/nodetreemodel/config.go
user: login: hush-hush
body: Do we know where this is used ? Could is be possible to migrate use cases to UnsetForSource instead ?
created_at: 2025-03-20T10:22:34Z
author_association: MEMBER
diff_hunk: @@ -272,7 +272,15 @@ func init() {
 		datadog = viperconfig.NewConfig("datadog", "DD", strings.NewReplacer(".", "_")) // nolint: forbidigo // legit use case
 	}
 
-	systemProbe = viperconfig.NewConfig("system-probe", "DD", strings.NewReplacer(".", "_")) // nolint: forbidigo // legit use case
+	if envvar == "enable" {
+		systemProbe = nodetreemodel.NewConfig("system-probe", "DD", strings.NewReplacer(".", "_")) // nolint: forbidigo // legit use case
+	} else if envvar == "tee" {
+		viperConfig := viperconfig.NewConfig("system-probe", "DD", strings.NewReplacer(".", "_"))      // nolint: forbidigo // legit use case
+		nodetreeConfig := nodetreemodel.NewConfig("system-probe", "DD", strings.NewReplacer(".", "_")) // nolint: forbidigo // legit use case
+		systemProbe = teeconfig.NewTeeConfig(viperConfig, nodetreeConfig)
+	} else {
+		systemProbe = viperconfig.NewConfig("system-probe", "DD", strings.NewReplacer(".", "_")) // nolint: forbidigo // legit use case
+	}
path: pkg/config/setup/config.go
user: login: dustmop
body: Good point, done.
created_at: 2025-03-21T14:18:17Z
author_association: CONTRIBUTOR
diff_hunk: @@ -215,6 +215,10 @@ func (c *ntmConfig) Set(key string, newValue interface{}, source model.Source) {
 
 // SetWithoutSource assigns the value to the given key using source Unknown
 func (c *ntmConfig) SetWithoutSource(key string, value interface{}) {
+	if value == nil {
+		c.UnsetForSource(key, model.SourceUnknown)
+		return
+	}
path: pkg/config/nodetreemodel/config.go
user: login: dustmop
body: I was initially concerned with having feature drift, but then again the method `SetWithoutSource` should only be used in tests, and there only a small number of tests that rely on this behavior of how `AllSettings` acts. I updated the tests.
created_at: 2025-03-21T14:19:40Z
author_association: CONTRIBUTOR

PR comments:
user: login: agent-platform-auto-pr[bot]
created_at: 2025-03-19T18:37:16Z
body: ## Uncompressed package size comparison

Comparison with [ancestor](https://github.com/DataDog/datadog-agent/commit/b30326f1879c01044d0fc98dd2023acd119bdab3) `b30326f1879c01044d0fc98dd2023acd119bdab3`


<details>
  <summary> Diff per package </summary>

|package|diff|status|size|ancestor|threshold|
|--|--|--|--|--|--|
|datadog-agent-amd64-deb|0.00MB|✅|807.14MB|807.14MB|0.50MB|
|datadog-agent-x86_64-rpm|0.00MB|✅|816.98MB|816.98MB|0.50MB|
|datadog-agent-x86_64-suse|0.00MB|✅|816.98MB|816.98MB|0.50MB|
|datadog-agent-arm64-deb|0.00MB|✅|798.14MB|798.14MB|0.50MB|
|datadog-agent-aarch64-rpm|0.00MB|✅|807.97MB|807.97MB|0.50MB|
|datadog-dogstatsd-amd64-deb|0.00MB|✅|39.48MB|39.48MB|0.50MB|
|datadog-dogstatsd-x86_64-rpm|0.00MB|✅|39.56MB|39.56MB|0.50MB|
|datadog-dogstatsd-x86_64-suse|0.00MB|✅|39.56MB|39.56MB|0.50MB|
|datadog-dogstatsd-arm64-deb|0.00MB|✅|38.00MB|38.00MB|0.50MB|
|datadog-heroku-agent-amd64-deb|0.00MB|✅|442.27MB|442.27MB|0.50MB|
|datadog-iot-agent-amd64-deb|0.00MB|✅|61.12MB|61.12MB|0.50MB|
|datadog-iot-agent-x86_64-rpm|0.00MB|✅|61.19MB|61.19MB|0.50MB|
|datadog-iot-agent-x86_64-suse|0.00MB|✅|61.19MB|61.19MB|0.50MB|
|datadog-iot-agent-arm64-deb|0.00MB|✅|58.41MB|58.41MB|0.50MB|
|datadog-iot-agent-aarch64-rpm|0.00MB|✅|58.49MB|58.49MB|0.50MB|

</details>

## Decision
✅ Passed



author_association: CONTRIBUTOR
user: login: agent-platform-auto-pr[bot]
created_at: 2025-03-19T18:38:46Z
body: ## Test changes on VM

Use this command from [test-infra-definitions](https://github.com/DataDog/test-infra-definitions) to manually test this PR changes on a VM:
```sh
dda inv aws.create-vm --pipeline-id=59892185 --os-family=ubuntu
```
Note: This applies to commit **ab18b0e9**
author_association: CONTRIBUTOR
user: login: agent-platform-auto-pr[bot]
created_at: 2025-03-19T18:40:17Z
body: ## Static quality checks ✅

Please find below the results from static quality gates


<details>
<summary>Successful checks</summary>

### Info

|Result|Quality gate|On disk size|On disk size limit|On wire size|On wire size limit|
|----|----|----|----|----|----|
|✅|static_quality_gate_agent_deb_amd64|781.27 MiB|801.8 MiB|190.73 MiB|202.62 MiB|
|✅|static_quality_gate_agent_deb_arm64|772.72 MiB|793.14 MiB|173.52 MiB|184.51 MiB|
|✅|static_quality_gate_agent_rpm_amd64|781.23 MiB|801.79 MiB|193.22 MiB|205.03 MiB|
|✅|static_quality_gate_agent_rpm_arm64|772.77 MiB|793.09 MiB|174.27 MiB|186.44 MiB|
|✅|static_quality_gate_agent_suse_amd64|781.34 MiB|801.81 MiB|193.22 MiB|205.03 MiB|
|✅|static_quality_gate_agent_suse_arm64|772.63 MiB|793.14 MiB|174.27 MiB|186.44 MiB|
|✅|static_quality_gate_dogstatsd_deb_amd64|37.73 MiB|47.67 MiB|9.78 MiB|19.78 MiB|
|✅|static_quality_gate_dogstatsd_deb_arm64|36.32 MiB|46.27 MiB|8.48 MiB|18.49 MiB|
|✅|static_quality_gate_dogstatsd_rpm_amd64|37.73 MiB|47.67 MiB|9.79 MiB|19.79 MiB|
|✅|static_quality_gate_dogstatsd_suse_amd64|37.73 MiB|47.67 MiB|9.79 MiB|19.79 MiB|
|✅|static_quality_gate_iot_agent_deb_amd64|58.37 MiB|69.0 MiB|14.74 MiB|24.8 MiB|
|✅|static_quality_gate_iot_agent_deb_arm64|55.78 MiB|66.4 MiB|12.73 MiB|22.8 MiB|
|✅|static_quality_gate_iot_agent_rpm_amd64|58.37 MiB|69.0 MiB|14.76 MiB|24.8 MiB|
|✅|static_quality_gate_iot_agent_rpm_arm64|55.78 MiB|66.4 MiB|12.77 MiB|22.8 MiB|
|✅|static_quality_gate_iot_agent_suse_amd64|58.37 MiB|69.0 MiB|14.76 MiB|24.8 MiB|
|✅|static_quality_gate_docker_agent_amd64|865.94 MiB|886.12 MiB|291.35 MiB|304.21 MiB|
|✅|static_quality_gate_docker_agent_arm64|880.63 MiB|900.79 MiB|277.82 MiB|290.47 MiB|
|✅|static_quality_gate_docker_agent_jmx_amd64|1.04 GiB|1.06 GiB|366.46 MiB|379.33 MiB|
|✅|static_quality_gate_docker_agent_jmx_arm64|1.04 GiB|1.06 GiB|348.9 MiB|361.55 MiB|
|✅|static_quality_gate_docker_dogstatsd_amd64|45.87 MiB|55.78 MiB|17.3 MiB|27.28 MiB|
|✅|static_quality_gate_docker_dogstatsd_arm64|44.48 MiB|54.45 MiB|16.16 MiB|26.16 MiB|
|✅|static_quality_gate_docker_cluster_agent_amd64|262.79 MiB|274.78 MiB|105.52 MiB|116.28 MiB|
|✅|static_quality_gate_docker_cluster_agent_arm64|278.75 MiB|290.82 MiB|100.36 MiB|111.12 MiB|

</details>

author_association: CONTRIBUTOR
user: login: cit-pr-commenter[bot]
created_at: 2025-03-19T18:57:27Z
body: ## Regression Detector

# Regression Detector Results

[Metrics dashboard](https://app.datadoghq.com/dashboard/ykh-ua8-vcu/SMP-Regression-Detector-Metrics?fromUser=true&refresh_mode=paused&tpl_var_run-id%5B0%5D=e3230600-6dd7-4a3f-86f0-c5026742655f&view=spans&from_ts=1742912363000&to_ts=1742912963000&live=false)  
[Target profiles](https://app.datadoghq.com/profiling/explorer?query=env%3Asingle-machine-performance%20service%3Adatadog-agent%20job_id%3Ae3230600-6dd7-4a3f-86f0-c5026742655f&agg_m=count&agg_m_source=base&agg_t=count&fromUser=false&viz=stream&start=1742905163000&end=1742916563000&paused=true)  
Run ID: e3230600-6dd7-4a3f-86f0-c5026742655f

Baseline: b30326f1879c01044d0fc98dd2023acd119bdab3
Comparison: ab18b0e984f7e80097950f357a55ec00824ad7ec
[Diff](b30326f1879c01044d0fc98dd2023acd119bdab3...ab18b0e984f7e80097950f357a55ec00824ad7ec)


## Optimization Goals: ✅ No significant changes detected


<details>
<summary><h2>
Fine details of change detection per experiment
</h2></summary>

| perf | experiment                                   | goal               | Δ mean % | Δ mean % CI    | trials | links                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
|------|----------------------------------------------|--------------------|----------|----------------|--------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| ➖   | tcp_syslog_to_blackhole                      | ingress throughput | +2.11    | [+1.99, +2.24] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Atcp_syslog_to_blackhole%20run_id%3Ae3230600-6dd7-4a3f-86f0-c5026742655f&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742905163000&to_ts=1742916563000&live=false)                                                                                                                                                                                                                                                                                                                                                                    |
| ➖   | file_tree                                    | memory utilization | +0.70    | [+0.57, +0.83] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_tree%20run_id%3Ae3230600-6dd7-4a3f-86f0-c5026742655f&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742905163000&to_ts=1742916563000&live=false)                                                                                                                                                                                                                                                                                                                                                                                  |
| ➖   | otlp_ingest_logs                             | memory utilization | +0.53    | [+0.37, +0.70] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Aotlp_ingest_logs%20run_id%3Ae3230600-6dd7-4a3f-86f0-c5026742655f&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742905163000&to_ts=1742916563000&live=false)                                                                                                                                                                                                                                                                                                                                                                           |
| ➖   | quality_gate_idle                            | memory utilization | +0.49    | [+0.43, +0.55] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Aquality_gate_idle%20run_id%3Ae3230600-6dd7-4a3f-86f0-c5026742655f&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742905163000&to_ts=1742916563000&live=false) [bounds checks dashboard](https://app.datadoghq.com/dashboard/vz3-jd5-bdi?fromUser=true&refresh_mode=paused&tpl_var_experiment%5B0%5D=quality_gate_idle&tpl_var_job_id%5B0%5D=e3230600-6dd7-4a3f-86f0-c5026742655f&tpl_var_run-id%5B0%5D=e3230600-6dd7-4a3f-86f0-c5026742655f&view=spans&from_ts=1742912363000&to_ts=1742912963000&live=false)                           |
| ➖   | uds_dogstatsd_20mb_12k_contexts_20_senders   | memory utilization | +0.35    | [+0.32, +0.39] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Auds_dogstatsd_20mb_12k_contexts_20_senders%20run_id%3Ae3230600-6dd7-4a3f-86f0-c5026742655f&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742905163000&to_ts=1742916563000&live=false)                                                                                                                                                                                                                                                                                                                                                 |
| ➖   | quality_gate_logs                            | % cpu utilization  | +0.26    | [-2.58, +3.10] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Aquality_gate_logs%20run_id%3Ae3230600-6dd7-4a3f-86f0-c5026742655f&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742905163000&to_ts=1742916563000&live=false)                                                                                                                                                                                                                                                                                                                                                                          |
| ➖   | file_to_blackhole_1000ms_latency             | egress throughput  | +0.17    | [-0.61, +0.94] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_1000ms_latency%20run_id%3Ae3230600-6dd7-4a3f-86f0-c5026742655f&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742905163000&to_ts=1742916563000&live=false)                                                                                                                                                                                                                                                                                                                                                           |
| ➖   | file_to_blackhole_1000ms_latency_linear_load | egress throughput  | +0.08    | [-0.39, +0.55] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_1000ms_latency_linear_load%20run_id%3Ae3230600-6dd7-4a3f-86f0-c5026742655f&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742905163000&to_ts=1742916563000&live=false)                                                                                                                                                                                                                                                                                                                                               |
| ➖   | uds_dogstatsd_to_api_cpu                     | % cpu utilization  | +0.05    | [-0.77, +0.87] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Auds_dogstatsd_to_api_cpu%20run_id%3Ae3230600-6dd7-4a3f-86f0-c5026742655f&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742905163000&to_ts=1742916563000&live=false)                                                                                                                                                                                                                                                                                                                                                                   |
| ➖   | file_to_blackhole_500ms_latency              | egress throughput  | +0.04    | [-0.74, +0.82] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_500ms_latency%20run_id%3Ae3230600-6dd7-4a3f-86f0-c5026742655f&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742905163000&to_ts=1742916563000&live=false)                                                                                                                                                                                                                                                                                                                                                            |
| ➖   | file_to_blackhole_0ms_latency_http2          | egress throughput  | +0.03    | [-0.77, +0.84] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_0ms_latency_http2%20run_id%3Ae3230600-6dd7-4a3f-86f0-c5026742655f&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742905163000&to_ts=1742916563000&live=false)                                                                                                                                                                                                                                                                                                                                                        |
| ➖   | tcp_dd_logs_filter_exclude                   | ingress throughput | +0.01    | [-0.02, +0.04] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Atcp_dd_logs_filter_exclude%20run_id%3Ae3230600-6dd7-4a3f-86f0-c5026742655f&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742905163000&to_ts=1742916563000&live=false)                                                                                                                                                                                                                                                                                                                                                                 |
| ➖   | file_to_blackhole_100ms_latency              | egress throughput  | +0.00    | [-0.68, +0.68] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_100ms_latency%20run_id%3Ae3230600-6dd7-4a3f-86f0-c5026742655f&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742905163000&to_ts=1742916563000&live=false)                                                                                                                                                                                                                                                                                                                                                            |
| ➖   | uds_dogstatsd_to_api                         | ingress throughput | -0.01    | [-0.28, +0.26] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Auds_dogstatsd_to_api%20run_id%3Ae3230600-6dd7-4a3f-86f0-c5026742655f&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742905163000&to_ts=1742916563000&live=false)                                                                                                                                                                                                                                                                                                                                                                       |
| ➖   | file_to_blackhole_300ms_latency              | egress throughput  | -0.02    | [-0.65, +0.62] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_300ms_latency%20run_id%3Ae3230600-6dd7-4a3f-86f0-c5026742655f&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742905163000&to_ts=1742916563000&live=false)                                                                                                                                                                                                                                                                                                                                                            |
| ➖   | file_to_blackhole_0ms_latency                | egress throughput  | -0.02    | [-0.83, +0.79] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_0ms_latency%20run_id%3Ae3230600-6dd7-4a3f-86f0-c5026742655f&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742905163000&to_ts=1742916563000&live=false)                                                                                                                                                                                                                                                                                                                                                              |
| ➖   | file_to_blackhole_0ms_latency_http1          | egress throughput  | -0.05    | [-0.84, +0.74] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_0ms_latency_http1%20run_id%3Ae3230600-6dd7-4a3f-86f0-c5026742655f&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742905163000&to_ts=1742916563000&live=false)                                                                                                                                                                                                                                                                                                                                                        |
| ➖   | quality_gate_idle_all_features               | memory utilization | -0.17    | [-0.26, -0.07] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Aquality_gate_idle_all_features%20run_id%3Ae3230600-6dd7-4a3f-86f0-c5026742655f&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742905163000&to_ts=1742916563000&live=false) [bounds checks dashboard](https://app.datadoghq.com/dashboard/vz3-jd5-bdi?fromUser=true&refresh_mode=paused&tpl_var_experiment%5B0%5D=quality_gate_idle_all_features&tpl_var_job_id%5B0%5D=e3230600-6dd7-4a3f-86f0-c5026742655f&tpl_var_run-id%5B0%5D=e3230600-6dd7-4a3f-86f0-c5026742655f&view=spans&from_ts=1742912363000&to_ts=1742912963000&live=false) |
| ➖   | otlp_ingest_metrics                          | memory utilization | -1.03    | [-1.17, -0.89] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Aotlp_ingest_metrics%20run_id%3Ae3230600-6dd7-4a3f-86f0-c5026742655f&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742905163000&to_ts=1742916563000&live=false)                                                                                                                                                                                                                                                                                                                                                                        |
| ➖   | otlp_ingest_traces                           | memory utilization | -2.43    | [-2.89, -1.96] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Aotlp_ingest_traces%20run_id%3Ae3230600-6dd7-4a3f-86f0-c5026742655f&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742905163000&to_ts=1742916563000&live=false)                                                                                                                                                                                                                                                                                                                                                                         |
</details>


<details>
<summary><h2>
Bounds Checks: ✅ Passed
</h2></summary>

| perf | experiment                                   | bounds_check_name  | replicates_passed | links                                                                                                                                                                                                                                                                                                                                                       |
|------|----------------------------------------------|--------------------|-------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| ✅   | file_to_blackhole_0ms_latency                | lost_bytes         | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_0ms_latency                | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_0ms_latency_http1          | lost_bytes         | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_0ms_latency_http1          | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_0ms_latency_http2          | lost_bytes         | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_0ms_latency_http2          | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_1000ms_latency             | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_1000ms_latency_linear_load | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_100ms_latency              | lost_bytes         | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_100ms_latency              | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_300ms_latency              | lost_bytes         | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_300ms_latency              | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_500ms_latency              | lost_bytes         | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_500ms_latency              | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | quality_gate_idle                            | intake_connections | 10/10             | [bounds checks dashboard](https://app.datadoghq.com/dashboard/vz3-jd5-bdi?fromUser=true&refresh_mode=paused&tpl_var_experiment%5B0%5D=quality_gate_idle&tpl_var_job_id%5B0%5D=e3230600-6dd7-4a3f-86f0-c5026742655f&tpl_var_run-id%5B0%5D=e3230600-6dd7-4a3f-86f0-c5026742655f&view=spans&from_ts=1742912363000&to_ts=1742912963000&live=false)              |
| ✅   | quality_gate_idle                            | memory_usage       | 10/10             | [bounds checks dashboard](https://app.datadoghq.com/dashboard/vz3-jd5-bdi?fromUser=true&refresh_mode=paused&tpl_var_experiment%5B0%5D=quality_gate_idle&tpl_var_job_id%5B0%5D=e3230600-6dd7-4a3f-86f0-c5026742655f&tpl_var_run-id%5B0%5D=e3230600-6dd7-4a3f-86f0-c5026742655f&view=spans&from_ts=1742912363000&to_ts=1742912963000&live=false)              |
| ✅   | quality_gate_idle_all_features               | intake_connections | 10/10             | [bounds checks dashboard](https://app.datadoghq.com/dashboard/vz3-jd5-bdi?fromUser=true&refresh_mode=paused&tpl_var_experiment%5B0%5D=quality_gate_idle_all_features&tpl_var_job_id%5B0%5D=e3230600-6dd7-4a3f-86f0-c5026742655f&tpl_var_run-id%5B0%5D=e3230600-6dd7-4a3f-86f0-c5026742655f&view=spans&from_ts=1742912363000&to_ts=1742912963000&live=false) |
| ✅   | quality_gate_idle_all_features               | memory_usage       | 10/10             | [bounds checks dashboard](https://app.datadoghq.com/dashboard/vz3-jd5-bdi?fromUser=true&refresh_mode=paused&tpl_var_experiment%5B0%5D=quality_gate_idle_all_features&tpl_var_job_id%5B0%5D=e3230600-6dd7-4a3f-86f0-c5026742655f&tpl_var_run-id%5B0%5D=e3230600-6dd7-4a3f-86f0-c5026742655f&view=spans&from_ts=1742912363000&to_ts=1742912963000&live=false) |
| ✅   | quality_gate_logs                            | intake_connections | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | quality_gate_logs                            | lost_bytes         | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | quality_gate_logs                            | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
</details>


<details>
<summary><h2>
Explanation
</h2></summary>

**Confidence level:** 90.00%
**Effect size tolerance:** |Δ mean %| ≥ 5.00%

Performance changes are noted in the **perf** column of each table:

* ✅ = significantly better comparison variant performance
* ❌ = significantly worse comparison variant performance
* ➖ = no significant change in performance

A regression test is an A/B test of target performance in a repeatable rig, where "performance" is measured as "comparison variant minus baseline variant" for an optimization goal (e.g., ingress throughput). Due to intrinsic variability in measuring that goal, we can only estimate its mean value for each experiment; we report uncertainty in that value as a 90.00% confidence interval denoted "Δ mean % CI".

For each experiment, we decide whether a change in performance is a "regression" -- a change worth investigating further -- if all of the following criteria are true:

1. Its estimated |Δ mean %| ≥ 5.00%, indicating the change is big enough to merit a closer look.

2. Its 90.00% confidence interval "Δ mean % CI" does not contain zero, indicating that *if our statistical model is accurate*, there is at least a 90.00% chance there is a difference in performance between baseline and comparison variants.

3. Its configuration does not mark it "erratic".
</details>

## CI Pass/Fail Decision

✅ **Passed.** All Quality Gates passed.

- **quality_gate_idle**, bounds check **intake_connections**: 10/10 replicas passed. Gate passed.
- **quality_gate_idle**, bounds check **memory_usage**: 10/10 replicas passed. Gate passed.
- **quality_gate_logs**, bounds check **intake_connections**: 10/10 replicas passed. Gate passed.
- **quality_gate_logs**, bounds check **memory_usage**: 10/10 replicas passed. Gate passed.
- **quality_gate_logs**, bounds check **lost_bytes**: 10/10 replicas passed. Gate passed.
- **quality_gate_idle_all_features**, bounds check **intake_connections**: 10/10 replicas passed. Gate passed.
- **quality_gate_idle_all_features**, bounds check **memory_usage**: 10/10 replicas passed. Gate passed.
author_association: NONE
user: login: dustmop
created_at: 2025-03-25T13:50:02Z
body: /merge
author_association: CONTRIBUTOR
user: login: dd-devflow[bot]
created_at: 2025-03-25T13:50:07Z
body: <!-- "FeedbackWorkflowId":"3cc075a4-8a2b-4839-a7e9-1132c389a7d9_36" -->
[View all feedbacks in Devflow UI.](https://devflow.static-app.us1.prod.dog/logs/3cc075a4-8a2b-4839-a7e9-1132c389a7d9_36)
`2025-03-25 13:50:07 UTC` :information_source: Start processing command `/merge`

---
`2025-03-25 13:50:16 UTC` :information_source: **MergeQueue:** waiting for PR to be ready

This merge request is not mergeable yet, because of pending checks/missing approvals. It will be added to the queue as soon as checks pass and/or get approvals.
**Note:** if you pushed new commits since the last approval, you may need additional approval.
You can remove it from the waiting list with `/remove` command.

---
`2025-03-25 15:55:10 UTC` :information_source: **MergeQueue:** merge request added to the queue

The expected merge time in `main` is approximately 2h (p90).

---
`2025-03-25 16:32:45 UTC` :information_source: **MergeQueue:** This merge request was merged

author_association: NONE
user: login: dustmop
created_at: 2025-03-25T13:50:09Z
body: /merge -c

author_association: CONTRIBUTOR
user: login: dd-devflow[bot]
created_at: 2025-03-25T13:50:12Z
body: <!-- "FeedbackWorkflowId":"60834d54-7f3d-4bba-8cc8-abf343dea279_36" -->
[View all feedbacks in Devflow UI.](https://devflow.static-app.us1.prod.dog/logs/60834d54-7f3d-4bba-8cc8-abf343dea279_36)
`2025-03-25 13:50:12 UTC` :information_source: Start processing command `/merge -c`
If you need support, contact us on Slack [#devflow](https://dd.slack.com/archives/C045N8L63PE)!

---
`2025-03-25 13:50:13 UTC` :x: **Devflow:** *`/merge -c`*

This merge request is not in the queue and can't be unqueued

To get help about command usage, write `/merge --help`

If you need support, contact us on Slack [#devflow](https://dd.slack.com/archives/C045N8L63PE) with those [details](https://atlas.ddbuild.io/namespaces/default/workflows/mq_routing_pr_DataDog_datadog-agent_35280_1742910611061/60834d54-7f3d-4bba-8cc8-abf343dea279)!

author_association: NONE
user: login: dustmop
created_at: 2025-03-25T14:38:27Z
body: /merge
author_association: CONTRIBUTOR
user: login: dd-devflow[bot]
created_at: 2025-03-25T14:38:31Z
body: <!-- "FeedbackWorkflowId":"e9beb232-9d93-4289-993b-4c4cd30e6576_36" -->
[View all feedbacks in Devflow UI.](https://devflow.static-app.us1.prod.dog/logs/e9beb232-9d93-4289-993b-4c4cd30e6576_36)
`2025-03-25 14:38:31 UTC` :information_source: Start processing command `/merge`

---
`2025-03-25 14:38:34 UTC` :x: MergeQueue

PR already in the queue with status waiting

author_association: NONE