Repository: DataDog/datadog-agent
Pull request #34600: (fleet) idempotent dir/ownership in postinsts
Author: login: arbll
Labels: name: changelog/no-changelog
description: No changelog entry needed
name: qa/done
description: QA done before merge and regressions are covered by tests
name: team/fleet
name: medium review
description: PR review might take time
Created: 2025-03-01T14:33:12Z | Merged: 2025-03-03T09:09:15Z

This PR adds idempotent tools in the agent and installer `postinst` on linux. 

There are 3 goals:
- Prepare the agent / installer => agent + installer transition
- Reduce the risk of permission bugs between versions
- Align 1-1 with the permissions we have in the deb/rpm `postinst`

It also fixes a bug I noticed adding this code:
- When we upgrade the agent through experiments we `chownRecursive` the agent folder to `dd-agent:dd-agent`
- Unfortunately we forget to also apply the `root:root` ownership of the system probe stuff
- This leads to a broken probe (the probe checks that those files are owned by the expected user in its logic)



Diff:
diff --git a/pkg/fleet/installer/packages/datadog_agent.go b/pkg/fleet/installer/packages/datadog_agent.go
index e72b9e3292ec..86611ad2b461 100644
--- a/pkg/fleet/installer/packages/datadog_agent.go
+++ b/pkg/fleet/installer/packages/datadog_agent.go
@@ -11,15 +11,13 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"io/fs"
 	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
 
 	"github.com/DataDog/datadog-agent/pkg/fleet/installer/installinfo"
+	"github.com/DataDog/datadog-agent/pkg/fleet/installer/packages/file"
 	"github.com/DataDog/datadog-agent/pkg/fleet/installer/packages/packagemanager"
 	"github.com/DataDog/datadog-agent/pkg/fleet/installer/packages/systemd"
+	"github.com/DataDog/datadog-agent/pkg/fleet/installer/packages/user"
 	"github.com/DataDog/datadog-agent/pkg/fleet/installer/telemetry"
 	"github.com/DataDog/datadog-agent/pkg/util/log"
 )
@@ -57,19 +55,30 @@ var (
 )
 
 var (
-	rootOwnedConfigPaths = []string{
-		"security-agent.yaml",
-		"system-probe.yaml",
-		"inject/tracer.yaml",
-		"inject",
-		"managed",
-	}
-	// matches omnibus/package-scripts/agent-deb/postinst
-	rootOwnedAgentPaths = []string{
-		"embedded/bin/system-probe",
-		"embedded/bin/security-agent",
-		"embedded/share/system-probe/ebpf",
-		"embedded/share/system-probe/java",
+	// agentDirectories are the directories that the agent needs to function
+	agentDirectories = file.Directories{
+		{Path: "/etc/datadog-agent", Mode: 0755, Owner: "dd-agent", Group: "dd-agent"},
+		{Path: "/var/log/datadog", Mode: 0755, Owner: "dd-agent", Group: "dd-agent"},
+	}
+
+	// agentConfigPermissions are the ownerships and modes that are enforced on the agent configuration files
+	agentConfigPermissions = file.Permissions{
+		{Path: ".", Owner: "dd-agent", Group: "dd-agent", Recursive: true},
+		{Path: "managed", Owner: "root", Group: "root", Recursive: true},
+		{Path: "inject", Owner: "root", Group: "root", Recursive: true},
+		{Path: "compliance.d", Owner: "root", Group: "root", Recursive: true},
+		{Path: "runtime-security.d", Owner: "root", Group: "root", Recursive: true},
+		{Path: "system-probe.yaml", Owner: "dd-agent", Group: "dd-agent", Mode: 0440},
+		{Path: "security-agent.yaml", Owner: "dd-agent", Group: "dd-agent", Mode: 0440},
+	}
+
+	// agentPackagePermissions are the ownerships and modes that are enforced on the agent package files
+	agentPackagePermissions = file.Permissions{
+		{Path: ".", Owner: "dd-agent", Group: "dd-agent", Recursive: true},
+		{Path: "embedded/bin/system-probe", Owner: "root", Group: "root"},
+		{Path: "embedded/bin/security-agent", Owner: "root", Group: "root"},
+		{Path: "embedded/share/system-probe/ebpf", Owner: "root", Group: "root", Recursive: true},
+		{Path: "embedded/share/system-probe/java", Owner: "root", Group: "root", Recursive: true},
 	}
 )
 
@@ -100,6 +109,28 @@ func SetupAgent(ctx context.Context, _ []string) (err error) {
 		span.Finish(err)
 	}()
 
+	// 1. Ensure the dd-agent user and group exist
+	if err = user.EnsureAgentUserAndGroup(ctx); err != nil {
+		return fmt.Errorf("failed to create dd-agent user and group: %v", err)
+	}
+	// 2. Add install info metadata if it doesn't exist
+	if err = installinfo.WriteInstallInfo("manual_update"); err != nil {
+		return fmt.Errorf("failed to write install info: %v", err)
+	}
+	// 3. Ensure config/log/package directories are created and have the correct permissions
+	if err = agentDirectories.Ensure(); err != nil {
+		return fmt.Errorf("failed to create directories: %v", err)
+	}
+	if err = agentPackagePermissions.Ensure("/opt/datadog-packages/datadog-agent/stable"); err != nil {
+		return fmt.Errorf("failed to set package ownerships: %v", err)
+	}
+	if err = agentConfigPermissions.Ensure("/etc/datadog-agent"); err != nil {
+		return fmt.Errorf("failed to set config ownerships: %v", err)
+	}
+	if err = file.EnsureSymlink("/opt/datadog-packages/datadog-agent/stable/bin/agent/agent", agentSymlink); err != nil {
+		return fmt.Errorf("failed to create symlink: %v", err)
+	}
+	// 4. Install the agent systemd units
 	for _, unit := range stableUnits {
 		if err = systemd.WriteEmbeddedUnit(ctx, unit); err != nil {
 			return fmt.Errorf("failed to load %s: %v", unit, err)
@@ -110,48 +141,13 @@ func SetupAgent(ctx context.Context, _ []string) (err error) {
 			return fmt.Errorf("failed to load %s: %v", unit, err)
 		}
 	}
-	if err = os.MkdirAll("/etc/datadog-agent", 0755); err != nil {
-		return fmt.Errorf("failed to create /etc/datadog-agent: %v", err)
-	}
-	ddAgentUID, ddAgentGID, err := getAgentIDs()
-	if err != nil {
-		return fmt.Errorf("error getting dd-agent user and group IDs: %w", err)
-	}
-
-	if err = os.Chown("/etc/datadog-agent", ddAgentUID, ddAgentGID); err != nil {
-		return fmt.Errorf("failed to chown /etc/datadog-agent: %v", err)
-	}
-	if err = chownRecursive("/etc/datadog-agent", ddAgentUID, ddAgentGID, rootOwnedConfigPaths); err != nil {
-		return fmt.Errorf("failed to chown /etc/datadog-agent: %v", err)
-	}
-	if err = chownRecursive("/opt/datadog-packages/datadog-agent/stable/", ddAgentUID, ddAgentGID, rootOwnedAgentPaths); err != nil {
-		return fmt.Errorf("failed to chown /opt/datadog-packages/datadog-agent/stable/: %v", err)
-	}
-	// Give root:datadog-agent permissions to system-probe and security-agent config files if they exist
-	if err = os.Chown("/etc/datadog-agent/system-probe.yaml", 0, ddAgentGID); err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("failed to chown /etc/datadog-agent/system-probe.yaml: %v", err)
-	}
-	if err = os.Chown("/etc/datadog-agent/security-agent.yaml", 0, ddAgentGID); err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("failed to chown /etc/datadog-agent/security-agent.yaml: %v", err)
-	}
-
 	if err = systemd.Reload(ctx); err != nil {
 		return fmt.Errorf("failed to reload systemd daemon: %v", err)
 	}
-
 	// enabling the agentUnit only is enough as others are triggered by it
 	if err = systemd.EnableUnit(ctx, agentUnit); err != nil {
 		return fmt.Errorf("failed to enable %s: %v", agentUnit, err)
 	}
-	if err = exec.CommandContext(ctx, "ln", "-sf", "/opt/datadog-packages/datadog-agent/stable/bin/agent/agent", agentSymlink).Run(); err != nil {
-		return fmt.Errorf("failed to create symlink: %v", err)
-	}
-
-	// write installinfo before start, or the agent could write it
-	if err = installinfo.WriteInstallInfo("manual_update"); err != nil {
-		return fmt.Errorf("failed to write install info: %v", err)
-	}
-
 	_, err = os.Stat("/etc/datadog-agent/datadog.yaml")
 	if err != nil && !os.IsNotExist(err) {
 		return fmt.Errorf("failed to check if /etc/datadog-agent/datadog.yaml exists: %v", err)
@@ -212,39 +208,10 @@ func RemoveAgent(ctx context.Context) error {
 	return nil
 }
 
-func chownRecursive(path string, uid int, gid int, ignorePaths []string) error {
-	return filepath.WalkDir(path, func(p string, _ fs.DirEntry, err error) error {
-		if err != nil {
-			if os.IsNotExist(err) {
-				return nil
-			}
-			return err
-		}
-		relPath, err := filepath.Rel(path, p)
-		if err != nil {
-			return err
-		}
-		for _, ignore := range ignorePaths {
-			if relPath == ignore || strings.HasPrefix(relPath, ignore+string(os.PathSeparator)) {
-				return nil
-			}
-		}
-		err = os.Chown(p, uid, gid)
-		if err != nil && os.IsNotExist(err) {
-			return nil
-		}
-		return err
-	})
-}
-
 // StartAgentExperiment starts the agent experiment
 func StartAgentExperiment(ctx context.Context) error {
-	ddAgentUID, ddAgentGID, err := getAgentIDs()
-	if err != nil {
-		return fmt.Errorf("error getting dd-agent user and group IDs: %w", err)
-	}
-	if err = chownRecursive("/opt/datadog-packages/datadog-agent/experiment/", ddAgentUID, ddAgentGID, rootOwnedAgentPaths); err != nil {
-		return fmt.Errorf("failed to chown /opt/datadog-packages/datadog-agent/experiment/: %v", err)
+	if err := agentPackagePermissions.Ensure("/opt/datadog-packages/datadog-agent/experiment"); err != nil {
+		return fmt.Errorf("failed to set package ownerships: %v", err)
 	}
 	return systemd.StartUnit(ctx, agentExp, "--no-block")
 }
diff --git a/pkg/fleet/installer/packages/datadog_installer.go b/pkg/fleet/installer/packages/datadog_installer.go
index b0df56282ed3..650d7fec5b1d 100644
--- a/pkg/fleet/installer/packages/datadog_installer.go
+++ b/pkg/fleet/installer/packages/datadog_installer.go
@@ -10,37 +10,30 @@ package packages
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"os"
 	"os/exec"
-	"os/user"
-	"strconv"
 
+	"github.com/DataDog/datadog-agent/pkg/fleet/installer/packages/file"
 	"github.com/DataDog/datadog-agent/pkg/fleet/installer/packages/systemd"
+	"github.com/DataDog/datadog-agent/pkg/fleet/installer/packages/user"
 	"github.com/DataDog/datadog-agent/pkg/util/log"
 )
 
 const (
+	installerSymlink = "/usr/bin/datadog-installer"
 	installerUnit    = "datadog-installer.service"
 	installerUnitExp = "datadog-installer-exp.service"
 )
 
 var installerUnits = []string{installerUnit, installerUnitExp}
 
-func addDDAgentUser(ctx context.Context) error {
-	if _, err := user.Lookup("dd-agent"); err == nil {
-		return nil
-	}
-	return exec.CommandContext(ctx, "useradd", "--system", "--shell", "/usr/sbin/nologin", "--home", "/opt/datadog-packages", "--no-create-home", "--no-user-group", "-g", "dd-agent", "dd-agent").Run()
-}
-
-func addDDAgentGroup(ctx context.Context) error {
-	if _, err := user.LookupGroup("dd-agent"); err == nil {
-		return nil
+var (
+	installerDirectories = file.Directories{
+		{Path: "/opt/datadog-packages/run", Mode: 0755, Owner: "dd-agent", Group: "dd-agent"},
+		{Path: "/opt/datadog-packages/tmp", Mode: 0755, Owner: "dd-agent", Group: "dd-agent"},
 	}
-	return exec.CommandContext(ctx, "groupadd", "--system", "dd-agent").Run()
-}
+)
 
 // PrepareInstaller prepares the installer
 func PrepareInstaller(ctx context.Context) error {
@@ -62,73 +55,25 @@ func SetupInstaller(ctx context.Context) (err error) {
 		}
 	}()
 
-	if err = addDDAgentGroup(ctx); err != nil {
-		return fmt.Errorf("error creating dd-agent group: %w", err)
-	}
-	if err = addDDAgentUser(ctx); err != nil {
-		return fmt.Errorf("error creating dd-agent user: %w", err)
-	}
-	err = exec.CommandContext(ctx, "usermod", "-g", "dd-agent", "dd-agent").Run()
-	if err != nil {
-		return fmt.Errorf("error adding dd-agent user to dd-agent group: %w", err)
-	}
-	ddAgentUID, ddAgentGID, err := getAgentIDs()
-	if err != nil {
-		return fmt.Errorf("error getting dd-agent user and group IDs: %w", err)
-	}
-	err = os.MkdirAll("/etc/datadog-agent", 0755)
-	if err != nil {
-		return fmt.Errorf("error creating /etc/datadog-agent: %w", err)
-	}
-	err = os.MkdirAll("/var/log/datadog", 0755)
-	if err != nil {
-		return fmt.Errorf("error creating /var/log/datadog: %w", err)
-	}
-	err = os.MkdirAll("/opt/datadog-packages/run", 0755)
-	if err != nil {
-		return fmt.Errorf("error creating /opt/datadog-packages/run: %w", err)
-	}
-	// Run directory can already be created by the RC client
-	err = os.Chmod("/opt/datadog-packages/run", 0755)
-	if err != nil {
-		return fmt.Errorf("error changing permissions of /opt/datadog-packages/run: %w", err)
-	}
-	err = os.Chown("/etc/datadog-agent", ddAgentUID, ddAgentGID)
+	// 1. Ensure the dd-agent user and group exist
+	err = user.EnsureAgentUserAndGroup(ctx)
 	if err != nil {
-		return fmt.Errorf("error changing owner of /etc/datadog-agent: %w", err)
+		return fmt.Errorf("error ensuring dd-agent user and group: %w", err)
 	}
-	err = os.Chown("/var/log/datadog", ddAgentUID, ddAgentGID)
-	if err != nil {
-		return fmt.Errorf("error changing owner of /var/log/datadog: %w", err)
-	}
-	err = os.Chown("/opt/datadog-packages/run", ddAgentUID, ddAgentGID)
-	if err != nil {
-		return fmt.Errorf("error changing owner of /opt/datadog-packages/run: %w", err)
+	// 2. Ensure the installer and agent directories exist and have the correct permissions
+	if err = installerDirectories.Ensure(); err != nil {
+		return fmt.Errorf("error ensuring installer directories: %w", err)
 	}
-	// Symlink /opt/datadog-packages/run to /var/run/datadog-installer for backwards compatibility
-	// This is a best effort, so we won't fail on it.
-	if err := os.Symlink("/opt/datadog-packages/run", "/var/run/datadog-installer"); err != nil && !os.IsExist(err) {
-		log.Warnf("failed to symlink /opt/datadog-packages/run to /var/run/datadog-installer: %s", err.Error())
+	if err = agentDirectories.Ensure(); err != nil {
+		return fmt.Errorf("error ensuring agent directories: %w", err)
 	}
-	// Enforce that the directory exists. It should be created by the bootstrapper but
-	// older versions don't do it
-	err = os.MkdirAll("/opt/datadog-packages/tmp", 0755)
-	if err != nil {
-		return fmt.Errorf("error creating /opt/datadog-packages/tmp: %w", err)
-	}
-	err = os.Chown("/opt/datadog-packages/tmp", ddAgentUID, ddAgentGID)
-	if err != nil {
-		return fmt.Errorf("error changing owner of /opt/datadog-packages/tmp: %w", err)
+	if err = file.EnsureSymlink("/opt/datadog-packages/datadog-installer/stable/bin/installer/installer", installerSymlink); err != nil {
+		return fmt.Errorf("error creating symlink /usr/bin/datadog-installer: %w", err)
 	}
-	// Create installer path symlink
-	err = os.Symlink("/opt/datadog-packages/datadog-installer/stable/bin/installer/installer", "/usr/bin/datadog-installer")
-	if err != nil && errors.Is(err, os.ErrExist) {
-		log.Info("Installer symlink already exists, skipping")
-	} else if err != nil {
-		return fmt.Errorf("error creating symlink to /usr/bin/datadog-installer: %w", err)
+	if err = file.EnsureSymlink("/opt/datadog-packages/run", "/var/run/datadog-installer"); err != nil {
+		return fmt.Errorf("error creating symlink /var/run/datadog-installer: %w", err)
 	}
-
-	// Check if systemd is running, if not return early
+	// 3. Install the installer systemd units
 	systemdRunning, err := systemd.IsRunning()
 	if err != nil {
 		return fmt.Errorf("error checking if systemd is running: %w", err)
@@ -137,45 +82,20 @@ func SetupInstaller(ctx context.Context) (err error) {
 		log.Infof("Installer: systemd is not running, skipping unit setup")
 		return nil
 	}
-
 	for _, unit := range installerUnits {
 		if err = systemd.WriteEmbeddedUnit(ctx, unit); err != nil {
 			return err
 		}
 	}
-
 	if err = systemd.Reload(ctx); err != nil {
 		return err
 	}
-
 	if err = systemd.EnableUnit(ctx, installerUnit); err != nil {
 		return err
 	}
-
 	return startInstallerStable(ctx)
 }
 
-// getAgentIDs returns the UID and GID of the dd-agent user and group.
-func getAgentIDs() (uid, gid int, err error) {
-	ddAgentUser, err := user.Lookup("dd-agent")
-	if err != nil {
-		return -1, -1, fmt.Errorf("dd-agent user not found: %w", err)
-	}
-	ddAgentGroup, err := user.LookupGroup("dd-agent")
-	if err != nil {
-		return -1, -1, fmt.Errorf("dd-agent group not found: %w", err)
-	}
-	ddAgentUID, err := strconv.Atoi(ddAgentUser.Uid)
-	if err != nil {
-		return -1, -1, fmt.Errorf("error converting dd-agent UID to int: %w", err)
-	}
-	ddAgentGID, err := strconv.Atoi(ddAgentGroup.Gid)
-	if err != nil {
-		return -1, -1, fmt.Errorf("error converting dd-agent GID to int: %w", err)
-	}
-	return ddAgentUID, ddAgentGID, nil
-}
-
 // startInstallerStable starts the stable systemd units for the installer
 func startInstallerStable(ctx context.Context) (err error) {
 	_, err = os.Stat("/etc/datadog-agent/datadog.yaml")
diff --git a/pkg/fleet/installer/packages/file/file.go b/pkg/fleet/installer/packages/file/file.go
new file mode 100644
index 000000000000..b3744c85857b
--- /dev/null
+++ b/pkg/fleet/installer/packages/file/file.go
@@ -0,0 +1,178 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-present Datadog, Inc.
+
+//go:build !windows
+
+// Package file offers filesystem utils geared towards idempotent operations.
+package file
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/user"
+	"path/filepath"
+	"strconv"
+)
+
+// Directory represents a desired state for a directory.
+type Directory struct {
+	Path  string
+	Mode  os.FileMode
+	Owner string
+	Group string
+}
+
+// Directories is a collection of directories.
+type Directories []Directory
+
+// Ensure ensures that the directory is created with the desired permissions.
+func (d Directory) Ensure() error {
+	uid, gid, err := getUserAndGroup(d.Owner, d.Group)
+	if err != nil {
+		return fmt.Errorf("error getting user and group IDs: %w", err)
+	}
+	err = os.MkdirAll(d.Path, d.Mode)
+	if err != nil {
+		return fmt.Errorf("error creating directory: %w", err)
+	}
+	err = os.Chown(d.Path, uid, gid)
+	if err != nil {
+		return fmt.Errorf("error changing directory ownership: %w", err)
+	}
+	err = os.Chmod(d.Path, d.Mode)
+	if err != nil {
+		return fmt.Errorf("error changing directory mode: %w", err)
+	}
+	return nil
+}
+
+// Ensure ensures that the directories are created with the desired permissions.
+func (ds Directories) Ensure() error {
+	for _, d := range ds {
+		if err := d.Ensure(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Permission represents the desired ownership and mode of a file.
+type Permission struct {
+	Path      string
+	Owner     string
+	Group     string
+	Mode      os.FileMode
+	Recursive bool
+}
+
+// Permissions is a collection of Permission.
+type Permissions []Permission
+
+// Ensure ensures that the file ownership and mode are set to the desired state.
+func (p Permission) Ensure(rootPath string) error {
+	rootFile := filepath.Join(rootPath, p.Path)
+	_, err := os.Stat(rootFile)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("error stating root path: %w", err)
+	}
+	// Resolve symlinks to ensure we're changing the permissions of the actual file and avoid issues with `filepath.Walk`.
+	rootFile, err = filepath.EvalSymlinks(rootFile)
+	if err != nil {
+		return fmt.Errorf("error resolving symlink: %w", err)
+	}
+	files := []string{rootFile}
+	if p.Recursive {
+		files, err = filesInDir(rootFile)
+		if err != nil {
+			return fmt.Errorf("error getting files in directory: %w", err)
+		}
+	}
+	for _, file := range files {
+		if p.Owner != "" && p.Group != "" {
+			if err := chown(file, p.Owner, p.Group); err != nil {
+				return fmt.Errorf("error changing file ownership: %w", err)
+			}
+		}
+		if p.Mode != 0 {
+			err := os.Chmod(file, p.Mode)
+			if err != nil {
+				return fmt.Errorf("error changing file mode: %w", err)
+			}
+		}
+	}
+	return nil
+}
+
+// Ensure ensures that the file ownership and mode are set to the desired state.
+func (ps Permissions) Ensure(rootPath string) error {
+	for _, o := range ps {
+		if err := o.Ensure(rootPath); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// EnsureSymlink ensures that the symlink is created.
+func EnsureSymlink(source, target string) error {
+	if err := os.RemoveAll(target); err != nil {
+		return fmt.Errorf("error removing existing symlink: %w", err)
+	}
+	if err := os.Symlink(source, target); err != nil {
+		return fmt.Errorf("error creating symlink: %w", err)
+	}
+	return nil
+}
+
+func getUserAndGroup(username, group string) (uid, gid int, err error) {
+	rawUID, err := user.Lookup(username)
+	if err != nil {
+		return 0, 0, fmt.Errorf("error looking up user: %w", err)
+	}
+	rawGID, err := user.LookupGroup(group)
+	if err != nil {
+		return 0, 0, fmt.Errorf("error looking up group: %w", err)
+	}
+	uid, err = strconv.Atoi(rawUID.Uid)
+	if err != nil {
+		return 0, 0, fmt.Errorf("error converting UID to int: %w", err)
+	}
+	gid, err = strconv.Atoi(rawGID.Gid)
+	if err != nil {
+		return 0, 0, fmt.Errorf("error converting GID to int: %w", err)
+	}
+	return uid, gid, nil
+}
+
+func chown(path string, username string, group string) (err error) {
+	uid, gid, err := getUserAndGroup(username, group)
+	if err != nil {
+		return fmt.Errorf("error getting user and group IDs: %w", err)
+	}
+	err = os.Chown(path, uid, gid)
+	if err != nil {
+		return fmt.Errorf("error changing file ownership: %w", err)
+	}
+	return nil
+}
+
+func filesInDir(dir string) ([]string, error) {
+	var files []string
+	err := filepath.Walk(dir, func(path string, _ os.FileInfo, err error) error {
+		if err != nil {
+			return fmt.Errorf("error walking path: %w", err)
+		}
+		files = append(files, path)
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return files, nil
+}
diff --git a/pkg/fleet/installer/packages/user/user.go b/pkg/fleet/installer/packages/user/user.go
new file mode 100644
index 000000000000..c34b2985761e
--- /dev/null
+++ b/pkg/fleet/installer/packages/user/user.go
@@ -0,0 +1,39 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-present Datadog, Inc.
+
+//go:build !windows
+
+// Package user offers an interface over user and group management
+package user
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"os/user"
+)
+
+// EnsureAgentUserAndGroup ensures that the user and group required by the agent are present on the system.
+func EnsureAgentUserAndGroup(ctx context.Context) error {
+	if _, err := user.LookupGroup("dd-agent"); err == nil {
+		return nil
+	}
+	err := exec.CommandContext(ctx, "groupadd", "--system", "dd-agent").Run()
+	if err != nil {
+		return fmt.Errorf("error creating dd-agent group: %w", err)
+	}
+	if _, err := user.Lookup("dd-agent"); err == nil {
+		return nil
+	}
+	err = exec.CommandContext(ctx, "useradd", "--system", "--shell", "/usr/sbin/nologin", "--home", "/opt/datadog-packages", "--no-create-home", "--no-user-group", "-g", "dd-agent", "dd-agent").Run()
+	if err != nil {
+		return fmt.Errorf("error creating dd-agent user: %w", err)
+	}
+	err = exec.CommandContext(ctx, "usermod", "-g", "dd-agent", "dd-agent").Run()
+	if err != nil {
+		return fmt.Errorf("error adding dd-agent user to dd-agent group: %w", err)
+	}
+	return nil
+}
diff --git a/test/new-e2e/tests/installer/script/default_script_test.go b/test/new-e2e/tests/installer/script/default_script_test.go
index d2cbc3e8af9d..5bc98a2f3273 100644
--- a/test/new-e2e/tests/installer/script/default_script_test.go
+++ b/test/new-e2e/tests/installer/script/default_script_test.go
@@ -63,8 +63,8 @@ func (s *installScriptDefaultSuite) TestInstall() {
 
 	// Config files exist
 	state.AssertFileExists("/etc/datadog-agent/datadog.yaml", 0640, "dd-agent", "dd-agent")
-	state.AssertFileExists("/etc/datadog-agent/system-probe.yaml", 0640, "root", "dd-agent")
-	state.AssertFileExists("/etc/datadog-agent/security-agent.yaml", 0640, "root", "dd-agent")
+	state.AssertFileExists("/etc/datadog-agent/system-probe.yaml", 0440, "dd-agent", "dd-agent")
+	state.AssertFileExists("/etc/datadog-agent/security-agent.yaml", 0440, "dd-agent", "dd-agent")
 	state.AssertPathDoesNotExist("/opt/datadog-packages/datadog-apm-library-ruby") // Not in DD_APM_INSTRUMENTATION_LIBRARIES
 
 	// Units started


Reviews:
user: login: BaptisteFoy
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-03T08:12:24Z
user: login: arbll
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-03T08:42:37Z
user: login: arbll
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-03T08:44:28Z

Review comments:
diff_hunk: @@ -100,6 +109,28 @@ func SetupAgent(ctx context.Context, _ []string) (err error) {
 		span.Finish(err)
 	}()
 
+	// 1. Ensure the dd-agent user and group exist
+	if err = user.EnsureAgentUserAndGroup(ctx); err != nil {
+		return fmt.Errorf("failed to create dd-agent user and group: %v", err)
+	}
+	// 2. Add install info metadata if it doesn't exist
+	if err = installinfo.WriteInstallInfo("manual_update"); err != nil {
+		return fmt.Errorf("failed to write install info: %v", err)
+	}
+	// 3. Ensure config/log/package directories are created and have the correct permissions
+	if err = agentDirectories.Ensure(); err != nil {
+		return fmt.Errorf("failed to create directories: %v", err)
+	}
+	if err = agentPackagePermissions.Ensure("/opt/datadog-packages/datadog-agent/stable"); err != nil {
+		return fmt.Errorf("failed to set package ownerships: %v", err)
+	}
+	if err = agentConfigPermissions.Ensure("/etc/datadog-agent"); err != nil {
+		return fmt.Errorf("failed to set config ownerships: %v", err)
+	}
+	if err = file.EnsureSymlink("/opt/datadog-packages/datadog-agent/stable/bin/agent/agent", agentSymlink); err != nil {
+		return fmt.Errorf("failed to create symlink: %v", err)
+	}
+	// 4. Install the agent systemd units
path: pkg/fleet/installer/packages/datadog_agent.go
user: login: BaptisteFoy
body: Question: shouldn't we run some of this on promote? e.g. the config permissions one
created_at: 2025-03-03T07:51:50Z
author_association: CONTRIBUTOR
path: pkg/fleet/installer/packages/file/file.go
user: login: BaptisteFoy
body: i guess we could reuse some of these helpers in the install script

not blocking this pr though
created_at: 2025-03-03T08:09:49Z
author_association: CONTRIBUTOR
path: pkg/fleet/installer/packages/file/file.go
user: login: BaptisteFoy
body: also i wonder if we could maybe merge this helper with the windows one (in installer_paths_windows.go)
created_at: 2025-03-03T08:12:11Z
author_association: CONTRIBUTOR
diff_hunk: @@ -100,6 +109,28 @@ func SetupAgent(ctx context.Context, _ []string) (err error) {
 		span.Finish(err)
 	}()
 
+	// 1. Ensure the dd-agent user and group exist
+	if err = user.EnsureAgentUserAndGroup(ctx); err != nil {
+		return fmt.Errorf("failed to create dd-agent user and group: %v", err)
+	}
+	// 2. Add install info metadata if it doesn't exist
+	if err = installinfo.WriteInstallInfo("manual_update"); err != nil {
+		return fmt.Errorf("failed to write install info: %v", err)
+	}
+	// 3. Ensure config/log/package directories are created and have the correct permissions
+	if err = agentDirectories.Ensure(); err != nil {
+		return fmt.Errorf("failed to create directories: %v", err)
+	}
+	if err = agentPackagePermissions.Ensure("/opt/datadog-packages/datadog-agent/stable"); err != nil {
+		return fmt.Errorf("failed to set package ownerships: %v", err)
+	}
+	if err = agentConfigPermissions.Ensure("/etc/datadog-agent"); err != nil {
+		return fmt.Errorf("failed to set config ownerships: %v", err)
+	}
+	if err = file.EnsureSymlink("/opt/datadog-packages/datadog-agent/stable/bin/agent/agent", agentSymlink); err != nil {
+		return fmt.Errorf("failed to create symlink: %v", err)
+	}
+	// 4. Install the agent systemd units
path: pkg/fleet/installer/packages/datadog_agent.go
user: login: arbll
body: For sure but let's solve this in a separate PR, we still need a plan to address the remaining risks of the experiment process.
created_at: 2025-03-03T08:42:37Z
author_association: MEMBER
path: pkg/fleet/installer/packages/file/file.go
user: login: arbll
body: Maybe for windows, for now I don't want to disturb the ongoing work they have merging agent&msi
created_at: 2025-03-03T08:44:28Z
author_association: MEMBER

PR comments:
user: login: cit-pr-commenter[bot]
created_at: 2025-03-01T14:37:29Z
body: ## Go Package Import Differences

Baseline: f4f49dad5434a504059af76b83b53aa99677168f
Comparison: cf05f67dbcb3129b86c8d401adcad1d4e6ff2fa2

<table><thead><tr><th>binary</th><th>os</th><th>arch</th><th>change</th></tr></thead><tbody>
<tr><td>agent</td><td>linux</td><td>amd64</td><td><details><summary>+2, -0</summary>
<pre lang='diff'>
+github.com/DataDog/datadog-agent/pkg/fleet/installer/packages/file
+github.com/DataDog/datadog-agent/pkg/fleet/installer/packages/user
</pre></details></td></tr>
<tr><td>agent</td><td>linux</td><td>arm64</td><td><details><summary>+2, -0</summary>
<pre lang='diff'>
+github.com/DataDog/datadog-agent/pkg/fleet/installer/packages/file
+github.com/DataDog/datadog-agent/pkg/fleet/installer/packages/user
</pre></details></td></tr>
</tbody></table>
author_association: NONE
user: login: agent-platform-auto-pr[bot]
created_at: 2025-03-01T15:08:30Z
body: ## Uncompressed package size comparison

Comparison with [ancestor](https://github.com/DataDog/datadog-agent/commit/f4f49dad5434a504059af76b83b53aa99677168f) `f4f49dad5434a504059af76b83b53aa99677168f`


<details>
  <summary> Diff per package </summary>

|package|diff|status|size|ancestor|threshold|
|--|--|--|--|--|--|
|datadog-agent-arm64-deb|0.00MB|✅|806.22MB|806.21MB|0.50MB|
|datadog-agent-x86_64-rpm|0.00MB|✅|825.05MB|825.05MB|0.50MB|
|datadog-agent-x86_64-suse|0.00MB|✅|825.05MB|825.05MB|0.50MB|
|datadog-agent-aarch64-rpm|0.00MB|✅|815.99MB|815.99MB|0.50MB|
|datadog-agent-amd64-deb|0.00MB|✅|815.25MB|815.25MB|0.50MB|
|datadog-dogstatsd-amd64-deb|0.00MB|✅|39.42MB|39.42MB|0.50MB|
|datadog-dogstatsd-x86_64-rpm|0.00MB|✅|39.50MB|39.50MB|0.50MB|
|datadog-dogstatsd-x86_64-suse|0.00MB|✅|39.50MB|39.50MB|0.50MB|
|datadog-dogstatsd-arm64-deb|0.00MB|✅|37.96MB|37.96MB|0.50MB|
|datadog-heroku-agent-amd64-deb|0.00MB|✅|440.67MB|440.67MB|0.50MB|
|datadog-iot-agent-amd64-deb|0.00MB|✅|62.10MB|62.10MB|0.50MB|
|datadog-iot-agent-x86_64-rpm|0.00MB|✅|62.17MB|62.17MB|0.50MB|
|datadog-iot-agent-x86_64-suse|0.00MB|✅|62.17MB|62.17MB|0.50MB|
|datadog-iot-agent-arm64-deb|0.00MB|✅|59.34MB|59.34MB|0.50MB|
|datadog-iot-agent-aarch64-rpm|0.00MB|✅|59.41MB|59.41MB|0.50MB|

</details>

## Decision
✅ Passed



author_association: CONTRIBUTOR
user: login: agent-platform-auto-pr[bot]
created_at: 2025-03-01T15:10:40Z
body: ## Test changes on VM

Use this command from [test-infra-definitions](https://github.com/DataDog/test-infra-definitions) to manually test this PR changes on a VM:
```sh
inv aws.create-vm --pipeline-id=57443829 --os-family=ubuntu
```
Note: This applies to commit **cf05f67d**
author_association: CONTRIBUTOR
user: login: agent-platform-auto-pr[bot]
created_at: 2025-03-01T15:13:13Z
body: ## Static quality checks ✅

Please find below the results from static quality gates


<details>
<summary>Successful checks</summary>

### Info

|Result|Quality gate|On disk size|On disk size limit|On wire size|On wire size limit|
|----|----|----|----|----|----|
|✅|static_quality_gate_agent_deb_amd64|789.0MiB|801.8MiB|192.26MiB|202.62MiB|
|✅|static_quality_gate_agent_deb_arm64|780.34MiB|793.14MiB|173.99MiB|184.51MiB|
|✅|static_quality_gate_agent_rpm_amd64|788.9MiB|801.79MiB|194.32MiB|205.03MiB|
|✅|static_quality_gate_agent_rpm_arm64|780.21MiB|793.09MiB|175.89MiB|186.44MiB|
|✅|static_quality_gate_agent_suse_amd64|788.89MiB|801.81MiB|194.32MiB|205.03MiB|
|✅|static_quality_gate_agent_suse_arm64|780.34MiB|793.14MiB|175.89MiB|186.44MiB|
|✅|static_quality_gate_dogstatsd_deb_amd64|37.67MiB|47.67MiB|9.78MiB|19.78MiB|
|✅|static_quality_gate_dogstatsd_deb_arm64|36.28MiB|46.27MiB|8.49MiB|18.49MiB|
|✅|static_quality_gate_dogstatsd_rpm_amd64|37.67MiB|47.67MiB|9.79MiB|19.79MiB|
|✅|static_quality_gate_dogstatsd_suse_amd64|37.67MiB|47.67MiB|9.79MiB|19.79MiB|
|✅|static_quality_gate_iot_agent_deb_amd64|59.3MiB|69.0MiB|14.9MiB|24.8MiB|
|✅|static_quality_gate_iot_agent_deb_arm64|56.66MiB|66.4MiB|12.83MiB|22.8MiB|
|✅|static_quality_gate_iot_agent_rpm_amd64|59.3MiB|69.0MiB|14.92MiB|24.8MiB|
|✅|static_quality_gate_iot_agent_rpm_arm64|56.67MiB|66.4MiB|12.86MiB|22.8MiB|
|✅|static_quality_gate_iot_agent_suse_amd64|59.3MiB|69.0MiB|14.92MiB|24.8MiB|
|✅|static_quality_gate_docker_agent_amd64|873.63MiB|886.12MiB|293.87MiB|304.21MiB|
|✅|static_quality_gate_docker_agent_arm64|888.28MiB|900.79MiB|280.12MiB|290.47MiB|
|✅|static_quality_gate_docker_agent_jmx_amd64|1.05GiB|1.06GiB|368.99MiB|379.33MiB|
|✅|static_quality_gate_docker_agent_jmx_arm64|1.05GiB|1.06GiB|351.23MiB|361.55MiB|
|✅|static_quality_gate_docker_dogstatsd_amd64|45.82MiB|55.78MiB|17.28MiB|27.28MiB|
|✅|static_quality_gate_docker_dogstatsd_arm64|44.46MiB|54.45MiB|16.17MiB|26.16MiB|
|✅|static_quality_gate_docker_cluster_agent_amd64|265.0MiB|274.78MiB|106.36MiB|116.28MiB|
|✅|static_quality_gate_docker_cluster_agent_arm64|280.97MiB|290.82MiB|101.19MiB|111.12MiB|

</details>

author_association: CONTRIBUTOR
user: login: cit-pr-commenter[bot]
created_at: 2025-03-01T15:35:48Z
body: ## Regression Detector

# Regression Detector Results

[Metrics dashboard](https://app.datadoghq.com/dashboard/ykh-ua8-vcu/SMP-Regression-Detector-Metrics?fromUser=true&refresh_mode=paused&tpl_var_run-id%5B0%5D=7536b78e-c4c6-495d-8539-41e02453e621&view=spans&from_ts=1740912965000&to_ts=1740913565000&live=false)  
[Target profiles](https://app.datadoghq.com/profiling/explorer?query=env%3Asingle-machine-performance%20service%3Adatadog-agent%20job_id%3A7536b78e-c4c6-495d-8539-41e02453e621&agg_m=count&agg_m_source=base&agg_t=count&fromUser=false&viz=stream&start=1740905765000&end=1740917165000&paused=true)  
Run ID: 7536b78e-c4c6-495d-8539-41e02453e621

Baseline: f4f49dad5434a504059af76b83b53aa99677168f
Comparison: cf05f67dbcb3129b86c8d401adcad1d4e6ff2fa2
[Diff](f4f49dad5434a504059af76b83b53aa99677168f...cf05f67dbcb3129b86c8d401adcad1d4e6ff2fa2)


## Optimization Goals: ✅ No significant changes detected


<details>
<summary><h2>
Fine details of change detection per experiment
</h2></summary>

| perf | experiment                                   | goal               | Δ mean % | Δ mean % CI    | trials | links                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
|------|----------------------------------------------|--------------------|----------|----------------|--------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| ➖   | uds_dogstatsd_to_api_cpu                     | % cpu utilization  | +1.39    | [+0.50, +2.27] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Auds_dogstatsd_to_api_cpu%20run_id%3A7536b78e-c4c6-495d-8539-41e02453e621&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1740905765000&to_ts=1740917165000&live=false)                                                                                                                                                                                                                                                                                                                                                                   |
| ➖   | quality_gate_idle                            | memory utilization | +0.44    | [+0.40, +0.47] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Aquality_gate_idle%20run_id%3A7536b78e-c4c6-495d-8539-41e02453e621&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1740905765000&to_ts=1740917165000&live=false) [bounds checks dashboard](https://app.datadoghq.com/dashboard/vz3-jd5-bdi?fromUser=true&refresh_mode=paused&tpl_var_experiment%5B0%5D=quality_gate_idle&tpl_var_job_id%5B0%5D=7536b78e-c4c6-495d-8539-41e02453e621&tpl_var_run-id%5B0%5D=7536b78e-c4c6-495d-8539-41e02453e621&view=spans&from_ts=1740912965000&to_ts=1740913565000&live=false)                           |
| ➖   | quality_gate_logs                            | % cpu utilization  | +0.22    | [-2.72, +3.16] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Aquality_gate_logs%20run_id%3A7536b78e-c4c6-495d-8539-41e02453e621&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1740905765000&to_ts=1740917165000&live=false)                                                                                                                                                                                                                                                                                                                                                                          |
| ➖   | file_to_blackhole_1000ms_latency_linear_load | egress throughput  | +0.18    | [-0.29, +0.65] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_1000ms_latency_linear_load%20run_id%3A7536b78e-c4c6-495d-8539-41e02453e621&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1740905765000&to_ts=1740917165000&live=false)                                                                                                                                                                                                                                                                                                                                               |
| ➖   | file_tree                                    | memory utilization | +0.08    | [+0.02, +0.14] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_tree%20run_id%3A7536b78e-c4c6-495d-8539-41e02453e621&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1740905765000&to_ts=1740917165000&live=false)                                                                                                                                                                                                                                                                                                                                                                                  |
| ➖   | file_to_blackhole_500ms_latency              | egress throughput  | +0.07    | [-0.70, +0.85] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_500ms_latency%20run_id%3A7536b78e-c4c6-495d-8539-41e02453e621&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1740905765000&to_ts=1740917165000&live=false)                                                                                                                                                                                                                                                                                                                                                            |
| ➖   | file_to_blackhole_0ms_latency_http1          | egress throughput  | +0.03    | [-0.78, +0.84] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_0ms_latency_http1%20run_id%3A7536b78e-c4c6-495d-8539-41e02453e621&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1740905765000&to_ts=1740917165000&live=false)                                                                                                                                                                                                                                                                                                                                                        |
| ➖   | file_to_blackhole_100ms_latency              | egress throughput  | +0.02    | [-0.67, +0.71] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_100ms_latency%20run_id%3A7536b78e-c4c6-495d-8539-41e02453e621&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1740905765000&to_ts=1740917165000&live=false)                                                                                                                                                                                                                                                                                                                                                            |
| ➖   | file_to_blackhole_0ms_latency_http2          | egress throughput  | +0.01    | [-0.79, +0.81] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_0ms_latency_http2%20run_id%3A7536b78e-c4c6-495d-8539-41e02453e621&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1740905765000&to_ts=1740917165000&live=false)                                                                                                                                                                                                                                                                                                                                                        |
| ➖   | file_to_blackhole_1000ms_latency             | egress throughput  | +0.01    | [-0.77, +0.78] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_1000ms_latency%20run_id%3A7536b78e-c4c6-495d-8539-41e02453e621&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1740905765000&to_ts=1740917165000&live=false)                                                                                                                                                                                                                                                                                                                                                           |
| ➖   | file_to_blackhole_0ms_latency                | egress throughput  | +0.00    | [-0.78, +0.78] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_0ms_latency%20run_id%3A7536b78e-c4c6-495d-8539-41e02453e621&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1740905765000&to_ts=1740917165000&live=false)                                                                                                                                                                                                                                                                                                                                                              |
| ➖   | tcp_dd_logs_filter_exclude                   | ingress throughput | -0.00    | [-0.02, +0.01] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Atcp_dd_logs_filter_exclude%20run_id%3A7536b78e-c4c6-495d-8539-41e02453e621&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1740905765000&to_ts=1740917165000&live=false)                                                                                                                                                                                                                                                                                                                                                                 |
| ➖   | uds_dogstatsd_to_api                         | ingress throughput | -0.01    | [-0.27, +0.25] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Auds_dogstatsd_to_api%20run_id%3A7536b78e-c4c6-495d-8539-41e02453e621&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1740905765000&to_ts=1740917165000&live=false)                                                                                                                                                                                                                                                                                                                                                                       |
| ➖   | file_to_blackhole_300ms_latency              | egress throughput  | -0.01    | [-0.63, +0.62] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_300ms_latency%20run_id%3A7536b78e-c4c6-495d-8539-41e02453e621&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1740905765000&to_ts=1740917165000&live=false)                                                                                                                                                                                                                                                                                                                                                            |
| ➖   | quality_gate_idle_all_features               | memory utilization | -0.07    | [-0.12, -0.02] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Aquality_gate_idle_all_features%20run_id%3A7536b78e-c4c6-495d-8539-41e02453e621&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1740905765000&to_ts=1740917165000&live=false) [bounds checks dashboard](https://app.datadoghq.com/dashboard/vz3-jd5-bdi?fromUser=true&refresh_mode=paused&tpl_var_experiment%5B0%5D=quality_gate_idle_all_features&tpl_var_job_id%5B0%5D=7536b78e-c4c6-495d-8539-41e02453e621&tpl_var_run-id%5B0%5D=7536b78e-c4c6-495d-8539-41e02453e621&view=spans&from_ts=1740912965000&to_ts=1740913565000&live=false) |
| ➖   | tcp_syslog_to_blackhole                      | ingress throughput | -0.47    | [-0.52, -0.41] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Atcp_syslog_to_blackhole%20run_id%3A7536b78e-c4c6-495d-8539-41e02453e621&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1740905765000&to_ts=1740917165000&live=false)                                                                                                                                                                                                                                                                                                                                                                    |
</details>


<details>
<summary><h2>
Bounds Checks: ✅ Passed
</h2></summary>

| perf | experiment                                   | bounds_check_name  | replicates_passed | links                                                                                                                                                                                                                                                                                                                                                       |
|------|----------------------------------------------|--------------------|-------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| ✅   | file_to_blackhole_0ms_latency                | lost_bytes         | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_0ms_latency                | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_0ms_latency_http1          | lost_bytes         | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_0ms_latency_http1          | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_0ms_latency_http2          | lost_bytes         | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_0ms_latency_http2          | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_1000ms_latency             | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_1000ms_latency_linear_load | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_100ms_latency              | lost_bytes         | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_100ms_latency              | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_300ms_latency              | lost_bytes         | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_300ms_latency              | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_500ms_latency              | lost_bytes         | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_500ms_latency              | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | quality_gate_idle                            | intake_connections | 10/10             | [bounds checks dashboard](https://app.datadoghq.com/dashboard/vz3-jd5-bdi?fromUser=true&refresh_mode=paused&tpl_var_experiment%5B0%5D=quality_gate_idle&tpl_var_job_id%5B0%5D=7536b78e-c4c6-495d-8539-41e02453e621&tpl_var_run-id%5B0%5D=7536b78e-c4c6-495d-8539-41e02453e621&view=spans&from_ts=1740912965000&to_ts=1740913565000&live=false)              |
| ✅   | quality_gate_idle                            | memory_usage       | 10/10             | [bounds checks dashboard](https://app.datadoghq.com/dashboard/vz3-jd5-bdi?fromUser=true&refresh_mode=paused&tpl_var_experiment%5B0%5D=quality_gate_idle&tpl_var_job_id%5B0%5D=7536b78e-c4c6-495d-8539-41e02453e621&tpl_var_run-id%5B0%5D=7536b78e-c4c6-495d-8539-41e02453e621&view=spans&from_ts=1740912965000&to_ts=1740913565000&live=false)              |
| ✅   | quality_gate_idle_all_features               | intake_connections | 10/10             | [bounds checks dashboard](https://app.datadoghq.com/dashboard/vz3-jd5-bdi?fromUser=true&refresh_mode=paused&tpl_var_experiment%5B0%5D=quality_gate_idle_all_features&tpl_var_job_id%5B0%5D=7536b78e-c4c6-495d-8539-41e02453e621&tpl_var_run-id%5B0%5D=7536b78e-c4c6-495d-8539-41e02453e621&view=spans&from_ts=1740912965000&to_ts=1740913565000&live=false) |
| ✅   | quality_gate_idle_all_features               | memory_usage       | 10/10             | [bounds checks dashboard](https://app.datadoghq.com/dashboard/vz3-jd5-bdi?fromUser=true&refresh_mode=paused&tpl_var_experiment%5B0%5D=quality_gate_idle_all_features&tpl_var_job_id%5B0%5D=7536b78e-c4c6-495d-8539-41e02453e621&tpl_var_run-id%5B0%5D=7536b78e-c4c6-495d-8539-41e02453e621&view=spans&from_ts=1740912965000&to_ts=1740913565000&live=false) |
| ✅   | quality_gate_logs                            | intake_connections | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | quality_gate_logs                            | lost_bytes         | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | quality_gate_logs                            | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
</details>


<details>
<summary><h2>
Explanation
</h2></summary>

**Confidence level:** 90.00%
**Effect size tolerance:** |Δ mean %| ≥ 5.00%

Performance changes are noted in the **perf** column of each table:

* ✅ = significantly better comparison variant performance
* ❌ = significantly worse comparison variant performance
* ➖ = no significant change in performance

A regression test is an A/B test of target performance in a repeatable rig, where "performance" is measured as "comparison variant minus baseline variant" for an optimization goal (e.g., ingress throughput). Due to intrinsic variability in measuring that goal, we can only estimate its mean value for each experiment; we report uncertainty in that value as a 90.00% confidence interval denoted "Δ mean % CI".

For each experiment, we decide whether a change in performance is a "regression" -- a change worth investigating further -- if all of the following criteria are true:

1. Its estimated |Δ mean %| ≥ 5.00%, indicating the change is big enough to merit a closer look.

2. Its 90.00% confidence interval "Δ mean % CI" does not contain zero, indicating that *if our statistical model is accurate*, there is at least a 90.00% chance there is a difference in performance between baseline and comparison variants.

3. Its configuration does not mark it "erratic".
</details>

## CI Pass/Fail Decision

✅ **Passed.** All Quality Gates passed.

- **quality_gate_idle_all_features**, bounds check **memory_usage**: 10/10 replicas passed. Gate passed.
- **quality_gate_idle_all_features**, bounds check **intake_connections**: 10/10 replicas passed. Gate passed.
- **quality_gate_idle**, bounds check **memory_usage**: 10/10 replicas passed. Gate passed.
- **quality_gate_idle**, bounds check **intake_connections**: 10/10 replicas passed. Gate passed.
- **quality_gate_logs**, bounds check **intake_connections**: 10/10 replicas passed. Gate passed.
- **quality_gate_logs**, bounds check **lost_bytes**: 10/10 replicas passed. Gate passed.
- **quality_gate_logs**, bounds check **memory_usage**: 10/10 replicas passed. Gate passed.
author_association: NONE
user: login: arbll
created_at: 2025-03-03T08:43:52Z
body: /merge
author_association: MEMBER
user: login: dd-devflow[bot]
created_at: 2025-03-03T08:43:55Z
body: <!-- "FeedbackWorkflowId":"8e809df6-088c-4482-bd86-67af4f7ff657_36" -->
[View all feedbacks in Devflow UI.](https://devflow.static-app.us1.prod.dog/logs/8e809df6-088c-4482-bd86-67af4f7ff657_36)
`2025-03-03 08:43:55 UTC` :information_source: Start processing command `/merge`

---
`2025-03-03 08:43:58 UTC` :information_source: **MergeQueue:** pull request added to the queue

The median merge time in `main` is 33m.

---
`2025-03-03 09:09:20 UTC` :information_source: **MergeQueue:** This merge request was merged

author_association: NONE