Repository: DataDog/datadog-agent
Pull request #35402: gpum: changed gpu_host tag implementation 
Author: login: val06
Labels: name: changelog/no-changelog
description: No changelog entry needed
name: team/ebpf-platform
name: qa/done
description: QA done before merge and regressions are covered by tests
name: medium review
description: PR review might take time
Created: 2025-03-24T15:12:06Z | Merged: 2025-03-25T12:29:36Z

### What does this PR do?

Changes the way we retrieve gpu_host tag, instead of using nvml API, scan /proc/driver/nvidia directory.
if present and has more than 1 entry, we generate the tag

### Motivation

Remove dependency on nvml lib properly mounted into core agent container.
This will allow us to use the `gpu_host` tag as a discovery feature to enable gpu monitoring product

### Describe how you validated your changes

added UTs

### Possible Drawbacks / Trade-offs

### Additional Notes
`/host/proc` is already mounted by the core agent

[Jira ticket](https://datadoghq.atlassian.net/browse/EBPF-685)


Diff:
diff --git a/pkg/gpu/tags/tags.go b/pkg/gpu/tags/tags.go
new file mode 100644
index 000000000000..d721c1a1b3bc
--- /dev/null
+++ b/pkg/gpu/tags/tags.go
@@ -0,0 +1,62 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-present Datadog, Inc.
+
+//go:build linux
+
+// Package tags provides gpu host tags to the host payload
+package tags
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/DataDog/datadog-agent/pkg/util/funcs"
+	"github.com/DataDog/datadog-agent/pkg/util/log"
+)
+
+// procFSRoot retrieves the current procfs dir we should use
+// this function is copied from kernel/fs.go because using that function directly violates the check_size,
+// as some artifacts have a major binary size increase
+var procFSRoot = funcs.MemoizeNoError(func() string {
+	if v := os.Getenv("HOST_PROC"); v != "" {
+		return v
+	}
+	if os.Getenv("DOCKER_DD_AGENT") != "" {
+		if _, err := os.Stat("/host"); err == nil {
+			return "/host/proc"
+		}
+	}
+	return "/proc"
+})
+
+// GetTags returns a slice of tags indicating GPU presence
+func GetTags() []string {
+	// Get the host's proc directory path
+	procPath := procFSRoot()
+	nvidiaPath := filepath.Join(procPath, "driver", "nvidia", "gpus")
+
+	// Check if the NVIDIA directory exists
+	if _, err := os.Stat(nvidiaPath); err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		log.Warnf("Failed to check NVIDIA GPU directory: %v", err)
+		return nil
+	}
+
+	// Read the directory to count GPU entries
+	entries, err := os.ReadDir(nvidiaPath)
+	if err != nil {
+		log.Warnf("Failed to read NVIDIA GPU directory: %v", err)
+		return nil
+	}
+
+	// If we have at least one entry, we have a GPU
+	if len(entries) > 0 {
+		return []string{"gpu_host:true"}
+	}
+
+	return nil
+}
diff --git a/pkg/gpu/tags/tags_nix.go b/pkg/gpu/tags/tags_nix.go
deleted file mode 100644
index f470cc846076..000000000000
--- a/pkg/gpu/tags/tags_nix.go
+++ /dev/null
@@ -1,70 +0,0 @@
-// Unless explicitly stated otherwise all files in this repository are licensed
-// under the Apache License Version 2.0.
-// This product includes software developed at Datadog (https://www.datadoghq.com/).
-// Copyright 2016-present Datadog, Inc.
-
-//go:build linux && nvml
-
-// Package tags provides GPU-related host tags
-package tags
-
-import (
-	"fmt"
-	"sync"
-
-	"github.com/NVIDIA/go-nvml/pkg/nvml"
-
-	"github.com/DataDog/datadog-agent/pkg/util/log"
-)
-
-// used for tests to mock the NVML library
-var nvmlLibrary nvml.Interface
-var mtx sync.Mutex
-
-func ensureNvmlLibrary(getLibrary func(opts ...nvml.LibraryOption) nvml.Interface) error {
-	// if the library is already initialized, return directly to avoid unnecessary locking
-	if nvmlLibrary != nil {
-		return nil
-	}
-
-	mtx.Lock()
-	defer mtx.Unlock()
-
-	// already initialized
-	if nvmlLibrary != nil {
-		return nil
-	}
-
-	lib := getLibrary()
-	if ret := lib.Init(); ret != nvml.SUCCESS && ret != nvml.ERROR_ALREADY_INITIALIZED {
-		return fmt.Errorf("Failed to get gpu host tags, failed to initialize NVML: %v", nvml.ErrorString(ret))
-	}
-
-	nvmlLibrary = lib
-	return nil
-}
-
-// getTags is the internal implementation of GetTags, exposed so that we can test with a mock NVML library
-// to ensure we're properly handling library initialization errors
-func getTags(getLibrary func(opts ...nvml.LibraryOption) nvml.Interface) []string {
-	if err := ensureNvmlLibrary(getLibrary); err != nil {
-		log.Warnf("Failed to get gpu host tags, failed to initialize NVML: %v", err)
-		return nil
-	}
-	count, ret := nvmlLibrary.DeviceGetCount()
-	if ret != nvml.SUCCESS {
-		log.Warnf("Failed to get gpu host tags, couldn't assess number of devices: %v", nvml.ErrorString(ret))
-		return nil
-	}
-
-	if count > 0 {
-		return []string{"gpu_host:true"}
-	}
-
-	return nil
-}
-
-// GetTags returns gpu_host:true if any NVIDIA GPUs are present, nil otherwise
-func GetTags() []string {
-	return getTags(nvml.New)
-}
diff --git a/pkg/gpu/tags/tags_noop.go b/pkg/gpu/tags/tags_noop.go
index aff3d73742bf..d7e221c623bc 100644
--- a/pkg/gpu/tags/tags_noop.go
+++ b/pkg/gpu/tags/tags_noop.go
@@ -3,7 +3,7 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2016-present Datadog, Inc.
 
-//go:build !linux || !nvml
+//go:build !linux
 
 // Package tags provides GPU-related host tags
 package tags
diff --git a/pkg/gpu/tags/tags_test.go b/pkg/gpu/tags/tags_test.go
index c318c0e47a32..adac96ff2857 100644
--- a/pkg/gpu/tags/tags_test.go
+++ b/pkg/gpu/tags/tags_test.go
@@ -3,68 +3,89 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2016-present Datadog, Inc.
 
-//go:build linux && nvml
+//go:build linux
 
 package tags
 
 import (
+	"fmt"
+	"os"
+	"path/filepath"
 	"testing"
-	"time"
 
-	"github.com/NVIDIA/go-nvml/pkg/nvml"
 	"github.com/stretchr/testify/assert"
 )
 
-type mockNVML struct {
-	nvml.Interface
-	deviceCount int
-	countError  nvml.Return
-	initError   nvml.Return
-}
-
-func (m *mockNVML) DeviceGetCount() (int, nvml.Return) {
-	return m.deviceCount, m.countError
-}
+func TestGetTags(t *testing.T) {
+	// Create a temporary directory for our test
+	tmpDir := t.TempDir()
 
-func (m *mockNVML) Init() nvml.Return {
-	return m.initError
-}
+	// Override procFSRoot to use our temporary directory
+	originalProcFSRoot := procFSRoot
+	procFSRoot = func() string { return tmpDir }
+	defer func() { procFSRoot = originalProcFSRoot }()
 
-func TestGetTags(t *testing.T) {
 	tests := []struct {
 		name     string
-		nvml     mockNVML
+		setup    func() error
 		wantTags []string
 	}{
 		{
-			name: "no GPUs",
-			nvml: mockNVML{
-				deviceCount: 0,
-				countError:  nvml.SUCCESS,
+			name: "no NVIDIA directory",
+			setup: func() error {
+				// Don't create anything, directory should not exist
+				return nil
 			},
 			wantTags: nil,
 		},
 		{
-			name: "has GPUs",
-			nvml: mockNVML{
-				deviceCount: 2,
-				countError:  nvml.SUCCESS,
+			name: "NVIDIA directory exists but empty",
+			setup: func() error {
+				nvidiaPath := filepath.Join(tmpDir, "driver", "nvidia", "gpus")
+				return os.MkdirAll(nvidiaPath, 0755)
+			},
+			wantTags: nil,
+		},
+		{
+			name: "NVIDIA directory with one GPU",
+			setup: func() error {
+				nvidiaPath := filepath.Join(tmpDir, "driver", "nvidia", "gpus")
+				if err := os.MkdirAll(nvidiaPath, 0755); err != nil {
+					return err
+				}
+				// Create a dummy GPU entry
+				return os.WriteFile(filepath.Join(nvidiaPath, "0"), []byte("dummy"), 0644)
 			},
 			wantTags: []string{"gpu_host:true"},
 		},
 		{
-			name: "device count error",
-			nvml: mockNVML{
-				countError: nvml.ERROR_UNKNOWN,
+			name: "NVIDIA directory with multiple GPUs",
+			setup: func() error {
+				nvidiaPath := filepath.Join(tmpDir, "driver", "nvidia", "gpus")
+				if err := os.MkdirAll(nvidiaPath, 0755); err != nil {
+					return err
+				}
+				// Create multiple dummy GPU entries
+				for i := 0; i < 2; i++ {
+					if err := os.WriteFile(filepath.Join(nvidiaPath, fmt.Sprintf("%d", i)), []byte("dummy"), 0644); err != nil {
+						return err
+					}
+				}
+				return nil
 			},
-			wantTags: nil,
+			wantTags: []string{"gpu_host:true"},
 		},
 		{
-			name: "init error",
-			nvml: mockNVML{
-				initError:   nvml.ERROR_UNKNOWN,
-				countError:  nvml.SUCCESS,
-				deviceCount: 10, // If the init check is not correct, we'll report the tag and the test will fail
+			name: "NVIDIA directory exists but not readable",
+			setup: func() error {
+				nvidiaPath := filepath.Join(tmpDir, "driver", "nvidia", "gpus")
+				if err := os.MkdirAll(nvidiaPath, 0755); err != nil {
+					return err
+				}
+				if err := os.Chmod(nvidiaPath, 0000); err != nil {
+					return err
+				}
+				return nil
 			},
 			wantTags: nil,
 		},
@@ -72,84 +93,20 @@ func TestGetTags(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			nvmlLibrary = nil
-			gotTags := getTags(func(_ ...nvml.LibraryOption) nvml.Interface { return &tt.nvml })
+			defer func() {
+				// Clean up any existing files before each test
+				if err := os.RemoveAll(filepath.Join(tmpDir, "driver")); err != nil {
+					t.Fatalf("Failed to clean up test directory: %v", err)
+				}
+			}()
+
+			if tt.setup != nil {
+				if err := tt.setup(); err != nil {
+					t.Fatalf("Setup failed: %v", err)
+				}
+			}
+			gotTags := GetTags()
 			assert.Equal(t, tt.wantTags, gotTags)
 		})
 	}
 }
-
-// BenchmarkGetTags requires libnvidia-ml.so to be present on the host.
-// This benchmark uses the real NVML library to measure actual performance,
-// as loading the native library is potentially the main bottleneck.
-func BenchmarkGetTags(b *testing.B) {
-	if res := nvml.Init(); res != nvml.SUCCESS && res != nvml.ERROR_ALREADY_INITIALIZED {
-		b.Fatalf("Failed to initialize NVML library")
-	}
-	_ = nvml.Shutdown()
-	b.ResetTimer()
-
-	for i := 0; i < b.N; i++ {
-		nvmlLibrary = nil
-		_ = GetTags()
-	}
-
-	// Verify the function completes within 500ms
-	b.StopTimer()
-	nvmlLibrary = nil
-	start := time.Now()
-	GetTags()
-	duration := time.Since(start)
-	if duration > 500*time.Millisecond {
-		b.Errorf("GetTags took %v, expected less than 500ms", duration)
-	} else {
-		b.Logf("GetTags took %v", duration)
-	}
-}
-
-func TestEnsureNvmlLibrary(t *testing.T) {
-	t.Run("library present", func(t *testing.T) {
-		// reset nvmlLibrary to nil to avoid interference from previous tests
-		nvmlLibrary = nil
-		mockLib := &mockNVML{
-			initError: nvml.SUCCESS,
-		}
-		newFunc := func(_ ...nvml.LibraryOption) nvml.Interface { return mockLib }
-		assert.NoError(t, ensureNvmlLibrary(newFunc))
-	})
-	t.Run("library absent", func(t *testing.T) {
-		// reset nvmlLibrary to nil to avoid interference from previous tests
-		nvmlLibrary = nil
-		mockLib := &mockNVML{
-			initError: nvml.ERROR_LIBRARY_NOT_FOUND,
-		}
-		newFunc := func(_ ...nvml.LibraryOption) nvml.Interface { return mockLib }
-		assert.Error(t, ensureNvmlLibrary(newFunc))
-	})
-
-	t.Run("library absent, second call fails too", func(t *testing.T) {
-		// reset nvmlLibrary to nil to avoid interference from previous tests
-		nvmlLibrary = nil
-		mockLib := &mockNVML{
-			initError: nvml.ERROR_LIBRARY_NOT_FOUND,
-		}
-		newFunc := func(_ ...nvml.LibraryOption) nvml.Interface { return mockLib }
-		assert.Error(t, ensureNvmlLibrary(newFunc))
-		assert.Error(t, ensureNvmlLibrary(newFunc))
-	})
-
-	t.Run("library absent, second call succeeds", func(t *testing.T) {
-		// reset nvmlLibrary to nil to avoid interference from previous tests
-		nvmlLibrary = nil
-		mockLibFail := &mockNVML{
-			initError: nvml.ERROR_LIBRARY_NOT_FOUND,
-		}
-		mockLibSuccess := &mockNVML{
-			initError: nvml.SUCCESS,
-		}
-		failFunc := func(_ ...nvml.LibraryOption) nvml.Interface { return mockLibFail }
-		successFunc := func(_ ...nvml.LibraryOption) nvml.Interface { return mockLibSuccess }
-		assert.Error(t, ensureNvmlLibrary(failFunc))
-		assert.NoError(t, ensureNvmlLibrary(successFunc))
-	})
-}


Reviews:
user: login: gjulianm
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T15:15:56Z
user: login: val06
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-25T09:57:12Z
user: login: gjulianm
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-25T09:58:57Z

Review comments:
diff_hunk: @@ -3,92 +3,111 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2016-present Datadog, Inc.
 
-//go:build linux && nvml
+//go:build linux
 
 package tags
 
 import (
+	"fmt"
+	"os"
+	"path/filepath"
 	"testing"
-	"time"
 
-	"github.com/NVIDIA/go-nvml/pkg/nvml"
+	"github.com/DataDog/datadog-agent/pkg/util/kernel"
 	"github.com/stretchr/testify/assert"
 )
 
-type mockNVML struct {
-	nvml.Interface
-	deviceCount int
-	countError  nvml.Return
-}
+func TestGetTags(t *testing.T) {
+	// Create a temporary directory for our test
+	tmpDir := t.TempDir()
 
-func (m *mockNVML) DeviceGetCount() (int, nvml.Return) {
-	return m.deviceCount, m.countError
-}
+	// Override ProcFSRoot to use our temporary directory
+	originalProcFSRoot := kernel.ProcFSRoot
+	kernel.ProcFSRoot = func() string { return tmpDir }
+	defer func() { kernel.ProcFSRoot = originalProcFSRoot }()
 
-func TestGetTags(t *testing.T) {
 	tests := []struct {
 		name     string
-		nvml     mockNVML
+		setup    func() error
 		wantTags []string
 	}{
 		{
-			name: "no GPUs",
-			nvml: mockNVML{
-				deviceCount: 0,
-				countError:  nvml.SUCCESS,
+			name: "no NVIDIA directory",
+			setup: func() error {
+				// Don't create anything, directory should not exist
+				return nil
+			},
+			wantTags: nil,
+		},
+		{
+			name: "NVIDIA directory exists but empty",
+			setup: func() error {
+				nvidiaPath := filepath.Join(tmpDir, "driver", "nvidia", "gpus")
+				return os.MkdirAll(nvidiaPath, 0755)
 			},
 			wantTags: nil,
 		},
 		{
-			name: "has GPUs",
-			nvml: mockNVML{
-				deviceCount: 2,
-				countError:  nvml.SUCCESS,
+			name: "NVIDIA directory with one GPU",
+			setup: func() error {
+				nvidiaPath := filepath.Join(tmpDir, "driver", "nvidia", "gpus")
+				if err := os.MkdirAll(nvidiaPath, 0755); err != nil {
+					return err
+				}
+				// Create a dummy GPU entry
+				return os.WriteFile(filepath.Join(nvidiaPath, "0"), []byte("dummy"), 0644)
+			},
+			wantTags: []string{"gpu_host:true"},
+		},
+		{
+			name: "NVIDIA directory with multiple GPUs",
+			setup: func() error {
+				nvidiaPath := filepath.Join(tmpDir, "driver", "nvidia", "gpus")
+				if err := os.MkdirAll(nvidiaPath, 0755); err != nil {
+					return err
+				}
+				// Create multiple dummy GPU entries
+				for i := 0; i < 2; i++ {
+					if err := os.WriteFile(filepath.Join(nvidiaPath, fmt.Sprintf("%d", i)), []byte("dummy"), 0644); err != nil {
+						return err
+					}
+				}
+				return nil
 			},
 			wantTags: []string{"gpu_host:true"},
 		},
 		{
-			name: "device count error",
-			nvml: mockNVML{
-				countError: nvml.ERROR_UNKNOWN,
+			name: "NVIDIA directory exists but not readable",
+			setup: func() error {
+				nvidiaPath := filepath.Join(tmpDir, "driver", "nvidia", "gpus")
+				if err := os.MkdirAll(nvidiaPath, 0755); err != nil {
+					return err
+				}
+				if err := os.Chmod(nvidiaPath, 0000); err != nil {
+					return err
+				}
+				return nil
 			},
 			wantTags: nil,
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			nvmlLibrary = &tt.nvml
-			gotTags := GetTags()
+			defer func() {
+				// Clean up any existing files before each test
+				if err := os.RemoveAll(filepath.Join(tmpDir, "driver")); err != nil {
+					t.Fatalf("Failed to clean up test directory: %v", err)
+				}
+			}()
path: pkg/gpu/tags/tags_test.go
user: login: gjulianm
body: nit: Instead of this, I'd create the tmpDir for each test and pass it as an argument to `setup()`, that way you're completely sure it's a new diretory tree
created_at: 2025-03-24T15:15:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -3,92 +3,111 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2016-present Datadog, Inc.
 
-//go:build linux && nvml
+//go:build linux
 
 package tags
 
 import (
+	"fmt"
+	"os"
+	"path/filepath"
 	"testing"
-	"time"
 
-	"github.com/NVIDIA/go-nvml/pkg/nvml"
+	"github.com/DataDog/datadog-agent/pkg/util/kernel"
 	"github.com/stretchr/testify/assert"
 )
 
-type mockNVML struct {
-	nvml.Interface
-	deviceCount int
-	countError  nvml.Return
-}
+func TestGetTags(t *testing.T) {
+	// Create a temporary directory for our test
+	tmpDir := t.TempDir()
 
-func (m *mockNVML) DeviceGetCount() (int, nvml.Return) {
-	return m.deviceCount, m.countError
-}
+	// Override ProcFSRoot to use our temporary directory
+	originalProcFSRoot := kernel.ProcFSRoot
+	kernel.ProcFSRoot = func() string { return tmpDir }
+	defer func() { kernel.ProcFSRoot = originalProcFSRoot }()
 
-func TestGetTags(t *testing.T) {
 	tests := []struct {
 		name     string
-		nvml     mockNVML
+		setup    func() error
 		wantTags []string
 	}{
 		{
-			name: "no GPUs",
-			nvml: mockNVML{
-				deviceCount: 0,
-				countError:  nvml.SUCCESS,
+			name: "no NVIDIA directory",
+			setup: func() error {
+				// Don't create anything, directory should not exist
+				return nil
+			},
+			wantTags: nil,
+		},
+		{
+			name: "NVIDIA directory exists but empty",
+			setup: func() error {
+				nvidiaPath := filepath.Join(tmpDir, "driver", "nvidia", "gpus")
+				return os.MkdirAll(nvidiaPath, 0755)
 			},
 			wantTags: nil,
 		},
 		{
-			name: "has GPUs",
-			nvml: mockNVML{
-				deviceCount: 2,
-				countError:  nvml.SUCCESS,
+			name: "NVIDIA directory with one GPU",
+			setup: func() error {
+				nvidiaPath := filepath.Join(tmpDir, "driver", "nvidia", "gpus")
+				if err := os.MkdirAll(nvidiaPath, 0755); err != nil {
+					return err
+				}
+				// Create a dummy GPU entry
+				return os.WriteFile(filepath.Join(nvidiaPath, "0"), []byte("dummy"), 0644)
+			},
+			wantTags: []string{"gpu_host:true"},
+		},
+		{
+			name: "NVIDIA directory with multiple GPUs",
+			setup: func() error {
+				nvidiaPath := filepath.Join(tmpDir, "driver", "nvidia", "gpus")
+				if err := os.MkdirAll(nvidiaPath, 0755); err != nil {
+					return err
+				}
+				// Create multiple dummy GPU entries
+				for i := 0; i < 2; i++ {
+					if err := os.WriteFile(filepath.Join(nvidiaPath, fmt.Sprintf("%d", i)), []byte("dummy"), 0644); err != nil {
+						return err
+					}
+				}
+				return nil
 			},
 			wantTags: []string{"gpu_host:true"},
 		},
 		{
-			name: "device count error",
-			nvml: mockNVML{
-				countError: nvml.ERROR_UNKNOWN,
+			name: "NVIDIA directory exists but not readable",
+			setup: func() error {
+				nvidiaPath := filepath.Join(tmpDir, "driver", "nvidia", "gpus")
+				if err := os.MkdirAll(nvidiaPath, 0755); err != nil {
+					return err
+				}
+				if err := os.Chmod(nvidiaPath, 0000); err != nil {
+					return err
+				}
+				return nil
 			},
 			wantTags: nil,
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			nvmlLibrary = &tt.nvml
-			gotTags := GetTags()
+			defer func() {
+				// Clean up any existing files before each test
+				if err := os.RemoveAll(filepath.Join(tmpDir, "driver")); err != nil {
+					t.Fatalf("Failed to clean up test directory: %v", err)
+				}
+			}()
path: pkg/gpu/tags/tags_test.go
user: login: val06
body: i would still need to have a cleanup function that removes those tmp dirs, so i don't really see a benefit in that approach
created_at: 2025-03-25T09:57:12Z
author_association: CONTRIBUTOR
diff_hunk: @@ -3,92 +3,111 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2016-present Datadog, Inc.
 
-//go:build linux && nvml
+//go:build linux
 
 package tags
 
 import (
+	"fmt"
+	"os"
+	"path/filepath"
 	"testing"
-	"time"
 
-	"github.com/NVIDIA/go-nvml/pkg/nvml"
+	"github.com/DataDog/datadog-agent/pkg/util/kernel"
 	"github.com/stretchr/testify/assert"
 )
 
-type mockNVML struct {
-	nvml.Interface
-	deviceCount int
-	countError  nvml.Return
-}
+func TestGetTags(t *testing.T) {
+	// Create a temporary directory for our test
+	tmpDir := t.TempDir()
 
-func (m *mockNVML) DeviceGetCount() (int, nvml.Return) {
-	return m.deviceCount, m.countError
-}
+	// Override ProcFSRoot to use our temporary directory
+	originalProcFSRoot := kernel.ProcFSRoot
+	kernel.ProcFSRoot = func() string { return tmpDir }
+	defer func() { kernel.ProcFSRoot = originalProcFSRoot }()
 
-func TestGetTags(t *testing.T) {
 	tests := []struct {
 		name     string
-		nvml     mockNVML
+		setup    func() error
 		wantTags []string
 	}{
 		{
-			name: "no GPUs",
-			nvml: mockNVML{
-				deviceCount: 0,
-				countError:  nvml.SUCCESS,
+			name: "no NVIDIA directory",
+			setup: func() error {
+				// Don't create anything, directory should not exist
+				return nil
+			},
+			wantTags: nil,
+		},
+		{
+			name: "NVIDIA directory exists but empty",
+			setup: func() error {
+				nvidiaPath := filepath.Join(tmpDir, "driver", "nvidia", "gpus")
+				return os.MkdirAll(nvidiaPath, 0755)
 			},
 			wantTags: nil,
 		},
 		{
-			name: "has GPUs",
-			nvml: mockNVML{
-				deviceCount: 2,
-				countError:  nvml.SUCCESS,
+			name: "NVIDIA directory with one GPU",
+			setup: func() error {
+				nvidiaPath := filepath.Join(tmpDir, "driver", "nvidia", "gpus")
+				if err := os.MkdirAll(nvidiaPath, 0755); err != nil {
+					return err
+				}
+				// Create a dummy GPU entry
+				return os.WriteFile(filepath.Join(nvidiaPath, "0"), []byte("dummy"), 0644)
+			},
+			wantTags: []string{"gpu_host:true"},
+		},
+		{
+			name: "NVIDIA directory with multiple GPUs",
+			setup: func() error {
+				nvidiaPath := filepath.Join(tmpDir, "driver", "nvidia", "gpus")
+				if err := os.MkdirAll(nvidiaPath, 0755); err != nil {
+					return err
+				}
+				// Create multiple dummy GPU entries
+				for i := 0; i < 2; i++ {
+					if err := os.WriteFile(filepath.Join(nvidiaPath, fmt.Sprintf("%d", i)), []byte("dummy"), 0644); err != nil {
+						return err
+					}
+				}
+				return nil
 			},
 			wantTags: []string{"gpu_host:true"},
 		},
 		{
-			name: "device count error",
-			nvml: mockNVML{
-				countError: nvml.ERROR_UNKNOWN,
+			name: "NVIDIA directory exists but not readable",
+			setup: func() error {
+				nvidiaPath := filepath.Join(tmpDir, "driver", "nvidia", "gpus")
+				if err := os.MkdirAll(nvidiaPath, 0755); err != nil {
+					return err
+				}
+				if err := os.Chmod(nvidiaPath, 0000); err != nil {
+					return err
+				}
+				return nil
 			},
 			wantTags: nil,
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			nvmlLibrary = &tt.nvml
-			gotTags := GetTags()
+			defer func() {
+				// Clean up any existing files before each test
+				if err := os.RemoveAll(filepath.Join(tmpDir, "driver")); err != nil {
+					t.Fatalf("Failed to clean up test directory: %v", err)
+				}
+			}()
path: pkg/gpu/tags/tags_test.go
user: login: gjulianm
body:  `t.TempDir` automatically cleans up the directory after the specific test is run
created_at: 2025-03-25T09:58:57Z
author_association: CONTRIBUTOR

PR comments:
user: login: cit-pr-commenter[bot]
created_at: 2025-03-24T15:23:42Z
body: ## Go Package Import Differences

Baseline: cdf3aca4e6ad11b58c605b8ad602f2de7a821d97
Comparison: dd34783d9c33f4e899d782207dfcde6d3bfe47e3

<table><thead><tr><th>binary</th><th>os</th><th>arch</th><th>change</th></tr></thead><tbody>
<tr><td>serverless</td><td>linux</td><td>amd64</td><td><details><summary>+1, -0</summary>
<pre lang='diff'>
+github.com/DataDog/datadog-agent/pkg/util/funcs
</pre></details></td></tr>
<tr><td>serverless</td><td>linux</td><td>arm64</td><td><details><summary>+1, -0</summary>
<pre lang='diff'>
+github.com/DataDog/datadog-agent/pkg/util/funcs
</pre></details></td></tr>
</tbody></table>
author_association: NONE
user: login: agent-platform-auto-pr[bot]
created_at: 2025-03-24T16:57:38Z
body: ## Test changes on VM

Use this command from [test-infra-definitions](https://github.com/DataDog/test-infra-definitions) to manually test this PR changes on a VM:
```sh
dda inv aws.create-vm --pipeline-id=59867460 --os-family=ubuntu
```
Note: This applies to commit **dd34783d**
author_association: CONTRIBUTOR
user: login: agent-platform-auto-pr[bot]
created_at: 2025-03-24T16:59:23Z
body: ## Uncompressed package size comparison

Comparison with [ancestor](https://github.com/DataDog/datadog-agent/commit/cdf3aca4e6ad11b58c605b8ad602f2de7a821d97) `cdf3aca4e6ad11b58c605b8ad602f2de7a821d97`
<details open>
<summary> Size reduction summary </summary>

|package|diff|status|size|ancestor|threshold|
|--|--|--|--|--|--|
|datadog-agent-aarch64-rpm|-0.00MB|✅|808.00MB|808.00MB|0.50MB|
|datadog-agent-arm64-deb|-0.00MB|✅|798.18MB|798.18MB|0.50MB|

</details>


<details open>
  <summary> Diff per package </summary>

|package|diff|status|size|ancestor|threshold|
|--|--|--|--|--|--|
|datadog-iot-agent-x86_64-rpm|0.01MB|⚠️|61.21MB|61.20MB|0.50MB|
|datadog-iot-agent-x86_64-suse|0.01MB|⚠️|61.21MB|61.20MB|0.50MB|
|datadog-dogstatsd-amd64-deb|0.01MB|⚠️|39.50MB|39.49MB|0.50MB|
|datadog-dogstatsd-x86_64-rpm|0.01MB|⚠️|39.58MB|39.57MB|0.50MB|
|datadog-dogstatsd-x86_64-suse|0.01MB|⚠️|39.58MB|39.57MB|0.50MB|
|datadog-iot-agent-amd64-deb|0.01MB|⚠️|61.14MB|61.13MB|0.50MB|
|datadog-dogstatsd-arm64-deb|0.00MB|✅|38.01MB|38.01MB|0.50MB|
|datadog-agent-amd64-deb|0.00MB|✅|807.18MB|807.18MB|0.50MB|
|datadog-agent-x86_64-rpm|0.00MB|✅|817.02MB|817.02MB|0.50MB|
|datadog-agent-x86_64-suse|0.00MB|✅|817.02MB|817.02MB|0.50MB|
|datadog-iot-agent-aarch64-rpm|0.00MB|✅|58.50MB|58.50MB|0.50MB|
|datadog-heroku-agent-amd64-deb|0.00MB|✅|442.30MB|442.30MB|0.50MB|
|datadog-iot-agent-arm64-deb|0.00MB|✅|58.43MB|58.43MB|0.50MB|

</details>

## Decision
⚠️ Warning



author_association: CONTRIBUTOR
user: login: agent-platform-auto-pr[bot]
created_at: 2025-03-24T17:02:22Z
body: ## Static quality checks ✅

Please find below the results from static quality gates


<details>
<summary>Successful checks</summary>

### Info

|Result|Quality gate|On disk size|On disk size limit|On wire size|On wire size limit|
|----|----|----|----|----|----|
|✅|static_quality_gate_agent_deb_amd64|781.3 MiB|801.8 MiB|190.75 MiB|202.62 MiB|
|✅|static_quality_gate_agent_deb_arm64|772.67 MiB|793.14 MiB|172.91 MiB|184.51 MiB|
|✅|static_quality_gate_agent_rpm_amd64|781.34 MiB|801.79 MiB|193.19 MiB|205.03 MiB|
|✅|static_quality_gate_agent_rpm_arm64|772.65 MiB|793.09 MiB|174.26 MiB|186.44 MiB|
|✅|static_quality_gate_agent_suse_amd64|781.29 MiB|801.81 MiB|193.19 MiB|205.03 MiB|
|✅|static_quality_gate_agent_suse_arm64|772.64 MiB|793.14 MiB|174.26 MiB|186.44 MiB|
|✅|static_quality_gate_dogstatsd_deb_amd64|37.74 MiB|47.67 MiB|9.78 MiB|19.78 MiB|
|✅|static_quality_gate_dogstatsd_deb_arm64|36.33 MiB|46.27 MiB|8.48 MiB|18.49 MiB|
|✅|static_quality_gate_dogstatsd_rpm_amd64|37.74 MiB|47.67 MiB|9.79 MiB|19.79 MiB|
|✅|static_quality_gate_dogstatsd_suse_amd64|37.74 MiB|47.67 MiB|9.79 MiB|19.79 MiB|
|✅|static_quality_gate_iot_agent_deb_amd64|58.39 MiB|69.0 MiB|14.75 MiB|24.8 MiB|
|✅|static_quality_gate_iot_agent_deb_arm64|55.8 MiB|66.4 MiB|12.75 MiB|22.8 MiB|
|✅|static_quality_gate_iot_agent_rpm_amd64|58.39 MiB|69.0 MiB|14.77 MiB|24.8 MiB|
|✅|static_quality_gate_iot_agent_rpm_arm64|55.8 MiB|66.4 MiB|12.77 MiB|22.8 MiB|
|✅|static_quality_gate_iot_agent_suse_amd64|58.39 MiB|69.0 MiB|14.77 MiB|24.8 MiB|
|✅|static_quality_gate_docker_agent_amd64|865.98 MiB|886.12 MiB|291.38 MiB|304.21 MiB|
|✅|static_quality_gate_docker_agent_arm64|880.66 MiB|900.79 MiB|277.83 MiB|290.47 MiB|
|✅|static_quality_gate_docker_agent_jmx_amd64|1.04 GiB|1.06 GiB|366.47 MiB|379.33 MiB|
|✅|static_quality_gate_docker_agent_jmx_arm64|1.04 GiB|1.06 GiB|348.93 MiB|361.55 MiB|
|✅|static_quality_gate_docker_dogstatsd_amd64|45.88 MiB|55.78 MiB|17.3 MiB|27.28 MiB|
|✅|static_quality_gate_docker_dogstatsd_arm64|44.51 MiB|54.45 MiB|16.17 MiB|26.16 MiB|
|✅|static_quality_gate_docker_cluster_agent_amd64|262.81 MiB|274.78 MiB|105.54 MiB|116.28 MiB|
|✅|static_quality_gate_docker_cluster_agent_arm64|278.77 MiB|290.82 MiB|100.38 MiB|111.12 MiB|

</details>

author_association: CONTRIBUTOR
user: login: cit-pr-commenter[bot]
created_at: 2025-03-24T17:18:14Z
body: ## Regression Detector

# Regression Detector Results

[Metrics dashboard](https://app.datadoghq.com/dashboard/ykh-ua8-vcu/SMP-Regression-Detector-Metrics?fromUser=true&refresh_mode=paused&tpl_var_run-id%5B0%5D=651ae890-29e0-4fe4-9063-fad94d655ec3&view=spans&from_ts=1742897547000&to_ts=1742898147000&live=false)  
[Target profiles](https://app.datadoghq.com/profiling/explorer?query=env%3Asingle-machine-performance%20service%3Adatadog-agent%20job_id%3A651ae890-29e0-4fe4-9063-fad94d655ec3&agg_m=count&agg_m_source=base&agg_t=count&fromUser=false&viz=stream&start=1742890347000&end=1742901747000&paused=true)  
Run ID: 651ae890-29e0-4fe4-9063-fad94d655ec3

Baseline: cdf3aca4e6ad11b58c605b8ad602f2de7a821d97
Comparison: dd34783d9c33f4e899d782207dfcde6d3bfe47e3
[Diff](cdf3aca4e6ad11b58c605b8ad602f2de7a821d97...dd34783d9c33f4e899d782207dfcde6d3bfe47e3)


## Optimization Goals: ✅ No significant changes detected


<details>
<summary><h2>
Fine details of change detection per experiment
</h2></summary>

| perf | experiment                                   | goal               | Δ mean % | Δ mean % CI    | trials | links                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
|------|----------------------------------------------|--------------------|----------|----------------|--------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| ➖   | uds_dogstatsd_to_api_cpu                     | % cpu utilization  | +2.65    | [+1.81, +3.50] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Auds_dogstatsd_to_api_cpu%20run_id%3A651ae890-29e0-4fe4-9063-fad94d655ec3&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742890347000&to_ts=1742901747000&live=false)                                                                                                                                                                                                                                                                                                                                                                   |
| ➖   | otlp_ingest_traces                           | memory utilization | +1.89    | [+1.42, +2.37] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Aotlp_ingest_traces%20run_id%3A651ae890-29e0-4fe4-9063-fad94d655ec3&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742890347000&to_ts=1742901747000&live=false)                                                                                                                                                                                                                                                                                                                                                                         |
| ➖   | otlp_ingest_metrics                          | memory utilization | +1.62    | [+1.47, +1.78] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Aotlp_ingest_metrics%20run_id%3A651ae890-29e0-4fe4-9063-fad94d655ec3&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742890347000&to_ts=1742901747000&live=false)                                                                                                                                                                                                                                                                                                                                                                        |
| ➖   | file_tree                                    | memory utilization | +1.15    | [+0.92, +1.37] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_tree%20run_id%3A651ae890-29e0-4fe4-9063-fad94d655ec3&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742890347000&to_ts=1742901747000&live=false)                                                                                                                                                                                                                                                                                                                                                                                  |
| ➖   | quality_gate_idle_all_features               | memory utilization | +0.75    | [+0.62, +0.89] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Aquality_gate_idle_all_features%20run_id%3A651ae890-29e0-4fe4-9063-fad94d655ec3&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742890347000&to_ts=1742901747000&live=false) [bounds checks dashboard](https://app.datadoghq.com/dashboard/vz3-jd5-bdi?fromUser=true&refresh_mode=paused&tpl_var_experiment%5B0%5D=quality_gate_idle_all_features&tpl_var_job_id%5B0%5D=651ae890-29e0-4fe4-9063-fad94d655ec3&tpl_var_run-id%5B0%5D=651ae890-29e0-4fe4-9063-fad94d655ec3&view=spans&from_ts=1742897547000&to_ts=1742898147000&live=false) |
| ➖   | file_to_blackhole_1000ms_latency             | egress throughput  | +0.36    | [-0.41, +1.13] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_1000ms_latency%20run_id%3A651ae890-29e0-4fe4-9063-fad94d655ec3&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742890347000&to_ts=1742901747000&live=false)                                                                                                                                                                                                                                                                                                                                                           |
| ➖   | file_to_blackhole_1000ms_latency_linear_load | egress throughput  | +0.18    | [-0.29, +0.65] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_1000ms_latency_linear_load%20run_id%3A651ae890-29e0-4fe4-9063-fad94d655ec3&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742890347000&to_ts=1742901747000&live=false)                                                                                                                                                                                                                                                                                                                                               |
| ➖   | otlp_ingest_logs                             | memory utilization | +0.08    | [-0.08, +0.24] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Aotlp_ingest_logs%20run_id%3A651ae890-29e0-4fe4-9063-fad94d655ec3&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742890347000&to_ts=1742901747000&live=false)                                                                                                                                                                                                                                                                                                                                                                           |
| ➖   | file_to_blackhole_100ms_latency              | egress throughput  | +0.02    | [-0.64, +0.69] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_100ms_latency%20run_id%3A651ae890-29e0-4fe4-9063-fad94d655ec3&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742890347000&to_ts=1742901747000&live=false)                                                                                                                                                                                                                                                                                                                                                            |
| ➖   | uds_dogstatsd_to_api                         | ingress throughput | +0.01    | [-0.29, +0.32] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Auds_dogstatsd_to_api%20run_id%3A651ae890-29e0-4fe4-9063-fad94d655ec3&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742890347000&to_ts=1742901747000&live=false)                                                                                                                                                                                                                                                                                                                                                                       |
| ➖   | file_to_blackhole_0ms_latency_http2          | egress throughput  | +0.00    | [-0.74, +0.75] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_0ms_latency_http2%20run_id%3A651ae890-29e0-4fe4-9063-fad94d655ec3&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742890347000&to_ts=1742901747000&live=false)                                                                                                                                                                                                                                                                                                                                                        |
| ➖   | file_to_blackhole_300ms_latency              | egress throughput  | +0.00    | [-0.63, +0.64] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_300ms_latency%20run_id%3A651ae890-29e0-4fe4-9063-fad94d655ec3&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742890347000&to_ts=1742901747000&live=false)                                                                                                                                                                                                                                                                                                                                                            |
| ➖   | tcp_dd_logs_filter_exclude                   | ingress throughput | +0.00    | [-0.01, +0.02] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Atcp_dd_logs_filter_exclude%20run_id%3A651ae890-29e0-4fe4-9063-fad94d655ec3&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742890347000&to_ts=1742901747000&live=false)                                                                                                                                                                                                                                                                                                                                                                 |
| ➖   | file_to_blackhole_500ms_latency              | egress throughput  | -0.00    | [-0.78, +0.77] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_500ms_latency%20run_id%3A651ae890-29e0-4fe4-9063-fad94d655ec3&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742890347000&to_ts=1742901747000&live=false)                                                                                                                                                                                                                                                                                                                                                            |
| ➖   | file_to_blackhole_0ms_latency                | egress throughput  | -0.01    | [-0.85, +0.82] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_0ms_latency%20run_id%3A651ae890-29e0-4fe4-9063-fad94d655ec3&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742890347000&to_ts=1742901747000&live=false)                                                                                                                                                                                                                                                                                                                                                              |
| ➖   | file_to_blackhole_0ms_latency_http1          | egress throughput  | -0.02    | [-0.77, +0.73] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Afile_to_blackhole_0ms_latency_http1%20run_id%3A651ae890-29e0-4fe4-9063-fad94d655ec3&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742890347000&to_ts=1742901747000&live=false)                                                                                                                                                                                                                                                                                                                                                        |
| ➖   | tcp_syslog_to_blackhole                      | ingress throughput | -0.19    | [-0.27, -0.12] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Atcp_syslog_to_blackhole%20run_id%3A651ae890-29e0-4fe4-9063-fad94d655ec3&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742890347000&to_ts=1742901747000&live=false)                                                                                                                                                                                                                                                                                                                                                                    |
| ➖   | quality_gate_idle                            | memory utilization | -0.20    | [-0.28, -0.12] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Aquality_gate_idle%20run_id%3A651ae890-29e0-4fe4-9063-fad94d655ec3&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742890347000&to_ts=1742901747000&live=false) [bounds checks dashboard](https://app.datadoghq.com/dashboard/vz3-jd5-bdi?fromUser=true&refresh_mode=paused&tpl_var_experiment%5B0%5D=quality_gate_idle&tpl_var_job_id%5B0%5D=651ae890-29e0-4fe4-9063-fad94d655ec3&tpl_var_run-id%5B0%5D=651ae890-29e0-4fe4-9063-fad94d655ec3&view=spans&from_ts=1742897547000&to_ts=1742898147000&live=false)                           |
| ➖   | uds_dogstatsd_20mb_12k_contexts_20_senders   | memory utilization | -0.50    | [-0.56, -0.43] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Auds_dogstatsd_20mb_12k_contexts_20_senders%20run_id%3A651ae890-29e0-4fe4-9063-fad94d655ec3&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742890347000&to_ts=1742901747000&live=false)                                                                                                                                                                                                                                                                                                                                                 |
| ➖   | quality_gate_logs                            | % cpu utilization  | -1.16    | [-4.02, +1.70] | 1      | [Logs](https://app.datadoghq.com/logs?query=experiment%3Aquality_gate_logs%20run_id%3A651ae890-29e0-4fe4-9063-fad94d655ec3&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts=1742890347000&to_ts=1742901747000&live=false)                                                                                                                                                                                                                                                                                                                                                                          |
</details>


<details>
<summary><h2>
Bounds Checks: ✅ Passed
</h2></summary>

| perf | experiment                                   | bounds_check_name  | replicates_passed | links                                                                                                                                                                                                                                                                                                                                                       |
|------|----------------------------------------------|--------------------|-------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| ✅   | file_to_blackhole_0ms_latency                | lost_bytes         | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_0ms_latency                | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_0ms_latency_http1          | lost_bytes         | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_0ms_latency_http1          | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_0ms_latency_http2          | lost_bytes         | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_0ms_latency_http2          | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_1000ms_latency             | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_1000ms_latency_linear_load | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_100ms_latency              | lost_bytes         | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_100ms_latency              | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_300ms_latency              | lost_bytes         | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_300ms_latency              | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_500ms_latency              | lost_bytes         | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | file_to_blackhole_500ms_latency              | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | quality_gate_idle                            | intake_connections | 10/10             | [bounds checks dashboard](https://app.datadoghq.com/dashboard/vz3-jd5-bdi?fromUser=true&refresh_mode=paused&tpl_var_experiment%5B0%5D=quality_gate_idle&tpl_var_job_id%5B0%5D=651ae890-29e0-4fe4-9063-fad94d655ec3&tpl_var_run-id%5B0%5D=651ae890-29e0-4fe4-9063-fad94d655ec3&view=spans&from_ts=1742897547000&to_ts=1742898147000&live=false)              |
| ✅   | quality_gate_idle                            | memory_usage       | 10/10             | [bounds checks dashboard](https://app.datadoghq.com/dashboard/vz3-jd5-bdi?fromUser=true&refresh_mode=paused&tpl_var_experiment%5B0%5D=quality_gate_idle&tpl_var_job_id%5B0%5D=651ae890-29e0-4fe4-9063-fad94d655ec3&tpl_var_run-id%5B0%5D=651ae890-29e0-4fe4-9063-fad94d655ec3&view=spans&from_ts=1742897547000&to_ts=1742898147000&live=false)              |
| ✅   | quality_gate_idle_all_features               | intake_connections | 10/10             | [bounds checks dashboard](https://app.datadoghq.com/dashboard/vz3-jd5-bdi?fromUser=true&refresh_mode=paused&tpl_var_experiment%5B0%5D=quality_gate_idle_all_features&tpl_var_job_id%5B0%5D=651ae890-29e0-4fe4-9063-fad94d655ec3&tpl_var_run-id%5B0%5D=651ae890-29e0-4fe4-9063-fad94d655ec3&view=spans&from_ts=1742897547000&to_ts=1742898147000&live=false) |
| ✅   | quality_gate_idle_all_features               | memory_usage       | 10/10             | [bounds checks dashboard](https://app.datadoghq.com/dashboard/vz3-jd5-bdi?fromUser=true&refresh_mode=paused&tpl_var_experiment%5B0%5D=quality_gate_idle_all_features&tpl_var_job_id%5B0%5D=651ae890-29e0-4fe4-9063-fad94d655ec3&tpl_var_run-id%5B0%5D=651ae890-29e0-4fe4-9063-fad94d655ec3&view=spans&from_ts=1742897547000&to_ts=1742898147000&live=false) |
| ✅   | quality_gate_logs                            | intake_connections | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | quality_gate_logs                            | lost_bytes         | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
| ✅   | quality_gate_logs                            | memory_usage       | 10/10             |                                                                                                                                                                                                                                                                                                                                                             |
</details>


<details>
<summary><h2>
Explanation
</h2></summary>

**Confidence level:** 90.00%
**Effect size tolerance:** |Δ mean %| ≥ 5.00%

Performance changes are noted in the **perf** column of each table:

* ✅ = significantly better comparison variant performance
* ❌ = significantly worse comparison variant performance
* ➖ = no significant change in performance

A regression test is an A/B test of target performance in a repeatable rig, where "performance" is measured as "comparison variant minus baseline variant" for an optimization goal (e.g., ingress throughput). Due to intrinsic variability in measuring that goal, we can only estimate its mean value for each experiment; we report uncertainty in that value as a 90.00% confidence interval denoted "Δ mean % CI".

For each experiment, we decide whether a change in performance is a "regression" -- a change worth investigating further -- if all of the following criteria are true:

1. Its estimated |Δ mean %| ≥ 5.00%, indicating the change is big enough to merit a closer look.

2. Its 90.00% confidence interval "Δ mean % CI" does not contain zero, indicating that *if our statistical model is accurate*, there is at least a 90.00% chance there is a difference in performance between baseline and comparison variants.

3. Its configuration does not mark it "erratic".
</details>

## CI Pass/Fail Decision

✅ **Passed.** All Quality Gates passed.

- **quality_gate_idle_all_features**, bounds check **intake_connections**: 10/10 replicas passed. Gate passed.
- **quality_gate_idle_all_features**, bounds check **memory_usage**: 10/10 replicas passed. Gate passed.
- **quality_gate_idle**, bounds check **memory_usage**: 10/10 replicas passed. Gate passed.
- **quality_gate_idle**, bounds check **intake_connections**: 10/10 replicas passed. Gate passed.
- **quality_gate_logs**, bounds check **lost_bytes**: 10/10 replicas passed. Gate passed.
- **quality_gate_logs**, bounds check **memory_usage**: 10/10 replicas passed. Gate passed.
- **quality_gate_logs**, bounds check **intake_connections**: 10/10 replicas passed. Gate passed.
author_association: NONE
user: login: val06
created_at: 2025-03-25T11:35:03Z
body: /merge
author_association: CONTRIBUTOR
user: login: dd-devflow[bot]
created_at: 2025-03-25T11:35:07Z
body: <!-- "FeedbackWorkflowId":"6feb2cc1-b896-4fff-bf3f-794e9f7a9c33_36" -->
[View all feedbacks in Devflow UI.](https://devflow.static-app.us1.prod.dog/logs/6feb2cc1-b896-4fff-bf3f-794e9f7a9c33_36)
`2025-03-25 11:35:07 UTC` :information_source: Start processing command `/merge`

---
`2025-03-25 11:35:11 UTC` :information_source: **MergeQueue:** pull request added to the queue

The expected merge time in `main` is approximately 2h (p90).

---
`2025-03-25 11:44:44 UTC` :information_source: **MergeQueue:** The merge request ahead of this one was cancelled, retrying soon

---
`2025-03-25 12:29:38 UTC` :information_source: **MergeQueue:** This merge request was merged

author_association: NONE