Repository: envoyproxy/ai-gateway
Pull request #377 (2025-02-19 18:13:40)

extproc: do not modify original headers when debug enabled
**Commit Message**
 When debug level enabled, extproc logs the headers on request paths, and it reduces the sensitive value, especially authorization header values as a security measure. However, previously it modifies the original object accidentally hence it had caused a hard-to-debug unexpected behavior. This fixes the bug and ensures that the reduction is not applied to the original objects.


Thanks for the report by Bloomberg team @yuzisun 

**Related Issues/PRs (if applicable)**

Fixes a bug in #217 

mathetake: @nacx PTAL

Patch:
diff --git a/filterapi/filterconfig.go b/filterapi/filterconfig.go
index f4e64d7..da5201c 100644
--- a/filterapi/filterconfig.go
+++ b/filterapi/filterconfig.go
@@ -186,14 +186,24 @@ type APIKeyAuth struct {
 }
 
 // UnmarshalConfigYaml reads the file at the given path and unmarshals it into a Config struct.
-func UnmarshalConfigYaml(path string) (*Config, error) {
+func UnmarshalConfigYaml(path string) (*Config, []byte, error) {
 	raw, err := os.ReadFile(path)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	var cfg Config
 	if err := yaml.Unmarshal(raw, &cfg); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	return &cfg, nil
+	return &cfg, raw, nil
+}
+
+// MustLoadDefaultConfig loads the default configuration.
+// This panics if the configuration fails to be loaded.
+func MustLoadDefaultConfig() (*Config, []byte) {
+	var cfg Config
+	if err := yaml.Unmarshal([]byte(DefaultConfig), &cfg); err != nil {
+		panic(err)
+	}
+	return &cfg, []byte(DefaultConfig)
 }
diff --git a/internal/controller/backend_security_policy.go b/internal/controller/backend_security_policy.go
index 9f8cae1..d0c7f47 100644
--- a/internal/controller/backend_security_policy.go
+++ b/internal/controller/backend_security_policy.go
@@ -88,6 +88,10 @@ func (c *backendSecurityPolicyController) Reconcile(ctx context.Context, req ctr
 				requeue, err = c.rotateCredential(ctx, &backendSecurityPolicy, *oidc, rotator)
 				if err != nil {
 					c.logger.Error(err, "failed to rotate OIDC exchange token, retry in one minute")
+				} else {
+					c.logger.Info(
+						fmt.Sprintf("successfully rotated credentials for %s in namespace %s of auth type %s, renewing in %f minutes",
+							req.Name, req.Namespace, backendSecurityPolicy.Spec.Type, requeue.Minutes()))
 				}
 			} else {
 				requeue = time.Until(rotationTime)
diff --git a/internal/extproc/server.go b/internal/extproc/server.go
index 444535a..0dbedfd 100644
--- a/internal/extproc/server.go
+++ b/internal/extproc/server.go
@@ -30,12 +30,11 @@ import (
 	"github.com/envoyproxy/ai-gateway/internal/llmcostcel"
 )
 
-const (
-	redactedKey = "[REDACTED]"
+var (
+	sensitiveHeaderRedactedValue = []byte("[REDACTED]")
+	sensitiveHeaderKeys          = []string{"authorization"}
 )
 
-var sensitiveHeaderKeys = []string{"authorization"}
-
 // Server implements the external processor server.
 type Server struct {
 	logger     *slog.Logger
@@ -185,7 +184,7 @@ func (s *Server) processMsg(ctx context.Context, p Processor, req *extprocv3.Pro
 		requestHdrs := req.GetRequestHeaders().Headers
 		// If DEBUG log level is enabled, filter sensitive headers before logging.
 		if s.logger.Enabled(ctx, slog.LevelDebug) {
-			filteredHdrs := filterSensitiveHeaders(requestHdrs, s.logger, sensitiveHeaderKeys)
+			filteredHdrs := filterSensitiveHeadersForLogging(requestHdrs, sensitiveHeaderKeys)
 			s.logger.Debug("request headers processing", slog.Any("request_headers", filteredHdrs))
 		}
 		resp, err := p.ProcessRequestHeaders(ctx, requestHdrs)
@@ -199,7 +198,7 @@ func (s *Server) processMsg(ctx context.Context, p Processor, req *extprocv3.Pro
 		resp, err := p.ProcessRequestBody(ctx, value.RequestBody)
 		// If DEBUG log level is enabled, filter sensitive body before logging.
 		if s.logger.Enabled(ctx, slog.LevelDebug) {
-			filteredBody := filterSensitiveBody(resp, s.logger, sensitiveHeaderKeys)
+			filteredBody := filterSensitiveBodyForLogging(resp, s.logger, sensitiveHeaderKeys)
 			s.logger.Debug("request body processed", slog.Any("response", filteredBody))
 		}
 		if err != nil {
@@ -239,59 +238,69 @@ func (s *Server) Watch(*grpc_health_v1.HealthCheckRequest, grpc_health_v1.Health
 	return status.Error(codes.Unimplemented, "Watch is not implemented")
 }
 
-// filterSensitiveHeaders filters out sensitive headers from the provided HeaderMap.
+// filterSensitiveHeadersForLogging filters out sensitive headers from the provided HeaderMap for logging.
 // Specifically, it redacts the value of the "authorization" header and logs this action.
-// The function returns a new HeaderMap with the filtered headers.
-func filterSensitiveHeaders(headers *corev3.HeaderMap, logger *slog.Logger, sensitiveKeys []string) *corev3.HeaderMap {
+// This returns a slice of [slog.Attr] of headers, where the value of sensitive headers is redacted.
+func filterSensitiveHeadersForLogging(headers *corev3.HeaderMap, sensitiveKeys []string) []slog.Attr {
 	if headers == nil {
-		logger.Debug("received nil HeaderMap, returning empty HeaderMap")
-		return &corev3.HeaderMap{}
+		return nil
 	}
-	filteredHeaders := &corev3.HeaderMap{}
-	for _, header := range headers.Headers {
+	filteredHeaders := make([]slog.Attr, len(headers.Headers))
+	for i, header := range headers.Headers {
 		// We convert the header key to lowercase to make the comparison case-insensitive but we don't modify the original header.
 		if slices.Contains(sensitiveKeys, strings.ToLower(header.GetKey())) {
-			logger.Debug("filtering sensitive header", slog.String("header_key", header.Key))
-			filteredHeaders.Headers = append(filteredHeaders.Headers, &corev3.HeaderValue{
-				Key:   header.Key,
-				Value: redactedKey,
-			})
+			filteredHeaders[i] = slog.String(header.GetKey(), string(sensitiveHeaderRedactedValue))
 		} else {
-			filteredHeaders.Headers = append(filteredHeaders.Headers, header)
+			if len(header.Value) > 0 {
+				filteredHeaders[i] = slog.String(header.GetKey(), header.Value)
+			} else if utf8.Valid(header.RawValue) {
+				filteredHeaders[i] = slog.String(header.GetKey(), string(header.RawValue))
+			}
 		}
 	}
 	return filteredHeaders
 }
 
-// filterSensitiveBody filters out sensitive information from the response body.
+// filterSensitiveBodyForLogging filters out sensitive information from the response body.
 // It creates a copy of the response body to avoid modifying the original body,
 // as the API Key is needed for the request. The function returns a new
 // ProcessingResponse with the filtered body for logging.
-func filterSensitiveBody(resp *extprocv3.ProcessingResponse, logger *slog.Logger, sensitiveKeys []string) *extprocv3.ProcessingResponse {
+func filterSensitiveBodyForLogging(resp *extprocv3.ProcessingResponse, logger *slog.Logger, sensitiveKeys []string) *extprocv3.ProcessingResponse {
 	if resp == nil {
-		logger.Debug("received nil ProcessingResponse, returning empty ProcessingResponse")
 		return &extprocv3.ProcessingResponse{}
 	}
-	filteredResp := &extprocv3.ProcessingResponse{
+	original := resp.Response.(*extprocv3.ProcessingResponse_RequestBody)
+	originalHeaderMutation := original.RequestBody.Response.GetHeaderMutation()
+	redactedHeaderMutation := &extprocv3.HeaderMutation{
+		RemoveHeaders: originalHeaderMutation.GetRemoveHeaders(),
+		SetHeaders:    make([]*corev3.HeaderValueOption, 0, len(originalHeaderMutation.GetSetHeaders())),
+	}
+	for _, setHeader := range originalHeaderMutation.GetSetHeaders() {
+		// We convert the header key to lowercase to make the comparison case-insensitive, but we don't modify the original header.
+		if slices.Contains(sensitiveKeys, strings.ToLower(setHeader.Header.GetKey())) {
+			logger.Debug("filtering sensitive header", slog.String("header_key", setHeader.Header.Key))
+			redactedHeaderMutation.SetHeaders = append(redactedHeaderMutation.SetHeaders, &corev3.HeaderValueOption{
+				Header: &corev3.HeaderValue{
+					Key:      setHeader.Header.Key,
+					RawValue: sensitiveHeaderRedactedValue,
+				},
+			})
+		} else {
+			redactedHeaderMutation.SetHeaders = append(redactedHeaderMutation.SetHeaders, setHeader)
+		}
+	}
+	return &extprocv3.ProcessingResponse{
 		Response: &extprocv3.ProcessingResponse_RequestBody{
 			RequestBody: &extprocv3.BodyResponse{
 				Response: &extprocv3.CommonResponse{
-					HeaderMutation:  resp.Response.(*extprocv3.ProcessingResponse_RequestBody).RequestBody.Response.GetHeaderMutation(),
-					BodyMutation:    resp.Response.(*extprocv3.ProcessingResponse_RequestBody).RequestBody.Response.GetBodyMutation(),
-					ClearRouteCache: resp.Response.(*extprocv3.ProcessingResponse_RequestBody).RequestBody.Response.GetClearRouteCache(),
+					HeaderMutation:  redactedHeaderMutation,
+					BodyMutation:    original.RequestBody.Response.GetBodyMutation(),
+					ClearRouteCache: original.RequestBody.Response.GetClearRouteCache(),
 				},
 			},
 		},
 		ModeOverride: resp.ModeOverride,
 	}
-	for _, setHeader := range filteredResp.Response.(*extprocv3.ProcessingResponse_RequestBody).RequestBody.Response.GetHeaderMutation().GetSetHeaders() {
-		// We convert the header key to lowercase to make the comparison case-insensitive but we don't modify the original header.
-		if slices.Contains(sensitiveKeys, strings.ToLower(setHeader.Header.GetKey())) {
-			logger.Debug("filtering sensitive header", slog.String("header_key", setHeader.Header.Key))
-			setHeader.Header.RawValue = []byte(redactedKey)
-		}
-	}
-	return filteredResp
 }
 
 // headersToMap converts a [corev3.HeaderMap] to a Go map for easier processing.
diff --git a/internal/extproc/watcher.go b/internal/extproc/watcher.go
index f6bba43..e3cd796 100644
--- a/internal/extproc/watcher.go
+++ b/internal/extproc/watcher.go
@@ -65,45 +65,47 @@ func (cw *configWatcher) watch(ctx context.Context, tick time.Duration) {
 // loadConfig loads a new config from the given path and updates the Receiver by
 // calling the [Receiver.Load].
 func (cw *configWatcher) loadConfig(ctx context.Context) error {
+	var (
+		cfg *filterapi.Config
+		raw []byte
+	)
+
 	stat, err := os.Stat(cw.path)
-	if err != nil {
+	switch {
+	case err != nil && os.IsNotExist(err):
+		// If the file does not exist, do not fail (which could lead to the extproc process to terminate)
+		// Instead, load the default configuration and keep running unconfigured
+		cfg, raw = filterapi.MustLoadDefaultConfig()
+	case err != nil:
 		return err
 	}
-	if stat.ModTime().Sub(cw.lastMod) <= 0 {
-		return nil
+
+	if cfg != nil {
+		cw.l.Info("config file does not exist; loading default config", slog.String("path", cw.path))
+		cw.lastMod = time.Now()
+	} else {
+		cw.l.Info("loading a new config", slog.String("path", cw.path))
+		if stat.ModTime().Sub(cw.lastMod) <= 0 {
+			return nil
+		}
+		cw.lastMod = stat.ModTime()
+		cfg, raw, err = filterapi.UnmarshalConfigYaml(cw.path)
+		if err != nil {
+			return err
+		}
 	}
-	cw.lastMod = stat.ModTime()
-	cw.l.Info("loading a new config", slog.String("path", cw.path))
 
 	// Print the diff between the old and new config.
 	if cw.l.Enabled(ctx, slog.LevelDebug) {
 		// Re-hydrate the current config file for later diffing.
 		previous := cw.current
-		cw.current, err = cw.getConfigString()
-		if err != nil {
-			return fmt.Errorf("failed to read the config file: %w", err)
-		}
-
+		cw.current = string(raw)
 		cw.diff(previous, cw.current)
 	}
 
-	cfg, err := filterapi.UnmarshalConfigYaml(cw.path)
-	if err != nil {
-		return err
-	}
 	return cw.rcv.LoadConfig(ctx, cfg)
 }
 
-// getConfigString gets a string representation of the current config
-// read from the path. This is only used for debug log path for diff prints.
-func (cw *configWatcher) getConfigString() (string, error) {
-	currentByte, err := os.ReadFile(cw.path)
-	if err != nil {
-		return "", err
-	}
-	return string(currentByte), nil
-}
-
 func (cw *configWatcher) diff(oldConfig, newConfig string) {
 	if oldConfig == "" {
 		return
diff --git a/site/docs/concepts/architecture/control-plane.md b/site/docs/concepts/architecture/control-plane.md
index 8909a37..18cb501 100644
--- a/site/docs/concepts/architecture/control-plane.md
+++ b/site/docs/concepts/architecture/control-plane.md
@@ -14,7 +14,7 @@ The control plane is responsible for configuring and managing the system. It con
 
 The control plane operates through a chain of components that work together to manage the configuration:
 
-1. The Envoy AI Gateway controller watches the AI Gateway CRD
+1. The Envoy AI Gateway controller watches AI Gateway Custom Resources (CRs)
 2. When changes are detected, it updates the Envoy Gateway configuration
 3. Envoy Gateway then updates the Envoy Proxy configuration
 4. The data plane (Envoy Proxy) processes AI traffic based on this configuration


Test patch:
diff --git a/filterapi/filterconfig_test.go b/filterapi/filterconfig_test.go
index d0cd64a..1f71285 100644
--- a/filterapi/filterconfig_test.go
+++ b/filterapi/filterconfig_test.go
@@ -12,7 +12,6 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
-	"k8s.io/apimachinery/pkg/util/yaml"
 
 	"github.com/envoyproxy/ai-gateway/filterapi"
 	"github.com/envoyproxy/ai-gateway/internal/extproc"
@@ -23,11 +22,15 @@ func TestDefaultConfig(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, server)
 
-	var cfg filterapi.Config
-	err = yaml.Unmarshal([]byte(filterapi.DefaultConfig), &cfg)
-	require.NoError(t, err)
+	cfg, raw := filterapi.MustLoadDefaultConfig()
+	require.Equal(t, []byte(filterapi.DefaultConfig), raw)
+	require.Equal(t, &filterapi.Config{
+		Schema:                   filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI},
+		SelectedBackendHeaderKey: "x-ai-eg-selected-backend",
+		ModelNameHeaderKey:       "x-ai-eg-model",
+	}, cfg)
 
-	err = server.LoadConfig(t.Context(), &cfg)
+	err = server.LoadConfig(t.Context(), cfg)
 	require.NoError(t, err)
 }
 
@@ -71,8 +74,9 @@ rules:
     value: gpt4.4444
 `
 	require.NoError(t, os.WriteFile(configPath, []byte(config), 0o600))
-	cfg, err := filterapi.UnmarshalConfigYaml(configPath)
+	cfg, raw, err := filterapi.UnmarshalConfigYaml(configPath)
 	require.NoError(t, err)
+	require.Equal(t, []byte(config), raw)
 	require.Equal(t, "ai_gateway_llm_ns", cfg.MetadataNamespace)
 	require.Equal(t, "token_usage_key", cfg.LLMRequestCosts[0].MetadataKey)
 	require.Equal(t, "OutputToken", string(cfg.LLMRequestCosts[0].Type))
@@ -92,13 +96,14 @@ rules:
 	require.Equal(t, "us-east-1", cfg.Rules[0].Backends[1].Auth.AWSAuth.Region)
 
 	t.Run("not found", func(t *testing.T) {
-		_, err := filterapi.UnmarshalConfigYaml("not-found.yaml")
+		_, _, err := filterapi.UnmarshalConfigYaml("not-found.yaml")
 		require.Error(t, err)
+		require.True(t, os.IsNotExist(err))
 	})
 	t.Run("invalid", func(t *testing.T) {
 		const invalidConfig = `{wefaf3q20,9u,f02`
 		require.NoError(t, os.WriteFile(configPath, []byte(invalidConfig), 0o600))
-		_, err := filterapi.UnmarshalConfigYaml(configPath)
+		_, _, err := filterapi.UnmarshalConfigYaml(configPath)
 		require.Error(t, err)
 	})
 }
diff --git a/internal/extproc/server_test.go b/internal/extproc/server_test.go
index 55d6aa2..c98e6cb 100644
--- a/internal/extproc/server_test.go
+++ b/internal/extproc/server_test.go
@@ -308,22 +308,26 @@ func TestServer_ProcessorSelection(t *testing.T) {
 	})
 }
 
-func TestFilterSensitiveHeaders(t *testing.T) {
-	logger, buf := newTestLoggerWithBuffer()
-	hm := &corev3.HeaderMap{Headers: []*corev3.HeaderValue{{Key: "foo", Value: "bar"}, {Key: "authorization", Value: "sensitive"}}}
-	filtered := filterSensitiveHeaders(hm, logger, []string{"authorization"})
-	require.Len(t, filtered.Headers, 2)
-	for _, h := range filtered.Headers {
-		if h.Key == "authorization" {
-			require.Equal(t, "[REDACTED]", h.Value)
-		} else {
-			require.Equal(t, "bar", h.Value)
-		}
+func Test_filterSensitiveHeadersForLogging(t *testing.T) {
+	hm := &corev3.HeaderMap{
+		Headers: []*corev3.HeaderValue{
+			{Key: "foo", Value: "bar"}, {Key: "dog", RawValue: []byte("cat")}, {Key: "authorization", Value: "sensitive"},
+		},
 	}
-	require.Contains(t, buf.String(), "filtering sensitive header")
+	filtered := filterSensitiveHeadersForLogging(hm, []string{"authorization"})
+	require.Equal(t, []slog.Attr{
+		slog.String("foo", "bar"),
+		slog.String("dog", "cat"),
+		slog.String("authorization", "[REDACTED]"),
+	}, filtered)
+	// Check original one should not be modified.
+	require.Len(t, hm.Headers, 3)
+	require.Contains(t, hm.Headers, &corev3.HeaderValue{Key: "foo", Value: "bar"})
+	require.Contains(t, hm.Headers, &corev3.HeaderValue{Key: "dog", RawValue: []byte("cat")})
+	require.Contains(t, hm.Headers, &corev3.HeaderValue{Key: "authorization", Value: "sensitive"})
 }
 
-func TestFilterSensitiveBody(t *testing.T) {
+func Test_filterSensitiveBodyForLogging(t *testing.T) {
 	logger, buf := newTestLoggerWithBuffer()
 	resp := &extprocv3.ProcessingResponse{
 		Response: &extprocv3.ProcessingResponse_RequestBody{
@@ -332,27 +336,36 @@ func TestFilterSensitiveBody(t *testing.T) {
 					HeaderMutation: &extprocv3.HeaderMutation{
 						SetHeaders: []*corev3.HeaderValueOption{
 							{Header: &corev3.HeaderValue{
-								Key:   ":path",
-								Value: "/model/some-random-model/converse",
+								Key:      ":path",
+								RawValue: []byte("/model/some-random-model/converse"),
 							}},
 							{Header: &corev3.HeaderValue{
-								Key:   "Authorization",
-								Value: "sensitive",
+								Key:      "Authorization",
+								RawValue: []byte("sensitive"),
 							}},
 						},
+						RemoveHeaders: []string{"x-envoy-original-path"},
 					},
 					BodyMutation: &extprocv3.BodyMutation{},
 				},
 			},
 		},
 	}
-	filtered := filterSensitiveBody(resp, logger, []string{"authorization"})
+	filtered := filterSensitiveBodyForLogging(resp, logger, []string{"authorization"})
 	require.NotNil(t, filtered)
-	for _, h := range filtered.Response.(*extprocv3.ProcessingResponse_RequestBody).RequestBody.Response.GetHeaderMutation().GetSetHeaders() {
-		if h.Header.Key == "Authorization" {
-			require.Equal(t, "[REDACTED]", string(h.Header.RawValue))
-		}
-	}
+	filteredMutation := filtered.Response.(*extprocv3.ProcessingResponse_RequestBody).RequestBody.Response.GetHeaderMutation()
+	require.Equal(t, []string{"x-envoy-original-path"}, filteredMutation.GetRemoveHeaders())
+	require.Equal(t, []*corev3.HeaderValueOption{
+		{Header: &corev3.HeaderValue{Key: ":path", RawValue: []byte("/model/some-random-model/converse")}},
+		{Header: &corev3.HeaderValue{Key: "Authorization", RawValue: []byte("[REDACTED]")}},
+	}, filteredMutation.GetSetHeaders())
+	// Original one should not be modified, otherwise it will be an unexpected behavior.
+	originalMutation := resp.Response.(*extprocv3.ProcessingResponse_RequestBody).RequestBody.Response.GetHeaderMutation()
+	require.Equal(t, []string{"x-envoy-original-path"}, originalMutation.GetRemoveHeaders())
+	require.Equal(t, []*corev3.HeaderValueOption{
+		{Header: &corev3.HeaderValue{Key: ":path", RawValue: []byte("/model/some-random-model/converse")}},
+		{Header: &corev3.HeaderValue{Key: "Authorization", RawValue: []byte("sensitive")}},
+	}, originalMutation.GetSetHeaders())
 	require.Contains(t, buf.String(), "filtering sensitive header")
 }
 
diff --git a/internal/extproc/watcher_test.go b/internal/extproc/watcher_test.go
index 0a7d742..27f9191 100644
--- a/internal/extproc/watcher_test.go
+++ b/internal/extproc/watcher_test.go
@@ -8,6 +8,7 @@ package extproc
 import (
 	"bytes"
 	"context"
+	"io"
 	"log/slog"
 	"os"
 	"strings"
@@ -15,6 +16,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/envoyproxy/ai-gateway/filterapi"
@@ -40,9 +42,30 @@ func (m *mockReceiver) getConfig() *filterapi.Config {
 	return m.cfg
 }
 
+var _ io.Writer = (*syncBuffer)(nil)
+
+// syncBuffer is a bytes.Buffer that is safe for concurrent read/write access.
+// used just in the tests to safely read the logs in assertions without data races.
+type syncBuffer struct {
+	mu sync.RWMutex
+	b  *bytes.Buffer
+}
+
+func (s *syncBuffer) Write(p []byte) (n int, err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.b.Write(p)
+}
+
+func (s *syncBuffer) String() string {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.b.String()
+}
+
 // newTestLoggerWithBuffer creates a new logger with a buffer for testing and asserting the output.
-func newTestLoggerWithBuffer() (*slog.Logger, *bytes.Buffer) {
-	buf := &bytes.Buffer{}
+func newTestLoggerWithBuffer() (*slog.Logger, *syncBuffer) {
+	buf := &syncBuffer{b: &bytes.Buffer{}}
 	logger := slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{
 		Level: slog.LevelDebug,
 	}))
@@ -54,7 +77,22 @@ func TestStartConfigWatcher(t *testing.T) {
 	path := tmpdir + "/config.yaml"
 	rcv := &mockReceiver{}
 
-	require.NoError(t, os.WriteFile(path, []byte{}, 0o600))
+	logger, buf := newTestLoggerWithBuffer()
+	err := StartConfigWatcher(t.Context(), path, rcv, logger, time.Millisecond*100)
+	require.NoError(t, err)
+
+	defaultCfg, _ := filterapi.MustLoadDefaultConfig()
+	require.NoError(t, err)
+
+	// Verify the default config has been loaded.
+	require.EventuallyWithT(t, func(c *assert.CollectT) {
+		assert.Equal(c, defaultCfg, rcv.getConfig())
+	}, 1*time.Second, 100*time.Millisecond)
+
+	// Verify the buffer contains the default config loading.
+	require.Eventually(t, func() bool {
+		return strings.Contains(buf.String(), "config file does not exist; loading default config")
+	}, 1*time.Second, 100*time.Millisecond, buf.String())
 
 	// Create the initial config file.
 	cfg := `
@@ -84,15 +122,10 @@ rules:
     value: gpt4.4444
 `
 	require.NoError(t, os.WriteFile(path, []byte(cfg), 0o600))
-	ctx, cancel := context.WithCancel(t.Context())
-	defer cancel()
-	logger, buf := newTestLoggerWithBuffer()
-	err := StartConfigWatcher(ctx, path, rcv, logger, time.Millisecond*100)
-	require.NoError(t, err)
 
 	// Initial loading should have happened.
 	require.Eventually(t, func() bool {
-		return rcv.getConfig() != nil
+		return rcv.getConfig() != defaultCfg
 	}, 1*time.Second, 100*time.Millisecond)
 	firstCfg := rcv.getConfig()
 	require.NotNil(t, firstCfg)