Repository: envoyproxy/gateway

Issue:
Connection issues with mirror service
*Description*:

We are trying to use the gateway to mirror traffic to another service, but are seeing a lot of connection issues due to the gateway using stale IPs as the pods change. Seems like the gateway isn't picking up updates to pod IPs. Our workaround is using `routingType: Service`. As suggested in [this issue](https://github.com/envoyproxy/gateway/issues/5326), updating our mirror service to `maxSurge: 1` and `maxUnavailable:0` does not work 

*Repro steps*:

1. Deploy gateway via helm chart v1.3.1
2. Create an http route with mirror to a second service
3. Send traffic to the gateway and mirror
4. In metrics, see many `Connection timeouts`. Or if you have cilium and hubble, 
```
cilium hubble port-forward
hubble observe --verdict DROPPED --follow
```

and see: 
`(world) Stale or unroutable IP DROPPED (TCP Flags: SYN)`



*Environment*:
v1.3.1 of the helm chart with default settings deployed to EKS 

*Logs*:
Related slack thread: https://envoyproxy.slack.com/archives/C03E6NHLESV/p1741887680582939

This is what we are seeing in the gateway pod logs: `envoy-gateway-c984bb466-sl4w7 envoy-gateway 2025-03-13T21:02:33.616Z	INFO	provider.search-gateway.nutrition-prod	kubernetes/status_updater.go:109	status unchanged, bypassing update	{"runner": "provider"}`

Discussion:
@arkodg: thanks for flagging this
the indexer needs to be enhanced to also account for the Backend in the Mirror Filter and not just in the BackendRef
https://github.com/envoyproxy/gateway/blob/e8500749cd41d4e4f968ba7044c577c29ccf95a1/internal/provider/kubernetes/indexers.go#L109@markwinter: @arkodg So just a loop to check for the RequestMirror filter and then add it to the backendRefs output right?@arkodg: yah @markwinter and a test in https://github.com/envoyproxy/gateway/blob/main/internal/provider/kubernetes/predicates_test.go to make sure that triggering a endpointSlice change related to a mirror service, triggers a reconcile

Solution patch:
diff --git a/internal/cmd/server.go b/internal/cmd/server.go
index f2b395b65..80808a420 100644
--- a/internal/cmd/server.go
+++ b/internal/cmd/server.go
@@ -28,6 +28,11 @@ import (
 	xdstranslatorrunner "github.com/envoyproxy/gateway/internal/xds/translator/runner"
 )
 
+type Runner interface {
+	Start(context.Context) error
+	Name() string
+}
+
 // cfgPath is the path to the EnvoyGateway configuration file.
 var cfgPath string
 
@@ -127,6 +132,18 @@ func getConfigByPath(logOut io.Writer, cfgPath string) (*config.Server, error) {
 // setupRunners starts all the runners required for the Envoy Gateway to
 // fulfill its tasks.
 func setupRunners(ctx context.Context, cfg *config.Server) (err error) {
+	channels := struct {
+		pResources *message.ProviderResources
+		xdsIR      *message.XdsIR
+		infraIR    *message.InfraIR
+		xds        *message.Xds
+	}{
+		pResources: new(message.ProviderResources),
+		xdsIR:      new(message.XdsIR),
+		infraIR:    new(message.InfraIR),
+		xds:        new(message.Xds),
+	}
+
 	// The Elected channel is used to block the tasks that are waiting for the leader to be elected.
 	// It will be closed once the leader is elected in the controller manager.
 	cfg.Elected = make(chan struct{})
@@ -134,99 +151,101 @@ func setupRunners(ctx context.Context, cfg *config.Server) (err error) {
 	// Setup the Extension Manager
 	var extMgr types.Manager
 	if cfg.EnvoyGateway.Provider.Type == egv1a1.ProviderTypeKubernetes {
-		extMgr, err = extensionregistry.NewManager(cfg)
-		if err != nil {
+		if extMgr, err = extensionregistry.NewManager(cfg); err != nil {
 			return err
 		}
 	}
 
-	pResources := new(message.ProviderResources)
-	// Start the Provider Service
-	// It fetches the resources from the configured provider type
-	// and publishes it.
-	// It also subscribes to status resources and once it receives
-	// a status resource back, it writes it out.
-	providerRunner := providerrunner.New(&providerrunner.Config{
-		Server:            *cfg,
-		ProviderResources: pResources,
-	})
-	if err = providerRunner.Start(ctx); err != nil {
-		return err
-	}
-
-	xdsIR := new(message.XdsIR)
-	infraIR := new(message.InfraIR)
-	// Start the GatewayAPI Translator Runner
-	// It subscribes to the provider resources, translates it to xDS IR
-	// and infra IR resources and publishes them.
-	gwRunner := gatewayapirunner.New(&gatewayapirunner.Config{
-		Server:            *cfg,
-		ProviderResources: pResources,
-		XdsIR:             xdsIR,
-		InfraIR:           infraIR,
-		ExtensionManager:  extMgr,
-	})
-	if err = gwRunner.Start(ctx); err != nil {
-		return err
-	}
-
-	xds := new(message.Xds)
-	// Start the Xds Translator Service
-	// It subscribes to the xdsIR, translates it into xds Resources and publishes it.
-	// It also computes the EnvoyPatchPolicy statuses and publishes it.
-	xdsTranslatorRunner := xdstranslatorrunner.New(&xdstranslatorrunner.Config{
-		Server:            *cfg,
-		XdsIR:             xdsIR,
-		Xds:               xds,
-		ExtensionManager:  extMgr,
-		ProviderResources: pResources,
-	})
-	if err = xdsTranslatorRunner.Start(ctx); err != nil {
-		return err
-	}
-
-	// Start the Infra Manager Runner
-	// It subscribes to the infraIR, translates it into Envoy Proxy infrastructure
-	// resources such as K8s deployment and services.
-	infraRunner := infrarunner.New(&infrarunner.Config{
-		Server:  *cfg,
-		InfraIR: infraIR,
-	})
-	if err = infraRunner.Start(ctx); err != nil {
-		return err
+	runners := []struct {
+		runner Runner
+	}{
+		{
+			// Start the Provider Service
+			// It fetches the resources from the configured provider type
+			// and publishes it.
+			// It also subscribes to status resources and once it receives
+			// a status resource back, it writes it out.
+			runner: providerrunner.New(&providerrunner.Config{
+				Server:            *cfg,
+				ProviderResources: channels.pResources,
+			}),
+		},
+		{
+			// Start the GatewayAPI Translator Runner
+			// It subscribes to the provider resources, translates it to xDS IR
+			// and infra IR resources and publishes them.
+			runner: gatewayapirunner.New(&gatewayapirunner.Config{
+				Server:            *cfg,
+				ProviderResources: channels.pResources,
+				XdsIR:             channels.xdsIR,
+				InfraIR:           channels.infraIR,
+				ExtensionManager:  extMgr,
+			}),
+		},
+		{
+			// Start the Xds Translator Service
+			// It subscribes to the xdsIR, translates it into xds Resources and publishes it.
+			// It also computes the EnvoyPatchPolicy statuses and publishes it.
+			runner: xdstranslatorrunner.New(&xdstranslatorrunner.Config{
+				Server:            *cfg,
+				XdsIR:             channels.xdsIR,
+				Xds:               channels.xds,
+				ExtensionManager:  extMgr,
+				ProviderResources: channels.pResources,
+			}),
+		},
+		{
+			// Start the Infra Manager Runner
+			// It subscribes to the infraIR, translates it into Envoy Proxy infrastructure
+			// resources such as K8s deployment and services.
+			runner: infrarunner.New(&infrarunner.Config{
+				Server:  *cfg,
+				InfraIR: channels.infraIR,
+			}),
+		},
+		{
+			// Start the xDS Server
+			// It subscribes to the xds Resources and configures the remote Envoy Proxy
+			// via the xDS Protocol.
+			runner: xdsserverrunner.New(&xdsserverrunner.Config{
+				Server: *cfg,
+				Xds:    channels.xds,
+			}),
+		},
 	}
 
-	// Start the xDS Server
-	// It subscribes to the xds Resources and configures the remote Envoy Proxy
-	// via the xDS Protocol.
-	xdsServerRunner := xdsserverrunner.New(&xdsserverrunner.Config{
-		Server: *cfg,
-		Xds:    xds,
-	})
-	if err = xdsServerRunner.Start(ctx); err != nil {
-		return err
+	// Start all runners
+	for _, r := range runners {
+		if err = startRunner(ctx, cfg, r.runner); err != nil {
+			return err
+		}
 	}
-
 	// Start the global rateLimit if it has been enabled through the config
 	if cfg.EnvoyGateway.RateLimit != nil {
 		// Start the Global RateLimit xDS Server
 		// It subscribes to the xds Resources and translates it to Envoy Ratelimit configuration.
 		rateLimitRunner := ratelimitrunner.New(&ratelimitrunner.Config{
 			Server: *cfg,
-			XdsIR:  xdsIR,
+			XdsIR:  channels.xdsIR,
 		})
-		if err = rateLimitRunner.Start(ctx); err != nil {
+		if err = startRunner(ctx, cfg, rateLimitRunner); err != nil {
 			return err
 		}
 	}
 
 	// Wait until done
 	<-ctx.Done()
+
 	// Close messages
-	pResources.Close()
-	xdsIR.Close()
-	infraIR.Close()
-	xds.Close()
+	closeChannels := []interface{ Close() }{
+		channels.pResources,
+		channels.xdsIR,
+		channels.infraIR,
+		channels.xds,
+	}
+	for _, ch := range closeChannels {
+		ch.Close()
+	}
 
 	cfg.Logger.Info("runners are shutting down")
 
@@ -239,3 +258,12 @@ func setupRunners(ctx context.Context, cfg *config.Server) (err error) {
 
 	return nil
 }
+
+func startRunner(ctx context.Context, cfg *config.Server, runner Runner) error {
+	cfg.Logger.Info("Starting runner", "name", runner.Name())
+	if err := runner.Start(ctx); err != nil {
+		cfg.Logger.Error(err, "Failed to start runner", "name", runner.Name())
+		return err
+	}
+	return nil
+}
diff --git a/internal/provider/kubernetes/indexers.go b/internal/provider/kubernetes/indexers.go
index ad698ba79..9866f6d73 100644
--- a/internal/provider/kubernetes/indexers.go
+++ b/internal/provider/kubernetes/indexers.go
@@ -120,6 +120,22 @@ func backendHTTPRouteIndexFunc(rawObj client.Object) []string {
 				)
 			}
 		}
+
+		// Check for a RequestMirror filter to also include the backendRef from that filter
+		for _, filter := range rule.Filters {
+			if filter.Type != gwapiv1.HTTPRouteFilterRequestMirror {
+				continue
+			}
+
+			mirrorBackendRef := filter.RequestMirror.BackendRef
+
+			backendRefs = append(backendRefs,
+				types.NamespacedName{
+					Namespace: gatewayapi.NamespaceDerefOr(mirrorBackendRef.Namespace, httproute.Namespace),
+					Name:      string(mirrorBackendRef.Name),
+				}.String(),
+			)
+		}
 	}
 	return backendRefs
 }



Test patch:
diff --git a/internal/provider/kubernetes/predicates_test.go b/internal/provider/kubernetes/predicates_test.go
index 61a564957..5c9dbba1c 100644
--- a/internal/provider/kubernetes/predicates_test.go
+++ b/internal/provider/kubernetes/predicates_test.go
@@ -463,6 +463,52 @@ func TestValidateEndpointSliceForReconcile(t *testing.T) {
 			endpointSlice: test.GetEndpointSlice(types.NamespacedName{Name: "endpointslice"}, "imported-service", true),
 			expect:        true,
 		},
+		{
+			name: "mirrored backend route exists",
+			configs: []client.Object{
+				test.GetGatewayClass("test-gc", egv1a1.GatewayControllerName, nil),
+				sampleGateway,
+				&gwapiv1.HTTPRoute{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "httproute-test",
+					},
+					Spec: gwapiv1.HTTPRouteSpec{
+						CommonRouteSpec: gwapiv1.CommonRouteSpec{
+							ParentRefs: []gwapiv1.ParentReference{
+								{Name: gwapiv1.ObjectName("scheduled-status-test")},
+							},
+						},
+						Rules: []gwapiv1.HTTPRouteRule{
+							{
+								BackendRefs: []gwapiv1.HTTPBackendRef{
+									{
+										BackendRef: gwapiv1.BackendRef{
+											BackendObjectReference: gwapiv1.BackendObjectReference{
+												Name: gwapiv1.ObjectName("service"),
+												Port: ptr.To(gwapiv1.PortNumber(80)),
+											},
+										},
+									},
+								},
+								Filters: []gwapiv1.HTTPRouteFilter{
+									{
+										Type: gwapiv1.HTTPRouteFilterRequestMirror,
+										RequestMirror: &gwapiv1.HTTPRequestMirrorFilter{
+											BackendRef: gwapiv1.BackendObjectReference{
+												Name: "mirror-service",
+												Port: ptr.To(gwapiv1.PortNumber(80)),
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			endpointSlice: test.GetEndpointSlice(types.NamespacedName{Name: "endpointslice"}, "mirror-service", false),
+			expect:        true,
+		},
 	}
 
 	// Create the reconciler.
diff --git a/test/fuzz/README.md b/test/fuzz/README.md
new file mode 100644
index 000000000..ef6addb47
--- /dev/null
+++ b/test/fuzz/README.md
@@ -0,0 +1,35 @@
+# Envoy Gateway Fuzzing
+
+Envoy Gateway fuzzers are run continuously on [OSS-Fuzz](https://google.github.io/oss-fuzz/). Design documents is
+available [here](https://gateway.envoyproxy.io/contributions/design/fuzzing/).
+
+## Local testing
+
+### Run the fuzzers locally
+
+Run following command from the root of the repository.
+
+```bash
+make go.test.fuzz FUZZ_TIME=10s
+```
+
+
+### Run the fuzzers locally with OSS-Fuzz infra
+
+To build the fuzzers using oss-fuzz infra, use the following commands:
+
+```bash
+git clone --depth=1 https://github.com/google/oss-fuzz.git
+
+cd oss-fuzz
+
+python3 infra/helper.py build_image gateway
+
+python3 infra/helper.py build_fuzzers gateway 
+```
+
+To run the fuzzers use `python3 infra/helper.py run_fuzzer gateway <Fuzzer Name>`. For example:
+
+```bash
+python3 infra/helper.py run_fuzzer gateway FuzzGatewayAPIToXDS
+```
\ No newline at end of file