Repository: kubernetes-sigs/gateway-api-inference-extension

Issue:
Conformance: refactor Conformance Test to Remove `InferenceModel` Dependency
This issue is a follow-up to #1001, which tracks the effort to decouple the core EPP logic from the InferenceModel resource. Once that refactoring is complete, the conformance tests should be updated to take advantage of the new, more modular architecture.

The GatewayFollowingEPPRouting conformance test, introduced in PR [#961](https://github.com/kubernetes-sigs/gateway-api-inference-extension/pull/961), currently requires the creation of an InferenceModel resource to run ([ source](https://github.com/kubernetes-sigs/gateway-api-inference-extension/pull/961?content_ref=ou+re+right+about+the+test+s+intent+the+need+for+the+inferencemodel+is+an+implementation+detail+of+the+testingepp#discussion_r2151072149)). This is an implementation detail of the testing EPP, which needs the resource to discover the full list of potential backend pods and identify the model it is serving

As discussed in the PR, the ideal state for this test is to not depend on the InferenceModel at all.

Hints from issue comments:
@zetxqx: We cannot safely removed the dependency on InferenceObejctive/InferenceModel because the change here: https://github.com/kubernetes-sigs/gateway-api-inference-extension/blob/main/pkg/epp/requestcontrol/director.go#L169-L172@nirrozenbaum: > We cannot safely removed the dependency on InferenceObejctive/InferenceModel because the change here: https://github.com/kubernetes-sigs/gateway-api-inference-extension/blob/main/pkg/epp/requestcontrol/director.go#L169-L172

@zetxqx can you explain the probelm?
if no objective is specified a default priority with zero value is used and then saturation check is bypassed.@zetxqx: @nirrozenbaum sorry, I meant  "we *can now* safely remove" instead of "we *cannot*". Believe it or not, I found myself doing some `phonetic spelling` while writing fast. I'll do more proof reading before posting things.

Solution patch:
diff --git a/conformance/resources/manifests/manifests.yaml b/conformance/resources/manifests/manifests.yaml
index a62275f..5b1f2d4 100644
--- a/conformance/resources/manifests/manifests.yaml
+++ b/conformance/resources/manifests/manifests.yaml
@@ -110,6 +110,7 @@ metadata:
   labels:
     app: secondary-inference-model-server
 spec:
+  replicas: 3
   selector:
     matchLabels:
       app: secondary-inference-model-server
@@ -245,6 +246,7 @@ spec:
   targetPortNumber: 3000
   extensionRef:
     name: secondary-endpoint-picker-svc
+    failureMode: FailOpen
 ---
 # --- Secondary Conformance EPP service Definition ---
 apiVersion: v1
diff --git a/conformance/utils/kubernetes/helpers.go b/conformance/utils/kubernetes/helpers.go
index 8b917c5..2fcaa0c 100644
--- a/conformance/utils/kubernetes/helpers.go
+++ b/conformance/utils/kubernetes/helpers.go
@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -361,7 +362,6 @@ func GetPod(t *testing.T, c client.Client, namespace string, selector labels.Sel
 		}
 		return false, nil
 	})
-
 	require.NoErrorf(t, waitErr, "timed out waiting for Pod with selector %s in namespace %s to be ready", selector.String(), namespace)
 	require.NotEmpty(t, pods.Items, "expected at least one pod for selector %s in namespace %s, but found none", selector.String(), namespace)
 
@@ -369,3 +369,43 @@ func GetPod(t *testing.T, c client.Client, namespace string, selector labels.Sel
 	t.Logf("Successfully found ready Pod %s with IP %s for selector %s", pod.Name, pod.Status.PodIP, selector.String())
 	return pod
 }
+
+// DeleteDeployment deletes the specified Deployment and waits until it is no longer
+// present in the cluster.
+func DeleteDeployment(t *testing.T, c client.Client, timeoutConfig gatewayapiconfig.TimeoutConfig, deploymentRef types.NamespacedName) error {
+	t.Helper()
+
+	deploymentToDelete := &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      deploymentRef.Name,
+			Namespace: deploymentRef.Namespace,
+		},
+	}
+
+	t.Logf("Deleting Deployment %s/%s...", deploymentRef.Namespace, deploymentRef.Name)
+	if err := c.Delete(context.Background(), deploymentToDelete); err != nil {
+		// If the resource is already gone, we don't consider it an error.
+		if !apierrors.IsNotFound(err) {
+			return fmt.Errorf("failed to delete Deployment %s/%s: %w", deploymentRef.Namespace, deploymentRef.Name, err)
+		}
+	}
+
+	// Wait for the Deployment to be fully removed.
+	waitErr := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, timeoutConfig.DeleteTimeout, true, func(ctx context.Context) (bool, error) {
+		var dep appsv1.Deployment
+		err := c.Get(ctx, deploymentRef, &dep)
+		if apierrors.IsNotFound(err) {
+			return true, nil
+		}
+		if err != nil {
+			return false, fmt.Errorf("error waiting for Deployment %s/%s to be deleted: %w", deploymentRef.Namespace, deploymentRef.Name, err)
+		}
+		return false, nil
+	})
+
+	if waitErr != nil {
+		return fmt.Errorf("timed out waiting for Deployment %s/%s to be deleted: %w", deploymentRef.Namespace, deploymentRef.Name, waitErr)
+	}
+	t.Logf("Successfully deleted Deployment %s/%s", deploymentRef.Namespace, deploymentRef.Name)
+	return nil
+}



Test patch:
diff --git a/conformance/tests/basic/epp_unavailable_fail_open.go b/conformance/tests/basic/epp_unavailable_fail_open.go
new file mode 100644
index 0000000..2f68606
--- /dev/null
+++ b/conformance/tests/basic/epp_unavailable_fail_open.go
@@ -0,0 +1,117 @@
+/*
+Copyright 2025 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package basic
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/gateway-api/conformance/utils/suite"
+	"sigs.k8s.io/gateway-api/pkg/features"
+
+	"sigs.k8s.io/gateway-api-inference-extension/conformance/tests"
+	k8sutils "sigs.k8s.io/gateway-api-inference-extension/conformance/utils/kubernetes"
+	trafficutils "sigs.k8s.io/gateway-api-inference-extension/conformance/utils/traffic"
+)
+
+func init() {
+	tests.ConformanceTests = append(tests.ConformanceTests, EppUnAvailableFailOpen)
+}
+
+var EppUnAvailableFailOpen = suite.ConformanceTest{
+	ShortName:   "EppUnAvailableFailOpen",
+	Description: "Inference gateway should send traffic to backends even when the EPP is unavailable (fail-open)",
+	Manifests:   []string{"tests/basic/epp_unavailable_fail_open.yaml"},
+	Features: []features.FeatureName{
+		features.FeatureName("SupportInferencePool"),
+		features.SupportGateway,
+	},
+	Test: func(t *testing.T, s *suite.ConformanceTestSuite) {
+		const (
+			appBackendNamespace    = "gateway-conformance-app-backend"
+			infraNamespace         = "gateway-conformance-infra"
+			hostname               = "secondary.example.com"
+			path                   = "/failopen-pool-test"
+			expectedPodReplicas    = 3
+			eppSelectionHeaderName = "test-epp-endpoint-selection"
+			appPodBackendPrefix    = "secondary-inference-model-server"
+			requestBody            = `{
+                "model": "conformance-fake-model",
+                "prompt": "Write as if you were a critic: San Francisco"
+            }`
+		)
+
+		httpRouteNN := types.NamespacedName{Name: "httproute-for-failopen-pool-gw", Namespace: appBackendNamespace}
+		gatewayNN := types.NamespacedName{Name: "conformance-secondary-gateway", Namespace: infraNamespace}
+		poolNN := types.NamespacedName{Name: "secondary-inference-pool", Namespace: appBackendNamespace}
+		eppDeploymentNN := types.NamespacedName{Name: "secondary-app-endpoint-picker", Namespace: appBackendNamespace}
+		backendPodLabels := map[string]string{"app": "secondary-inference-model-server"}
+
+		k8sutils.HTTPRouteMustBeAcceptedAndResolved(t, s.Client, s.TimeoutConfig, httpRouteNN, gatewayNN)
+		k8sutils.InferencePoolMustBeAcceptedByParent(t, s.Client, poolNN)
+		gwAddr := k8sutils.GetGatewayEndpoint(t, s.Client, s.TimeoutConfig, gatewayNN)
+
+		pods, err := k8sutils.GetPodsWithLabel(t, s.Client, appBackendNamespace, backendPodLabels)
+		require.NoError(t, err, "Failed to get backend pods")
+		require.Len(t, pods, expectedPodReplicas, "Expected to find %d backend pod, but found %d.", expectedPodReplicas, len(pods))
+
+		targetPodIP := pods[0].Status.PodIP
+		t.Run("Phase 1: Verify baseline connectivity with EPP available", func(t *testing.T) {
+			t.Log("Sending request to ensure the Gateway and EPP are working correctly...")
+			trafficutils.MakeRequestWithRequestParamAndExpectSuccess(
+				t,
+				s.RoundTripper,
+				s.TimeoutConfig,
+				gwAddr,
+				trafficutils.Request{
+					Host:      hostname,
+					Path:      path,
+					Headers:   map[string]string{eppSelectionHeaderName: targetPodIP},
+					Method:    http.MethodPost,
+					Body:      requestBody,
+					Backend:   pods[0].Name, // Make sure the request is from the targetPod when the EPP is alive.
+					Namespace: appBackendNamespace,
+				},
+			)
+		})
+
+		t.Run("Phase 2: Verify fail-open behavior after EPP becomes unavailable", func(t *testing.T) {
+			t.Log("Simulating an EPP failure by deleting its deployment...")
+			deleteErr := k8sutils.DeleteDeployment(t, s.Client, s.TimeoutConfig, eppDeploymentNN)
+			require.NoError(t, deleteErr, "Failed to delete the EPP deployment")
+
+			t.Log("Sending request again, expecting success to verify fail-open...")
+			trafficutils.MakeRequestWithRequestParamAndExpectSuccess(
+				t,
+				s.RoundTripper,
+				s.TimeoutConfig,
+				gwAddr,
+				trafficutils.Request{
+					Host:      hostname,
+					Path:      path,
+					Headers:   map[string]string{eppSelectionHeaderName: targetPodIP},
+					Method:    http.MethodPost,
+					Body:      requestBody,
+					Backend:   appPodBackendPrefix, // Only checks the prefix since the EPP is not alive and the response can return from any Pod.
+					Namespace: appBackendNamespace,
+				},
+			)
+		})
+	},
+}
diff --git a/conformance/tests/basic/epp_unavailable_fail_open.yaml b/conformance/tests/basic/epp_unavailable_fail_open.yaml
new file mode 100644
index 0000000..cd681cb
--- /dev/null
+++ b/conformance/tests/basic/epp_unavailable_fail_open.yaml
@@ -0,0 +1,36 @@
+# --- InferenceModel Definition ---
+# TODO: remove inferenceModel dependency https://github.com/kubernetes-sigs/gateway-api-inference-extension/issues/1002
+apiVersion: inference.networking.x-k8s.io/v1alpha2
+kind: InferenceModel
+metadata:
+  name: conformance-fake-model-server
+  namespace: gateway-conformance-app-backend
+spec:
+  modelName: conformance-fake-model
+  criticality: Critical # Mark it as critical to bypass the saturation check since the model server is fake and don't have such metrics. 
+  poolRef:
+    name: secondary-inference-pool
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: httproute-for-failopen-pool-gw
+  namespace: gateway-conformance-app-backend
+spec:
+  parentRefs:
+  - group: gateway.networking.k8s.io
+    kind: Gateway
+    name: conformance-secondary-gateway
+    namespace: gateway-conformance-infra
+    sectionName: http
+  hostnames:
+  - "secondary.example.com"
+  rules:
+  - backendRefs:
+    - group: inference.networking.x-k8s.io
+      kind: InferencePool
+      name: secondary-inference-pool # Use secondary-inferencePool because it has failureMode set to failOpen
+    matches:
+    - path:
+        type: PathPrefix
+        value: /failopen-pool-test
diff --git a/conformance/tests/basic/httproute_multiple_gateways_different_pools.go b/conformance/tests/basic/httproute_multiple_gateways_different_pools.go
new file mode 100644
index 0000000..251f55c
--- /dev/null
+++ b/conformance/tests/basic/httproute_multiple_gateways_different_pools.go
@@ -0,0 +1,118 @@
+/*
+Copyright 2025 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package basic
+
+import (
+	"net/http"
+	"testing"
+
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/gateway-api/conformance/utils/suite"
+
+	"sigs.k8s.io/gateway-api-inference-extension/conformance/tests"
+	k8sutils "sigs.k8s.io/gateway-api-inference-extension/conformance/utils/kubernetes"
+	"sigs.k8s.io/gateway-api-inference-extension/conformance/utils/traffic"
+)
+
+func init() {
+	tests.ConformanceTests = append(tests.ConformanceTests, HTTPRouteMultipleGatewaysDifferentPools)
+}
+
+var HTTPRouteMultipleGatewaysDifferentPools = suite.ConformanceTest{
+	ShortName:   "HTTPRouteMultipleGatewaysDifferentPools",
+	Description: "Validates two HTTPRoutes on different Gateways successfully referencing different InferencePools and routes traffic accordingly.",
+	Manifests:   []string{"tests/basic/httproute_multiple_gateways_different_pools.yaml"},
+	Test: func(t *testing.T, s *suite.ConformanceTestSuite) {
+		const (
+			appBackendNamespace = "gateway-conformance-app-backend"
+			infraNamespace      = "gateway-conformance-infra"
+
+			primaryGatewayName    = "conformance-primary-gateway"
+			routeForPrimaryGWName = "route-for-primary-gateway"
+			primaryPoolName       = "primary-inference-pool"
+			primaryBackendPodName = "primary-inference-model-server"
+			primaryRoutePath      = "/test-primary-gateway"
+			primaryRouteHostname  = "primary.example.com"
+
+			secondaryGatewayName    = "conformance-secondary-gateway"
+			routeForSecondaryGWName = "route-for-secondary-gateway"
+			secondaryPoolName       = "secondary-inference-pool"
+			secondaryBackendPodName = "secondary-inference-model-server"
+			secondaryRoutePath      = "/test-secondary-gateway"
+			secondaryRouteHostname  = "secondary.example.com"
+		)
+
+		routeForPrimaryGWNN := types.NamespacedName{Name: routeForPrimaryGWName, Namespace: appBackendNamespace}
+		routeForSecondaryGWNN := types.NamespacedName{Name: routeForSecondaryGWName, Namespace: appBackendNamespace}
+		primaryPoolNN := types.NamespacedName{Name: primaryPoolName, Namespace: appBackendNamespace}
+		secondaryPoolNN := types.NamespacedName{Name: secondaryPoolName, Namespace: appBackendNamespace}
+		primaryGatewayNN := types.NamespacedName{Name: primaryGatewayName, Namespace: infraNamespace}
+		secondaryGatewayNN := types.NamespacedName{Name: secondaryGatewayName, Namespace: infraNamespace}
+
+		t.Run("Primary HTTPRoute, InferencePool, and Gateway path: verify status and traffic", func(t *testing.T) {
+			k8sutils.HTTPRouteAndInferencePoolMustBeAcceptedAndRouteAccepted(
+				t,
+				s.Client,
+				routeForPrimaryGWNN,
+				primaryGatewayNN,
+				primaryPoolNN,
+			)
+
+			primaryGwAddr := k8sutils.GetGatewayEndpoint(t, s.Client, s.TimeoutConfig, primaryGatewayNN)
+
+			traffic.MakeRequestAndExpectEventuallyConsistentResponse(
+				t,
+				s.RoundTripper,
+				s.TimeoutConfig,
+				primaryGwAddr,
+				traffic.Request{
+					Host:               primaryRouteHostname,
+					Path:               primaryRoutePath,
+					ExpectedStatusCode: http.StatusOK,
+					Backend:            primaryBackendPodName,
+					Namespace:          appBackendNamespace,
+				},
+			)
+		})
+
+		t.Run("Secondary HTTPRoute, InferencePool, and Gateway path: verify status and traffic", func(t *testing.T) {
+			k8sutils.HTTPRouteAndInferencePoolMustBeAcceptedAndRouteAccepted(
+				t,
+				s.Client,
+				routeForSecondaryGWNN,
+				secondaryGatewayNN,
+				secondaryPoolNN,
+			)
+
+			secondaryGwAddr := k8sutils.GetGatewayEndpoint(t, s.Client, s.TimeoutConfig, secondaryGatewayNN)
+
+			traffic.MakeRequestAndExpectEventuallyConsistentResponse(
+				t,
+				s.RoundTripper,
+				s.TimeoutConfig,
+				secondaryGwAddr,
+				traffic.Request{
+					Host:               secondaryRouteHostname,
+					Path:               secondaryRoutePath,
+					ExpectedStatusCode: http.StatusOK,
+					Backend:            secondaryBackendPodName,
+					Namespace:          appBackendNamespace,
+				},
+			)
+		})
+	},
+}
diff --git a/conformance/tests/basic/httproute_multiple_gateways_different_pools.yaml b/conformance/tests/basic/httproute_multiple_gateways_different_pools.yaml
new file mode 100644
index 0000000..b8ec2d1
--- /dev/null
+++ b/conformance/tests/basic/httproute_multiple_gateways_different_pools.yaml
@@ -0,0 +1,44 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: route-for-primary-gateway
+  namespace: gateway-conformance-app-backend
+spec:
+  parentRefs:
+  - kind: Gateway
+    name: conformance-primary-gateway
+    namespace: gateway-conformance-infra
+  hostnames:
+  - "primary.example.com"
+  rules:
+  - backendRefs:
+    - group: inference.networking.x-k8s.io
+      kind: InferencePool
+      name: primary-inference-pool
+    matches:
+    - path:
+        type: PathPrefix
+        value: /test-primary-gateway
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: route-for-secondary-gateway
+  namespace: gateway-conformance-app-backend
+spec:
+  parentRefs:
+  - kind: Gateway
+    name: conformance-secondary-gateway
+    namespace: gateway-conformance-infra
+  hostnames:
+  - "secondary.example.com"
+  rules:
+  - backendRefs:
+    - group: inference.networking.x-k8s.io
+      kind: InferencePool
+      name: secondary-inference-pool
+    matches:
+    - path:
+        type: PathPrefix
+        value: /test-secondary-gateway