Repository: kubernetes-sigs/gateway-api-inference-extension

Issue:
Use Dedicated Namespace for Resources
The POC readme and e2e tests use the "default" namespace. To follow Kubernetes' best practices and preserve the "default" namespace, a dedicated namespace, i.e. `inf-ext-e2e`, should be used throughout.

Discussion:
@kfswain: I think it's reasonable for a getting started guide to utilize the default namespace, I think its acceptable to expect the user to know how to deploy in a specific namespace should they desire, and it lets the getting started manifests be that much cleaner. Open to alt opinions however, LMKWYT!@nirrozenbaum: I tend to agree with @danehans. 
almost every CNCF project I worked with has a dedicated ns for it’s resources.@kfswain: Which resources would we tuck into a specific namespace? The current expectation is that the user deploys the EPP themselves. We don't really have other deployables atm


For the Getting Started i think its fine to leave default and not make opinionated decisions for a guide of that format.@danehans: This issue was partially resolved when the EPP Helm chart was introduced. However, e2e is still bound to namespace `default` due to the Envoy ConfigMap containing namespace-specific values, e.g. `authority: vllm-llama3-8b-instruct-epp.default:9002`. As an initial solution, we should change the e2e namespace (`default`) to something specific for e2e testing, update the Envoy ConfigMap accordingly, and have the e2e suite create the e2e namespace.

Solution patch:
diff --git a/Makefile b/Makefile
index 66fe89d4..a1845560 100644
--- a/Makefile
+++ b/Makefile
@@ -232,7 +232,7 @@ bbr-image-local-load: bbr-image-local-build
 
 .PHONY: bbr-image-build
 bbr-image-build: ## Build the image using Docker Buildx.
-	$(IMAGE_BUILD_CMD) -f body-based-routing.Dockerfile -t $(BBR_IMAGE_TAG) \
+	$(IMAGE_BUILD_CMD) -f bbr.Dockerfile -t $(BBR_IMAGE_TAG) \
 		--platform=$(PLATFORMS) \
 		--build-arg BASE_IMAGE=$(BASE_IMAGE) \
 		--build-arg BUILDER_IMAGE=$(BUILDER_IMAGE) \
diff --git a/body-based-routing.Dockerfile b/bbr.Dockerfile
similarity index 76%
rename from body-based-routing.Dockerfile
rename to bbr.Dockerfile
index e0afcf20..03024e49 100644
--- a/body-based-routing.Dockerfile
+++ b/bbr.Dockerfile
@@ -18,13 +18,13 @@ RUN go mod download
 COPY cmd ./cmd
 COPY pkg ./pkg
 COPY internal ./internal
-WORKDIR /src/cmd/body-based-routing
-RUN go build -o /body-based-routing
+WORKDIR /src/cmd/bbr
+RUN go build -o /bbr
 
 ## Multistage deploy
 FROM ${BASE_IMAGE}
 
 WORKDIR /
-COPY --from=builder /body-based-routing /body-based-routing
+COPY --from=builder /bbr /bbr
 
-ENTRYPOINT ["/body-based-routing"]
+ENTRYPOINT ["/bbr"]
diff --git a/config/manifests/inferencepool-resources.yaml b/config/manifests/inferencepool-resources.yaml
index cef70d7f..4affa274 100644
--- a/config/manifests/inferencepool-resources.yaml
+++ b/config/manifests/inferencepool-resources.yaml
@@ -1,3 +1,6 @@
+# Note: If you change this file, please also change the file used for e2e tests!
+# 
+# https://github.com/kubernetes-sigs/gateway-api-inference-extension/blob/main/test/testdata/inferencepool-e2e.yaml
 apiVersion: inference.networking.x-k8s.io/v1alpha2
 kind: InferencePool
 metadata:
diff --git a/config/manifests/vllm/cpu-deployment.yaml b/config/manifests/vllm/cpu-deployment.yaml
index 6fb40950..827f2156 100644
--- a/config/manifests/vllm/cpu-deployment.yaml
+++ b/config/manifests/vllm/cpu-deployment.yaml
@@ -113,5 +113,8 @@ data:
         ensureExist:
           models:
           - base-model: Qwen/Qwen2.5-1.5B
-            id: food-review-1
+            id: food-review
+            source: SriSanth2345/Qwen-1.5B-Tweet-Generations
+          - base-model: Qwen/Qwen2.5-1.5B
+            id: cad-fabricator
             source: SriSanth2345/Qwen-1.5B-Tweet-Generations
\ No newline at end of file
diff --git a/pkg/epp/controller/pod_reconciler.go b/pkg/epp/controller/pod_reconciler.go
index 046561e4..494adeb7 100644
--- a/pkg/epp/controller/pod_reconciler.go
+++ b/pkg/epp/controller/pod_reconciler.go
@@ -30,6 +30,7 @@ import (
 	"sigs.k8s.io/gateway-api-inference-extension/api/v1alpha2"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/datastore"
 	logutil "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging"
+	podutil "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/pod"
 )
 
 type PodReconciler struct {
@@ -71,7 +72,7 @@ func (c *PodReconciler) SetupWithManager(mgr ctrl.Manager) error {
 
 func (c *PodReconciler) updateDatastore(logger logr.Logger, pod *corev1.Pod, pool *v1alpha2.InferencePool) {
 	namespacedName := types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}
-	if !pod.DeletionTimestamp.IsZero() || !c.Datastore.PoolLabelsMatch(pod.Labels) || !podIsReady(pod) {
+	if !pod.DeletionTimestamp.IsZero() || !c.Datastore.PoolLabelsMatch(pod.Labels) || !podutil.IsPodReady(pod) {
 		logger.V(logutil.DEBUG).Info("Pod removed or not added", "name", namespacedName)
 		c.Datastore.PodDelete(namespacedName)
 	} else {
@@ -82,15 +83,3 @@ func (c *PodReconciler) updateDatastore(logger logr.Logger, pod *corev1.Pod, poo
 		}
 	}
 }
-
-func podIsReady(pod *corev1.Pod) bool {
-	for _, condition := range pod.Status.Conditions {
-		if condition.Type == corev1.PodReady {
-			if condition.Status == corev1.ConditionTrue {
-				return true
-			}
-			break
-		}
-	}
-	return false
-}
diff --git a/pkg/epp/datastore/datastore.go b/pkg/epp/datastore/datastore.go
index 8ada3e64..dc81cb48 100644
--- a/pkg/epp/datastore/datastore.go
+++ b/pkg/epp/datastore/datastore.go
@@ -30,6 +30,7 @@ import (
 	"sigs.k8s.io/gateway-api-inference-extension/api/v1alpha2"
 	backendmetrics "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/backend/metrics"
 	logutil "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging"
+	podutil "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/pod"
 )
 
 const (
@@ -259,7 +260,7 @@ func (ds *datastore) PodResyncAll(ctx context.Context, ctrlClient client.Client,
 
 	activePods := make(map[string]bool)
 	for _, pod := range podList.Items {
-		if podIsReady(&pod) {
+		if podutil.IsPodReady(&pod) {
 			namespacedName := types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}
 			activePods[pod.Name] = true
 			if ds.PodUpdateOrAddIfNotExist(&pod, pool) {
@@ -308,16 +309,3 @@ func IsCritical(model *v1alpha2.InferenceModel) bool {
 	}
 	return false
 }
-
-// TODO: move out to share with pod_reconciler.go
-func podIsReady(pod *corev1.Pod) bool {
-	for _, condition := range pod.Status.Conditions {
-		if condition.Type == corev1.PodReady {
-			if condition.Status == corev1.ConditionTrue {
-				return true
-			}
-			break
-		}
-	}
-	return false
-}
diff --git a/pkg/epp/util/pod/pod.go b/pkg/epp/util/pod/pod.go
new file mode 100644
index 00000000..9f564024
--- /dev/null
+++ b/pkg/epp/util/pod/pod.go
@@ -0,0 +1,33 @@
+/*
+Copyright 2025 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pod
+
+import (
+	corev1 "k8s.io/api/core/v1"
+)
+
+func IsPodReady(pod *corev1.Pod) bool {
+	for _, condition := range pod.Status.Conditions {
+		if condition.Type == corev1.PodReady {
+			if condition.Status == corev1.ConditionTrue {
+				return true
+			}
+			break
+		}
+	}
+	return false
+}



Test patch:
diff --git a/test/e2e/epp/README.md b/test/e2e/epp/README.md
index 247e8b12..fcc974b8 100644
--- a/test/e2e/epp/README.md
+++ b/test/e2e/epp/README.md
@@ -28,6 +28,13 @@ Follow these steps to run the end-to-end tests:
    export HF_TOKEN=<MY_HF_TOKEN>
    ```
 
+1. **(Optional): Set the test namespace**: By default, the e2e test creates resources in the `inf-ext-e2e` namespace.
+   If you would like to change this namespace, set the following environment variable:
+
+   ```sh
+   export E2E_NS=<MY_NS>
+   ```
+
 1. **Run the Tests**: Run the `test-e2e` target:
 
    ```sh
diff --git a/test/e2e/epp/e2e_suite_test.go b/test/e2e/epp/e2e_suite_test.go
index 61ee2540..01ed639d 100644
--- a/test/e2e/epp/e2e_suite_test.go
+++ b/test/e2e/epp/e2e_suite_test.go
@@ -30,6 +30,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
@@ -55,9 +56,8 @@ const (
 	defaultInterval = time.Millisecond * 250
 	// defaultCurlInterval is the default interval to run the test curl command.
 	defaultCurlInterval = time.Second * 5
-	// nsName is the name of the Namespace used for tests.
-	// TODO [danehans]: Must be "default" until https://github.com/kubernetes-sigs/gateway-api-inference-extension/issues/227 is fixed
-	nsName = "default"
+	// defaultNsName is the default name of the Namespace used for tests. Can override using the E2E_NS environment variable.
+	defaultNsName = "inf-ext-e2e"
 	// modelServerName is the name of the model server test resources.
 	modelServerName = "vllm-llama3-8b-instruct"
 	// modelName is the test model name.
@@ -77,7 +77,7 @@ const (
 	// inferModelManifest is the manifest for the inference model CRD.
 	inferModelManifest = "../../../config/crd/bases/inference.networking.x-k8s.io_inferencemodels.yaml"
 	// inferExtManifest is the manifest for the inference extension test resources.
-	inferExtManifest = "../../../config/manifests/inferencepool-resources.yaml"
+	inferExtManifest = "../../testdata/inferencepool-e2e.yaml"
 	// envoyManifest is the manifest for the envoy proxy test resources.
 	envoyManifest = "../../testdata/envoy.yaml"
 	// modelServerManifestFilepathEnvVar is the env var that holds absolute path to the manifest for the model server test resource.
@@ -91,6 +91,7 @@ var (
 	kubeCli *kubernetes.Clientset
 	scheme  = runtime.NewScheme()
 	cfg     = config.GetConfigOrDie()
+	nsName  string
 )
 
 func TestAPIs(t *testing.T) {
@@ -101,6 +102,11 @@ func TestAPIs(t *testing.T) {
 }
 
 var _ = ginkgo.BeforeSuite(func() {
+	nsName = os.Getenv("E2E_NS")
+	if nsName == "" {
+		nsName = defaultNsName
+	}
+
 	ginkgo.By("Setting up the test suite")
 	setupSuite()
 
@@ -109,6 +115,8 @@ var _ = ginkgo.BeforeSuite(func() {
 })
 
 func setupInfra() {
+	createNamespace(cli, nsName)
+
 	modelServerManifestPath := readModelServerManifestPath()
 	modelServerManifestArray := getYamlsFromModelServerManifest(modelServerManifestPath)
 	if strings.Contains(modelServerManifestArray[0], "hf-token") {
@@ -118,6 +126,7 @@ func setupInfra() {
 		"inferencepools.inference.networking.x-k8s.io":  inferPoolManifest,
 		"inferencemodels.inference.networking.x-k8s.io": inferModelManifest,
 	}
+
 	createCRDs(cli, crds)
 	createInferExt(cli, inferExtManifest)
 	createClient(cli, clientManifest)
@@ -182,6 +191,17 @@ var (
 	curlInterval      = defaultCurlInterval
 )
 
+func createNamespace(k8sClient client.Client, ns string) {
+	ginkgo.By("Creating e2e namespace: " + ns)
+	obj := &corev1.Namespace{
+		ObjectMeta: v1.ObjectMeta{
+			Name: ns,
+		},
+	}
+	err := k8sClient.Create(ctx, obj)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to create e2e test namespace")
+}
+
 // namespaceExists ensures that a specified namespace exists and is ready for use.
 func namespaceExists(k8sClient client.Client, ns string) {
 	ginkgo.By("Ensuring namespace exists: " + ns)
@@ -276,8 +296,15 @@ func createHfSecret(k8sClient client.Client, secretPath string) {
 
 // createEnvoy creates the envoy proxy resources used for testing from the given filePath.
 func createEnvoy(k8sClient client.Client, filePath string) {
+	inManifests := readYaml(filePath)
+	ginkgo.By("Replacing placeholder namespace with E2E_NS environment variable")
+	outManifests := []string{}
+	for _, m := range inManifests {
+		outManifests = append(outManifests, strings.ReplaceAll(m, "$E2E_NS", nsName))
+	}
+
 	ginkgo.By("Creating envoy proxy resources from manifest: " + filePath)
-	applyYAMLFile(k8sClient, filePath)
+	createObjsFromYaml(k8sClient, outManifests)
 
 	// Wait for the configmap to exist before proceeding with test.
 	cfgMap := &corev1.ConfigMap{}
@@ -302,8 +329,15 @@ func createEnvoy(k8sClient client.Client, filePath string) {
 
 // createInferExt creates the inference extension resources used for testing from the given filePath.
 func createInferExt(k8sClient client.Client, filePath string) {
+	inManifests := readYaml(filePath)
+	ginkgo.By("Replacing placeholder namespace with E2E_NS environment variable")
+	outManifests := []string{}
+	for _, m := range inManifests {
+		outManifests = append(outManifests, strings.ReplaceAll(m, "$E2E_NS", nsName))
+	}
+
 	ginkgo.By("Creating inference extension resources from manifest: " + filePath)
-	applyYAMLFile(k8sClient, filePath)
+	createObjsFromYaml(k8sClient, outManifests)
 
 	// Wait for the clusterrole to exist.
 	testutils.EventuallyExists(ctx, func() error {
diff --git a/test/integration/epp/hermetic_test.go b/test/integration/epp/hermetic_test.go
index 93432637..ae2c6170 100644
--- a/test/integration/epp/hermetic_test.go
+++ b/test/integration/epp/hermetic_test.go
@@ -44,6 +44,7 @@ import (
 	"google.golang.org/protobuf/testing/protocmp"
 	"google.golang.org/protobuf/types/known/structpb"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -1691,27 +1692,13 @@ func BeforeSuite() func() {
 	}
 
 	for _, doc := range docs {
-		inferenceModel := &v1alpha2.InferenceModel{}
-		if err = yaml.Unmarshal(doc, inferenceModel); err != nil {
+		obj := &unstructured.Unstructured{}
+		if err = yaml.Unmarshal(doc, obj); err != nil {
 			logutil.Fatal(logger, err, "Can't unmarshal object", "document", doc)
 		}
-		if inferenceModel.Kind == "InferenceModel" {
-			logger.Info("Creating inference model", "model", inferenceModel)
-			if err := k8sClient.Create(context.Background(), inferenceModel); err != nil {
-				logutil.Fatal(logger, err, "Unable to create inferenceModel", "modelName", inferenceModel.Name)
-			}
-		}
-	}
-	for _, doc := range docs {
-		inferencePool := &v1alpha2.InferencePool{}
-		if err = yaml.Unmarshal(doc, inferencePool); err != nil {
-			logutil.Fatal(logger, err, "Can't unmarshal object", "document", doc)
-		}
-		if inferencePool.Kind == "InferencePool" {
-			logger.Info("Creating inference pool", "pool", inferencePool)
-			if err := k8sClient.Create(context.Background(), inferencePool); err != nil {
-				logutil.Fatal(logger, err, "Unable to create inferencePool", "poolName", inferencePool.Name)
-			}
+		logger.Info("Creating object", "kind", obj.GetKind(), "object", obj)
+		if err := k8sClient.Create(context.Background(), obj); err != nil {
+			logutil.Fatal(logger, err, "Unable to create object", "object", obj.GetName())
 		}
 	}
 
diff --git a/test/testdata/envoy.yaml b/test/testdata/envoy.yaml
index 62e6b4c5..3fff8598 100644
--- a/test/testdata/envoy.yaml
+++ b/test/testdata/envoy.yaml
@@ -100,7 +100,7 @@ data:
                           grpc_service:
                             envoy_grpc:
                               cluster_name: ext_proc
-                              authority: vllm-llama3-8b-instruct-epp.default:9002
+                              authority: vllm-llama3-8b-instruct-epp.$E2E_NS:9002
                             timeout: 10s
                           processing_mode:
                             request_header_mode: SEND
@@ -195,7 +195,7 @@ data:
                   - endpoint:
                       address:
                         socket_address:
-                          address: vllm-llama3-8b-instruct-epp.default
+                          address: vllm-llama3-8b-instruct-epp.$E2E_NS
                           port_value: 9002
                     health_status: HEALTHY
                     load_balancing_weight: 1
@@ -225,7 +225,7 @@ spec:
         image: docker.io/envoyproxy/envoy:distroless-v1.33.2
         args:
           - "--service-cluster" 
-          - "default/inference-gateway"
+          - "$E2E_NS/inference-gateway"
           - "--service-node"
           - "$(ENVOY_POD_NAME)"
           - "--log-level"
diff --git a/test/testdata/inferencepool-e2e.yaml b/test/testdata/inferencepool-e2e.yaml
new file mode 100644
index 00000000..79339c5b
--- /dev/null
+++ b/test/testdata/inferencepool-e2e.yaml
@@ -0,0 +1,126 @@
+apiVersion: inference.networking.x-k8s.io/v1alpha2
+kind: InferencePool
+metadata:
+  labels:
+  name: vllm-llama3-8b-instruct
+spec:
+  targetPortNumber: 8000
+  selector:
+    app: vllm-llama3-8b-instruct
+  extensionRef:
+    name: vllm-llama3-8b-instruct-epp
+    namespace: $E2E_NS
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: vllm-llama3-8b-instruct-epp
+  namespace: $E2E_NS
+spec:
+  selector:
+    app: vllm-llama3-8b-instruct-epp
+  ports:
+    - protocol: TCP
+      port: 9002
+      targetPort: 9002
+      appProtocol: http2
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: vllm-llama3-8b-instruct-epp
+  namespace: $E2E_NS
+  labels:
+    app: vllm-llama3-8b-instruct-epp
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: vllm-llama3-8b-instruct-epp
+  template:
+    metadata:
+      labels:
+        app: vllm-llama3-8b-instruct-epp
+    spec:
+      # Conservatively, this timeout should mirror the longest grace period of the pods within the pool
+      terminationGracePeriodSeconds: 130
+      containers:
+      - name: epp
+        image: us-central1-docker.pkg.dev/k8s-staging-images/gateway-api-inference-extension/epp:main
+        imagePullPolicy: Always
+        args:
+        - -poolName
+        - "vllm-llama3-8b-instruct"
+        - -poolNamespace
+        - "$E2E_NS"
+        - -v
+        - "4"
+        - --zap-encoder
+        - "json"
+        - -grpcPort
+        - "9002"
+        - -grpcHealthPort
+        - "9003"
+        env:
+        - name: USE_STREAMING
+          value: "true"
+        ports:
+        - containerPort: 9002
+        - containerPort: 9003
+        - name: metrics
+          containerPort: 9090
+        livenessProbe:
+          grpc:
+            port: 9003
+            service: inference-extension
+          initialDelaySeconds: 5
+          periodSeconds: 10
+        readinessProbe:
+          grpc:
+            port: 9003
+            service: inference-extension
+          initialDelaySeconds: 5
+          periodSeconds: 10
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: pod-read
+rules:
+- apiGroups: ["inference.networking.x-k8s.io"]
+  resources: ["inferencemodels"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: ["inference.networking.x-k8s.io"]
+  resources: ["inferencepools"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: ["discovery.k8s.io"]
+  resources: ["endpointslices"]
+  verbs: ["get", "watch", "list"]
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create
+--- 
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: pod-read-binding
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: $E2E_NS
+roleRef:
+  kind: ClusterRole
+  name: pod-read