Repository: kubernetes-sigs/kueue

Issue:
Alpha API Cohorts are not guarded by a feature gate
We would like to avoid supporting/shipping any alpha APIs in Kueue. We remove Topology/Cohorts in our deployment of Kueue. TAS is feature gates so this causes no issue.

It looks like Cohorts are expected to exist for the cohort controller. Is it possible to introduce a feature gate for cohort usage? Ideally we would like to not start the Cohort Controller if we don’t want any alpha functionality in Kueue. 

We see errors in the logs if we remove cohort CRD because the controller still gets started in the core set of controllers.


Hints from issue comments:
@mimowo: I see, I'm totally ok to guard the controller by a feature gate. We probably would use a beta feature gate (even though the API is alpha), so that it does not require a new manual step from users. I think we can consider it as a fully backward compatible bugfix then.@tenzen-y: SGTM@kannon92: Sounds good. I have https://github.com/kubernetes-sigs/kueue/pull/4870.



Solution patch:
diff --git a/pkg/controller/core/core.go b/pkg/controller/core/core.go
index 7f737e3a9..833e14f4c 100644
--- a/pkg/controller/core/core.go
+++ b/pkg/controller/core/core.go
@@ -24,6 +24,7 @@ import (
 	configapi "sigs.k8s.io/kueue/apis/config/v1beta1"
 	"sigs.k8s.io/kueue/pkg/cache"
 	"sigs.k8s.io/kueue/pkg/constants"
+	"sigs.k8s.io/kueue/pkg/features"
 	"sigs.k8s.io/kueue/pkg/queue"
 )
 
@@ -52,9 +53,13 @@ func SetupControllers(mgr ctrl.Manager, qManager *queue.Manager, cc *cache.Cache
 		fairSharingEnabled = cfg.FairSharing.Enable
 	}
 
-	cohortRec := NewCohortReconciler(mgr.GetClient(), cc, qManager, CohortReconcilerWithFairSharing(fairSharingEnabled))
-	if err := cohortRec.SetupWithManager(mgr, cfg); err != nil {
-		return "Cohort", err
+	watchers := []ClusterQueueUpdateWatcher{rfRec, acRec}
+	if features.Enabled(features.HierarchialCohorts) {
+		cohortRec := NewCohortReconciler(mgr.GetClient(), cc, qManager, CohortReconcilerWithFairSharing(fairSharingEnabled))
+		if err := cohortRec.SetupWithManager(mgr, cfg); err != nil {
+			return "Cohort", err
+		}
+		watchers = append(watchers, cohortRec)
 	}
 
 	cqRec := NewClusterQueueReconciler(
@@ -65,7 +70,7 @@ func SetupControllers(mgr ctrl.Manager, qManager *queue.Manager, cc *cache.Cache
 		WithReportResourceMetrics(cfg.Metrics.EnableClusterQueueResources),
 		WithQueueVisibilityClusterQueuesMaxCount(queueVisibilityClusterQueuesMaxCount(cfg)),
 		WithFairSharing(fairSharingEnabled),
-		WithWatchers(rfRec, acRec, cohortRec),
+		WithWatchers(watchers...),
 	)
 	if err := mgr.Add(cqRec); err != nil {
 		return "Unable to add ClusterQueue to manager", err
diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go
index 70daec9c3..acd693476 100644
--- a/pkg/features/kube_features.go
+++ b/pkg/features/kube_features.go
@@ -163,6 +163,12 @@ const (
 	//
 	// Enable to set use LeastAlloactedFit algorithm for TAS
 	TASProfileMixed featuregate.Feature = "TASProfileMixed"
+
+	// owner: @kannon92
+	// kep: https://github.com/kubernetes-sigs/kueue/tree/main/keps/79-hierarchical-cohorts
+	//
+	// Enable hierarchical cohorts controller
+	HierarchialCohorts featuregate.Feature = "HierarchialCohorts"
 )
 
 func init() {
@@ -252,6 +258,9 @@ var defaultVersionedFeatureGates = map[featuregate.Feature]featuregate.Versioned
 	TASProfileMixed: {
 		{Version: version.MustParse("0.11"), Default: false, PreRelease: featuregate.Deprecated},
 	},
+	HierarchialCohorts: {
+		{Version: version.MustParse("0.12"), Default: true, PreRelease: featuregate.Beta},
+	},
 }
 
 func SetFeatureGateDuringTest(tb testing.TB, f featuregate.Feature, value bool) {



Test patch:
diff --git a/pkg/util/testingjobs/deployment/wrappers.go b/pkg/util/testingjobs/deployment/wrappers.go
index 293233c4b..3cebbe778 100644
--- a/pkg/util/testingjobs/deployment/wrappers.go
+++ b/pkg/util/testingjobs/deployment/wrappers.go
@@ -160,3 +160,9 @@ func (d *DeploymentWrapper) TerminationGracePeriod(seconds int64) *DeploymentWra
 	d.Spec.Template.Spec.TerminationGracePeriodSeconds = &seconds
 	return d
 }
+
+func (d *DeploymentWrapper) SetTypeMeta() *DeploymentWrapper {
+	d.APIVersion = appsv1.SchemeGroupVersion.String()
+	d.Kind = "Deployment"
+	return d
+}
diff --git a/test/e2e/customconfigs/skipjobswithoutqueuename_test.go b/test/e2e/customconfigs/skipjobswithoutqueuename_test.go
index 253df3b43..7be08b5c1 100644
--- a/test/e2e/customconfigs/skipjobswithoutqueuename_test.go
+++ b/test/e2e/customconfigs/skipjobswithoutqueuename_test.go
@@ -30,6 +30,8 @@ import (
 	config "sigs.k8s.io/kueue/apis/config/v1beta1"
 	kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1"
 	"sigs.k8s.io/kueue/pkg/constants"
+	controllerconstants "sigs.k8s.io/kueue/pkg/controller/constants"
+	"sigs.k8s.io/kueue/pkg/controller/jobs/appwrapper"
 	workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job"
 	podcontroller "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants"
 	"sigs.k8s.io/kueue/pkg/util/testing"
@@ -100,7 +102,7 @@ var _ = ginkgo.Describe("ManageJobsWithoutQueueName", ginkgo.Ordered, func() {
 				jobLookupKey = types.NamespacedName{Name: testJob.Name, Namespace: ns.Name}
 				gomega.Eventually(func(g gomega.Gomega) {
 					g.Expect(k8sClient.Get(ctx, jobLookupKey, createdJob)).Should(gomega.Succeed())
-					createdJob.Labels["kueue.x-k8s.io/queue-name"] = "main"
+					createdJob.Labels[controllerconstants.QueueLabel] = localQueue.Name
 					g.Expect(k8sClient.Update(ctx, createdJob)).Should(gomega.Succeed())
 				}, util.Timeout, util.Interval).Should(gomega.Succeed())
 			})
@@ -152,7 +154,7 @@ var _ = ginkgo.Describe("ManageJobsWithoutQueueName", ginkgo.Ordered, func() {
 				awLookupKey := types.NamespacedName{Name: aw.Name, Namespace: ns.Name}
 				gomega.Eventually(func(g gomega.Gomega) {
 					g.Expect(k8sClient.Get(ctx, awLookupKey, createdAppWrapper)).Should(gomega.Succeed())
-					createdAppWrapper.Labels["kueue.x-k8s.io/queue-name"] = "main"
+					createdAppWrapper.Labels[controllerconstants.QueueLabel] = localQueue.Name
 					g.Expect(k8sClient.Update(ctx, createdAppWrapper)).Should(gomega.Succeed())
 				}, util.Timeout, util.Interval).Should(gomega.Succeed())
 			})
@@ -213,7 +215,7 @@ var _ = ginkgo.Describe("ManageJobsWithoutQueueName", ginkgo.Ordered, func() {
 				awLookupKey := types.NamespacedName{Name: aw.Name, Namespace: ns.Name}
 				gomega.Eventually(func(g gomega.Gomega) {
 					g.Expect(k8sClient.Get(ctx, awLookupKey, createdAppWrapper)).Should(gomega.Succeed())
-					createdAppWrapper.Labels["kueue.x-k8s.io/queue-name"] = "main"
+					createdAppWrapper.Labels[controllerconstants.QueueLabel] = localQueue.Name
 					g.Expect(k8sClient.Update(ctx, createdAppWrapper)).Should(gomega.Succeed())
 				}, util.Timeout, util.Interval).Should(gomega.Succeed())
 			})
@@ -302,7 +304,7 @@ var _ = ginkgo.Describe("ManageJobsWithoutQueueName", ginkgo.Ordered, func() {
 			ginkgo.By("verifying that the pods of the deployment are gated", func() {
 				gomega.Eventually(func(g gomega.Gomega) {
 					pods := &corev1.PodList{}
-					g.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Namespace),
+					g.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Name),
 						client.MatchingLabels(testDeploy.Spec.Selector.MatchLabels))).To(gomega.Succeed())
 					g.Expect(pods.Items).Should(gomega.HaveLen(2))
 					for _, pod := range pods.Items {
@@ -330,7 +332,7 @@ var _ = ginkgo.Describe("ManageJobsWithoutQueueName", ginkgo.Ordered, func() {
 			ginkgo.By("verifying that the pods of the deployment are running", func() {
 				gomega.Eventually(func(g gomega.Gomega) {
 					pods := &corev1.PodList{}
-					g.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Namespace),
+					g.Expect(k8sClient.List(ctx, pods, client.InNamespace("kube-system"),
 						client.MatchingLabels(testDeploy.Spec.Selector.MatchLabels))).To(gomega.Succeed())
 					g.Expect(pods.Items).Should(gomega.HaveLen(2))
 					for _, pod := range pods.Items {
@@ -343,7 +345,7 @@ var _ = ginkgo.Describe("ManageJobsWithoutQueueName", ginkgo.Ordered, func() {
 				pods := &corev1.PodList{}
 				gomega.Expect(k8sClient.Delete(ctx, testDeploy)).Should(gomega.Succeed())
 				gomega.Eventually(func(g gomega.Gomega) {
-					g.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Namespace),
+					g.Expect(k8sClient.List(ctx, pods, client.InNamespace("kube-system"),
 						client.MatchingLabels(testDeploy.Spec.Selector.MatchLabels))).To(gomega.Succeed())
 					g.Expect(pods.Items).Should(gomega.BeEmpty())
 				}, util.LongTimeout, util.Interval).Should(gomega.Succeed())
@@ -365,7 +367,7 @@ var _ = ginkgo.Describe("ManageJobsWithoutQueueName", ginkgo.Ordered, func() {
 			ginkgo.By("verifying that the pods of the StatefulSet are gated", func() {
 				gomega.Eventually(func(g gomega.Gomega) {
 					pods := &corev1.PodList{}
-					g.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Namespace),
+					g.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Name),
 						client.MatchingLabels(testSts.Spec.Selector.MatchLabels))).To(gomega.Succeed())
 					// If the first pod can't be scheduled, the second won't be created
 					g.Expect(pods.Items).Should(gomega.HaveLen(1))
@@ -393,7 +395,7 @@ var _ = ginkgo.Describe("ManageJobsWithoutQueueName", ginkgo.Ordered, func() {
 			ginkgo.By("verifying that the pods of the StatefulSet are running", func() {
 				gomega.Eventually(func(g gomega.Gomega) {
 					pods := &corev1.PodList{}
-					g.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Namespace),
+					g.Expect(k8sClient.List(ctx, pods, client.InNamespace("kube-system"),
 						client.MatchingLabels(testSts.Spec.Selector.MatchLabels))).To(gomega.Succeed())
 					g.Expect(pods.Items).Should(gomega.HaveLen(2))
 					for _, pod := range pods.Items {
@@ -406,11 +408,87 @@ var _ = ginkgo.Describe("ManageJobsWithoutQueueName", ginkgo.Ordered, func() {
 				pods := &corev1.PodList{}
 				gomega.Expect(k8sClient.Delete(ctx, testSts)).Should(gomega.Succeed())
 				gomega.Eventually(func(g gomega.Gomega) {
-					g.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Namespace),
+					g.Expect(k8sClient.List(ctx, pods, client.InNamespace("kube-system"),
 						client.MatchingLabels(testSts.Spec.Selector.MatchLabels))).To(gomega.Succeed())
 					g.Expect(pods.Items).Should(gomega.BeEmpty())
 				}, util.LongTimeout, util.Interval).Should(gomega.Succeed())
 			})
 		})
+
+		ginkgo.It("Should suspend Pods of a created Deployment in the test namespace", func() {
+			deploymentKey := types.NamespacedName{Name: "deployment", Namespace: ns.Name}
+			replicas := int32(3)
+
+			aw := awtesting.MakeAppWrapper("aw", ns.Name).
+				Suspend(false).
+				Component(awtesting.Component{
+					Template: testingdeploy.MakeDeployment(deploymentKey.Name, deploymentKey.Namespace).
+						Image(util.E2eTestAgnHostImage, util.BehaviorWaitForDeletion).
+						RequestAndLimit(corev1.ResourceCPU, "200m").
+						TerminationGracePeriod(1).
+						Replicas(3).
+						SetTypeMeta().
+						Obj(),
+					DeclaredPodSets: []awv1beta2.AppWrapperPodSet{{
+						Replicas: &replicas,
+						Path:     "template.spec.template",
+					}},
+				}).
+				Obj()
+
+			ginkgo.By("creating an AppWrapper", func() {
+				gomega.Expect(k8sClient.Create(ctx, aw)).To(gomega.Succeed())
+			})
+
+			ginkgo.By("verifying that the AppWrapper gets suspended", func() {
+				createdAppWrapper := &awv1beta2.AppWrapper{}
+				gomega.Eventually(func(g gomega.Gomega) {
+					g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(aw), createdAppWrapper)).To(gomega.Succeed())
+					g.Expect(createdAppWrapper.Spec.Suspend).To(gomega.BeTrue())
+				}, util.Timeout, util.Interval).Should(gomega.Succeed())
+			})
+
+			ginkgo.By("verifying that the Deployment doesn't create", func() {
+				createdDeployment := &appsv1.Deployment{}
+				gomega.Consistently(func(g gomega.Gomega) {
+					g.Expect(k8sClient.Get(ctx, deploymentKey, createdDeployment)).To(testing.BeNotFoundError())
+				}, util.ConsistentDuration, util.Interval).Should(gomega.Succeed())
+			})
+
+			ginkgo.By("setting the queue-name label", func() {
+				createdAppWrapper := &awv1beta2.AppWrapper{}
+				gomega.Eventually(func(g gomega.Gomega) {
+					g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(aw), createdAppWrapper)).Should(gomega.Succeed())
+					createdAppWrapper.Labels[controllerconstants.QueueLabel] = localQueue.Name
+					g.Expect(k8sClient.Update(ctx, createdAppWrapper)).Should(gomega.Succeed())
+				}, util.Timeout, util.Interval).Should(gomega.Succeed())
+			})
+
+			ginkgo.By("wait for AppWrapper to be unsuspended", func() {
+				createdAppWrapper := &awv1beta2.AppWrapper{}
+				gomega.Eventually(func(g gomega.Gomega) {
+					g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(aw), createdAppWrapper)).To(gomega.Succeed())
+					g.Expect(createdAppWrapper.Spec.Suspend).To(gomega.BeFalse())
+				}, util.Timeout, util.Interval).Should(gomega.Succeed())
+			})
+
+			ginkgo.By("Check that only one workload is created and admitted", func() {
+				createdWorkloads := &kueue.WorkloadList{}
+				gomega.Eventually(func(g gomega.Gomega) {
+					g.Expect(k8sClient.List(ctx, createdWorkloads, client.InNamespace(aw.Namespace))).To(gomega.Succeed())
+					g.Expect(createdWorkloads.Items).To(gomega.HaveLen(1))
+					g.Expect(createdWorkloads.Items[0].Name).To(gomega.Equal(appwrapper.GetWorkloadNameForAppWrapper(aw.Name, aw.UID)))
+					g.Expect(createdWorkloads.Items[0].Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted))
+				}, util.Timeout, util.Interval).Should(gomega.Succeed())
+			})
+
+			ginkgo.By("verifying that the Deployment ready", func() {
+				createdDeployment := &appsv1.Deployment{}
+				gomega.Eventually(func(g gomega.Gomega) {
+					g.Expect(k8sClient.Get(ctx, deploymentKey, createdDeployment)).To(gomega.Succeed())
+					g.Expect(createdDeployment.Status.ReadyReplicas).To(gomega.Equal(int32(3)))
+				}, util.LongTimeout, util.Interval).Should(gomega.Succeed())
+			})
+		})
 	})
 })
diff --git a/test/e2e/singlecluster/appwrapper_test.go b/test/e2e/singlecluster/appwrapper_test.go
index 14a287fe1..36caeb3b1 100644
--- a/test/e2e/singlecluster/appwrapper_test.go
+++ b/test/e2e/singlecluster/appwrapper_test.go
@@ -20,12 +20,16 @@ import (
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
 	awv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1"
+	"sigs.k8s.io/kueue/pkg/controller/jobs/appwrapper"
 	"sigs.k8s.io/kueue/pkg/util/testing"
 	awtesting "sigs.k8s.io/kueue/pkg/util/testingjobs/appwrapper"
+	testingdeploy "sigs.k8s.io/kueue/pkg/util/testingjobs/deployment"
 	utiltestingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job"
 	"sigs.k8s.io/kueue/test/util"
 )
@@ -74,7 +78,7 @@ var _ = ginkgo.Describe("AppWrapper", func() {
 		util.ExpectAllPodsInNamespaceDeleted(ctx, k8sClient, ns)
 	})
 
-	ginkgo.It("Should admit workloads that fit", func() {
+	ginkgo.It("Should admit Workload for Job", func() {
 		numPods := 2
 		aw := awtesting.MakeAppWrapper("appwrapper", ns.Name).
 			Component(awtesting.Component{
@@ -113,4 +117,56 @@ var _ = ginkgo.Describe("AppWrapper", func() {
 			util.ExpectObjectToBeDeleted(ctx, k8sClient, aw, true)
 		})
 	})
+
+	ginkgo.It("Should admit Workload for Deployment Pods", func() {
+		deploymentKey := types.NamespacedName{Name: "deployment", Namespace: ns.Name}
+		replicas := int32(3)
+		aw := awtesting.MakeAppWrapper("aw", ns.Name).
+			Queue(lq.Name).
+			Suspend(true).
+			Component(awtesting.Component{
+				Template: testingdeploy.MakeDeployment(deploymentKey.Name, deploymentKey.Namespace).
+					Image(util.E2eTestAgnHostImage, util.BehaviorWaitForDeletion).
+					RequestAndLimit(corev1.ResourceCPU, "200m").
+					TerminationGracePeriod(1).
+					Replicas(3).
+					SetTypeMeta().
+					Obj(),
+				DeclaredPodSets: []awv1beta2.AppWrapperPodSet{{
+					Replicas: &replicas,
+					Path:     "template.spec.template",
+				}},
+			}).
+			Obj()
+
+		ginkgo.By("Creating an AppWrapper", func() {
+			gomega.Expect(k8sClient.Create(ctx, aw)).To(gomega.Succeed())
+		})
+
+		ginkgo.By("Wait for AppWrapper to be unsuspended", func() {
+			createdAppWrapper := &awv1beta2.AppWrapper{}
+			gomega.Eventually(func(g gomega.Gomega) {
+				g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(aw), createdAppWrapper)).To(gomega.Succeed())
+				g.Expect(createdAppWrapper.Spec.Suspend).To(gomega.BeFalse())
+			}, util.LongTimeout, util.Interval).Should(gomega.Succeed())
+		})
+
+		ginkgo.By("Check that only one workload is created and admitted", func() {
+			createdWorkloads := &kueue.WorkloadList{}
+			gomega.Eventually(func(g gomega.Gomega) {
+				g.Expect(k8sClient.List(ctx, createdWorkloads, client.InNamespace(aw.Namespace))).To(gomega.Succeed())
+				g.Expect(createdWorkloads.Items).To(gomega.HaveLen(1))
+				g.Expect(createdWorkloads.Items[0].Name).To(gomega.Equal(appwrapper.GetWorkloadNameForAppWrapper(aw.Name, aw.UID)))
+				g.Expect(createdWorkloads.Items[0].Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted))
+			}, util.LongTimeout, util.Interval).Should(gomega.Succeed())
+		})
+
+		ginkgo.By("verifying that the Deployment ready", func() {
+			createdDeployment := &appsv1.Deployment{}
+			gomega.Eventually(func(g gomega.Gomega) {
+				g.Expect(k8sClient.Get(ctx, deploymentKey, createdDeployment)).To(gomega.Succeed())
+				g.Expect(createdDeployment.Status.ReadyReplicas).To(gomega.Equal(int32(3)))
+			}, util.LongTimeout, util.Interval).Should(gomega.Succeed())
+		})
+	})
 })
diff --git a/test/e2e/singlecluster/deployment_test.go b/test/e2e/singlecluster/deployment_test.go
index 62d89fa2c..e83bfa2af 100644
--- a/test/e2e/singlecluster/deployment_test.go
+++ b/test/e2e/singlecluster/deployment_test.go
@@ -100,7 +100,7 @@ var _ = ginkgo.Describe("Deployment", func() {
 		})
 
 		pods := &corev1.PodList{}
-		gomega.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Namespace),
+		gomega.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Name),
 			client.MatchingLabels(deployment.Spec.Selector.MatchLabels))).To(gomega.Succeed())
 
 		createdWorkloads := make([]*kueue.Workload, 0, len(pods.Items))
@@ -152,7 +152,7 @@ var _ = ginkgo.Describe("Deployment", func() {
 		})
 
 		pods := &corev1.PodList{}
-		gomega.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Namespace),
+		gomega.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Name),
 			client.MatchingLabels(deployment.Spec.Selector.MatchLabels))).To(gomega.Succeed())
 
 		createdWorkloads := make([]*kueue.Workload, 0, len(pods.Items))
@@ -189,14 +189,14 @@ var _ = ginkgo.Describe("Deployment", func() {
 
 		ginkgo.By("Check previous pods are deleted", func() {
 			gomega.Eventually(func(g gomega.Gomega) {
-				g.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Namespace),
+				g.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Name),
 					client.MatchingLabels(deployment.Spec.Selector.MatchLabels))).To(gomega.Succeed())
 				g.Expect(pods.Items).To(gomega.HaveLen(3))
 			}, util.LongTimeout, util.Interval).Should(gomega.Succeed())
 		})
 
 		ginkgo.By("Check that workloads are created and admitted", func() {
-			gomega.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Namespace),
+			gomega.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Name),
 				client.MatchingLabels(deployment.Spec.Selector.MatchLabels))).To(gomega.Succeed())
 			for _, p := range pods.Items {
 				createdWorkload := &kueue.Workload{}