Repository: kubernetes-sigs/kueue (go)

Issue:
Pending workloads not updated in ClusterQueue status at creation
**What happened**:

When workloads are created, we are not updating the corresponding CQ status.


**What you expected to happen**:


The status to be up to date

**How to reproduce it (as minimally and precisely as possible)**:

Repro: https://github.com/alculquicondor/kueue/commit/1ae8618cb687956bcc6fc361ea839490164faa29

**Anything else we need to know?**:

This is because we are only queueing the clusterqueue for admitted workloads in assignedWorkloadHandler https://github.com/kubernetes-sigs/kueue/blob/27b05145f94ac528ff92790efc70d62d8c9feafc/pkg/controller/core/clusterqueue_controller.go#L145

**Environment**:
- Kueue version (use `git describe --tags --dirty --always`): main


Pull request description:
Fix clusterqueue controller to update pending workloads
#### What type of PR is this?

/kind bug

#### What this PR does / why we need it:

To keep a correct track of number of pending workloads in a ClusterQueue

#### Which issue(s) this PR fixes:

Fixes #148

#### Special notes for your reviewer:



Patch:
diff --git a/api/v1alpha1/clusterqueue_types.go b/api/v1alpha1/clusterqueue_types.go
index 1f77ffe9f..18bca462b 100644
--- a/api/v1alpha1/clusterqueue_types.go
+++ b/api/v1alpha1/clusterqueue_types.go
@@ -247,8 +247,8 @@ type Quota struct {
 	// ceiling is the upper limit on the amount of resource requests that
 	// could be used by running workloads assigned to this quota at a point in time.
 	// Resources can be borrowed from unused guaranteed quota of other
-	// ClusterQueues in the same cohort. When null, it is unlimited.
-	Ceiling *resource.Quantity `json:"ceiling,omitempty"`
+	// ClusterQueues in the same cohort. When not set, it is unlimited.
+	Ceiling resource.Quantity `json:"ceiling,omitempty"`
 }
 
 // ClusterQueueStatus defines the observed state of ClusterQueue
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 59fff2bca..054cbe3a0 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -445,11 +445,7 @@ func (in *QueuedWorkloadStatus) DeepCopy() *QueuedWorkloadStatus {
 func (in *Quota) DeepCopyInto(out *Quota) {
 	*out = *in
 	out.Guaranteed = in.Guaranteed.DeepCopy()
-	if in.Ceiling != nil {
-		in, out := &in.Ceiling, &out.Ceiling
-		x := (*in).DeepCopy()
-		*out = &x
-	}
+	out.Ceiling = in.Ceiling.DeepCopy()
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Quota.
diff --git a/config/crd/bases/kueue.x-k8s.io_clusterqueues.yaml b/config/crd/bases/kueue.x-k8s.io_clusterqueues.yaml
index 350857d74..36d4e5923 100644
--- a/config/crd/bases/kueue.x-k8s.io_clusterqueues.yaml
+++ b/config/crd/bases/kueue.x-k8s.io_clusterqueues.yaml
@@ -199,7 +199,7 @@ spec:
                                   workloads assigned to this quota at a point in time.
                                   Resources can be borrowed from unused guaranteed
                                   quota of other ClusterQueues in the same cohort.
-                                  When null, it is unlimited.
+                                  When not set, it is unlimited.
                                 pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                                 x-kubernetes-int-or-string: true
                               guaranteed:
diff --git a/config/samples/single-clusterqueue-setup.yaml b/config/samples/single-clusterqueue-setup.yaml
index 0d0a1b902..203521b11 100644
--- a/config/samples/single-clusterqueue-setup.yaml
+++ b/config/samples/single-clusterqueue-setup.yaml
@@ -15,11 +15,13 @@ spec:
     - resourceFlavor: default
       quota:
         guaranteed: 9
+        ceiling: 9
   - name: "memory"
     flavors:
     - resourceFlavor: default
       quota:
         guaranteed: 36Gi
+        ceiling: 36Gi
 ---
 apiVersion: kueue.x-k8s.io/v1alpha1
 kind: Queue
diff --git a/docs/tasks/administer_cluster_resources.md b/docs/tasks/administer_cluster_resources.md
index 8a9083476..2bb085deb 100644
--- a/docs/tasks/administer_cluster_resources.md
+++ b/docs/tasks/administer_cluster_resources.md
@@ -44,16 +44,21 @@ spec:
     - resourceFlavor: default
       quota:
         guaranteed: 9
+        ceiling: 9
   - name: "memory"
     flavors:
     - resourceFlavor: default
       quota:
         guaranteed: 36Gi
+        ceiling: 36Gi
 ```
 
 This ClusterQueue governs the usage of [resource types](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-types)
 `cpu` and `memory`. Each resource type has a single [resource flavor](/docs/concepts/cluster_queue.md#resource-flavors),
 named `default` with a guaranteed quota.
+As there is a single ClusterQueue in the cluster, it is not possible for the
+ClusterQueue to borrow resources from anywhere. Thus, the `ceiling` is equal
+to the `guaranteed` quota.
 
 The empty `namespaceSelector` allows any namespace to use these resources.
 
diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go
index a5e46e9d4..d8bc033bc 100644
--- a/pkg/cache/cache.go
+++ b/pkg/cache/cache.go
@@ -97,7 +97,7 @@ type ClusterQueue struct {
 type FlavorLimits struct {
 	Name       string
 	Guaranteed int64
-	Ceiling    *int64
+	Ceiling    int64
 }
 
 func (c *Cache) newClusterQueue(cq *kueue.ClusterQueue) (*ClusterQueue, error) {
@@ -138,7 +138,7 @@ func (c *ClusterQueue) update(in *kueue.ClusterQueue, resourceFlavors map[string
 	return nil
 }
 
-// UpdateLabelKeys updates a ClusterQueue's LabelKeys based on the passed ResourceFlavors set.
+// UpdateLabelkeys updates a ClusterQueue's LabelKeys based on the passed ResourceFlavors set.
 // Exported only for testing.
 func (c *ClusterQueue) UpdateLabelKeys(flavors map[string]*kueue.ResourceFlavor) {
 	labelKeys := map[corev1.ResourceName]sets.String{}
@@ -473,9 +473,7 @@ func resourceLimitsByName(in []kueue.Resource) map[corev1.ResourceName][]FlavorL
 			fLimits := FlavorLimits{
 				Name:       string(f.ResourceFlavor),
 				Guaranteed: workload.ResourceValue(r.Name, f.Quota.Guaranteed),
-			}
-			if f.Quota.Ceiling != nil {
-				fLimits.Ceiling = pointer.Int64(workload.ResourceValue(r.Name, *f.Quota.Ceiling))
+				Ceiling:    workload.ResourceValue(r.Name, f.Quota.Ceiling),
 			}
 			flavors[i] = fLimits
 
diff --git a/pkg/controller/core/clusterqueue_controller.go b/pkg/controller/core/clusterqueue_controller.go
index d32221a39..e9806902e 100644
--- a/pkg/controller/core/clusterqueue_controller.go
+++ b/pkg/controller/core/clusterqueue_controller.go
@@ -140,52 +140,71 @@ func (r *ClusterQueue) Generic(e event.GenericEvent) bool {
 	return true
 }
 
-// assignedWorkloadHandler signals the controller to reconcile the ClusterQueue
+// workloadHandler signals the controller to reconcile the ClusterQueue
 // assigned to the workload in the event.
-type assignedWorkloadHandler struct{}
+type workloadHandler struct {
+	qManager *queue.Manager
+}
 
-func (h *assignedWorkloadHandler) Create(e event.CreateEvent, q workqueue.RateLimitingInterface) {
+func (h *workloadHandler) Create(e event.CreateEvent, q workqueue.RateLimitingInterface) {
 	w := e.Object.(*kueue.QueuedWorkload)
-	if w.Spec.Admission != nil {
-		q.AddAfter(requestForWorkloadClusterQueue(w), constants.UpdatesBatchPeriod)
+	req := h.requestForWorkloadClusterQueue(w)
+	if req != nil {
+		q.AddAfter(*req, constants.UpdatesBatchPeriod)
 	}
 }
 
-func (h *assignedWorkloadHandler) Update(e event.UpdateEvent, q workqueue.RateLimitingInterface) {
+func (h *workloadHandler) Update(e event.UpdateEvent, q workqueue.RateLimitingInterface) {
 	oldW := e.ObjectOld.(*kueue.QueuedWorkload)
-	if oldW.Spec.Admission != nil {
-		q.AddAfter(requestForWorkloadClusterQueue(oldW), constants.UpdatesBatchPeriod)
+	oldReq := h.requestForWorkloadClusterQueue(oldW)
+	if oldReq != nil {
+		q.AddAfter(*oldReq, constants.UpdatesBatchPeriod)
 	}
 	newW := e.ObjectNew.(*kueue.QueuedWorkload)
-	if newW.Spec.Admission != nil && (oldW.Spec.Admission == nil || workloadFinished(newW) ||
-		newW.Spec.Admission.ClusterQueue != oldW.Spec.Admission.ClusterQueue) {
-		q.AddAfter(requestForWorkloadClusterQueue(newW), constants.UpdatesBatchPeriod)
+	newReq := h.requestForWorkloadClusterQueue(newW)
+	// Enqueue the CQ for the new object if it's different from the old one.
+	if newReq != nil && (oldReq == nil || oldReq.Name != newReq.Name) {
+		q.AddAfter(*newReq, constants.UpdatesBatchPeriod)
 	}
 }
 
-func (h *assignedWorkloadHandler) Delete(e event.DeleteEvent, q workqueue.RateLimitingInterface) {
+func (h *workloadHandler) Delete(e event.DeleteEvent, q workqueue.RateLimitingInterface) {
 	w := e.Object.(*kueue.QueuedWorkload)
-	if w.Spec.Admission != nil {
-		q.AddAfter(requestForWorkloadClusterQueue(w), constants.UpdatesBatchPeriod)
+	req := h.requestForWorkloadClusterQueue(w)
+	if req != nil {
+		q.AddAfter(*req, constants.UpdatesBatchPeriod)
 	}
 }
 
-func (h *assignedWorkloadHandler) Generic(e event.GenericEvent, q workqueue.RateLimitingInterface) {
+func (h *workloadHandler) Generic(e event.GenericEvent, q workqueue.RateLimitingInterface) {
 }
 
-func requestForWorkloadClusterQueue(w *kueue.QueuedWorkload) reconcile.Request {
-	return reconcile.Request{
+func (h *workloadHandler) requestForWorkloadClusterQueue(w *kueue.QueuedWorkload) *reconcile.Request {
+	var name string
+	if w.Spec.Admission != nil {
+		name = string(w.Spec.Admission.ClusterQueue)
+	} else {
+		var ok bool
+		name, ok = h.qManager.ClusterQueueForWorkload(w)
+		if !ok {
+			return nil
+		}
+	}
+	return &reconcile.Request{
 		NamespacedName: types.NamespacedName{
-			Name: string(w.Spec.Admission.ClusterQueue),
+			Name: name,
 		},
 	}
 }
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *ClusterQueue) SetupWithManager(mgr ctrl.Manager) error {
+	wHandler := workloadHandler{
+		qManager: r.qManager,
+	}
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&kueue.ClusterQueue{}).
-		Watches(&source.Kind{Type: &kueue.QueuedWorkload{}}, &assignedWorkloadHandler{}).
+		Watches(&source.Kind{Type: &kueue.QueuedWorkload{}}, &wHandler).
 		WithEventFilter(r).
 		Complete(r)
 }
diff --git a/pkg/controller/core/queuedworkload_controller.go b/pkg/controller/core/queuedworkload_controller.go
index a5e791551..9274a4d6d 100644
--- a/pkg/controller/core/queuedworkload_controller.go
+++ b/pkg/controller/core/queuedworkload_controller.go
@@ -73,16 +73,15 @@ func (r *QueuedWorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 	log.V(2).Info("Reconciling QueuedWorkload")
 
 	status := workloadStatus(&wl)
-	q, qOK := r.queues.QueueExists(&wl)
-	if status == pending && !qOK {
+	if status == pending && !r.queues.QueueForWorkloadExists(&wl) {
 		err := workload.UpdateWorkloadStatusIfChanged(ctx, r.client, &wl, kueue.QueuedWorkloadAdmitted, corev1.ConditionFalse,
-			"Inadmissible", fmt.Sprintf("Queue %s for workload doesn't exist", wl.Spec.QueueName))
+			"Inadmissible", fmt.Sprintf("Queue %s doesn't exist", wl.Spec.QueueName))
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
-
-	if status == pending && !r.queues.ClusterQueueExists(&wl) {
+	cqName, cqOk := r.queues.ClusterQueueForWorkload(&wl)
+	if status == pending && !cqOk {
 		err := workload.UpdateWorkloadStatusIfChanged(ctx, r.client, &wl, kueue.QueuedWorkloadAdmitted, corev1.ConditionFalse,
-			"Inadmissible", fmt.Sprintf("ClusterQueue %s doesn't exist", q.ClusterQueue))
+			"Inadmissible", fmt.Sprintf("ClusterQueue %s doesn't exist", cqName))
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
diff --git a/pkg/queue/manager.go b/pkg/queue/manager.go
index fd09c81e5..fe45201ca 100644
--- a/pkg/queue/manager.go
+++ b/pkg/queue/manager.go
@@ -206,19 +206,26 @@ func (m *Manager) Pending(cq *kueue.ClusterQueue) int32 {
 	return int32(len(m.clusterQueues[cq.Name].heap.heap))
 }
 
-func (m *Manager) QueueExists(wl *kueue.QueuedWorkload) (*Queue, bool) {
+func (m *Manager) QueueForWorkloadExists(wl *kueue.QueuedWorkload) bool {
 	m.RLock()
 	defer m.RUnlock()
-	q, ok := m.queues[queueKeyForWorkload(wl)]
-	return q, ok
+	_, ok := m.queues[queueKeyForWorkload(wl)]
+	return ok
 
 }
 
-func (m *Manager) ClusterQueueExists(wl *kueue.QueuedWorkload) bool {
+// ClusterQueueForWorkload returns the name of the ClusterQueue where the
+// workload should be queued and whether it exists.
+// Returns empty string if the queue doesn't exist.
+func (m *Manager) ClusterQueueForWorkload(wl *kueue.QueuedWorkload) (string, bool) {
 	m.RLock()
 	defer m.RUnlock()
-	_, ok := m.clusterQueues[m.queues[queueKeyForWorkload(wl)].ClusterQueue]
-	return ok
+	q, ok := m.queues[queueKeyForWorkload(wl)]
+	if !ok {
+		return "", false
+	}
+	_, ok = m.clusterQueues[q.ClusterQueue]
+	return q.ClusterQueue, ok
 }
 
 // AddOrUpdateWorkload adds or updates workload to the corresponding queue.
diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go
index a00f68c49..f544c265b 100644
--- a/pkg/scheduler/scheduler.go
+++ b/pkg/scheduler/scheduler.go
@@ -348,7 +348,7 @@ func flavorSelector(spec *corev1.PodSpec, allowedKeys sets.String) nodeaffinity.
 // If it fits, also returns any borrowing required.
 func fitsFlavorLimits(name corev1.ResourceName, val int64, cq *cache.ClusterQueue, flavor *cache.FlavorLimits) (bool, int64) {
 	used := cq.UsedResources[name][flavor.Name]
-	if flavor.Ceiling != nil && used+val > *flavor.Ceiling {
+	if used+val > flavor.Ceiling {
 		// Past borrowing limit.
 		return false, 0
 	}
diff --git a/pkg/util/pointer/pointer.go b/pkg/util/pointer/pointer.go
index 63d6a0b0c..00f9e8b45 100644
--- a/pkg/util/pointer/pointer.go
+++ b/pkg/util/pointer/pointer.go
@@ -16,17 +16,8 @@ limitations under the License.
 
 package pointer
 
-import (
-	"k8s.io/apimachinery/pkg/api/resource"
-	"k8s.io/utils/pointer"
-)
+import "k8s.io/apimachinery/pkg/api/resource"
 
 func Quantity(q resource.Quantity) *resource.Quantity {
 	return &q
 }
-
-var (
-	Int32 = pointer.Int32
-	Int64 = pointer.Int64
-	Bool  = pointer.Bool
-)


Test patch:
diff --git a/pkg/cache/cache_test.go b/pkg/cache/cache_test.go
index adcfe542d..cdeb94e31 100644
--- a/pkg/cache/cache_test.go
+++ b/pkg/cache/cache_test.go
@@ -45,15 +45,16 @@ func TestCacheClusterQueueOperations(t *testing.T) {
 				RequestableResources: []kueue.Resource{
 					{
 						Name: corev1.ResourceCPU,
-						Flavors: []kueue.Flavor{{
-							ResourceFlavor: "default",
-							Quota: kueue.Quota{
-								Guaranteed: resource.MustParse("10"),
-								Ceiling:    pointer.Quantity(resource.MustParse("20")),
+						Flavors: []kueue.Flavor{
+							{
+								ResourceFlavor: "default",
+								Quota: kueue.Quota{
+									Guaranteed: resource.MustParse("10"),
+									Ceiling:    resource.MustParse("20"),
+								},
 							},
-						}},
-					},
-				},
+						},
+					}},
 				Cohort: "one",
 			},
 		},
@@ -62,15 +63,9 @@ func TestCacheClusterQueueOperations(t *testing.T) {
 			Spec: kueue.ClusterQueueSpec{
 				RequestableResources: []kueue.Resource{
 					{
-						Name: corev1.ResourceCPU,
-						Flavors: []kueue.Flavor{{
-							ResourceFlavor: "default",
-							Quota: kueue.Quota{
-								Guaranteed: resource.MustParse("15"),
-							},
-						}},
-					},
-				},
+						Name:    corev1.ResourceCPU,
+						Flavors: []kueue.Flavor{{ResourceFlavor: "default"}},
+					}},
 				Cohort: "one",
 			},
 		},
@@ -112,7 +107,7 @@ func TestCacheClusterQueueOperations(t *testing.T) {
 				"a": {
 					Name: "a",
 					RequestableResources: map[corev1.ResourceName][]FlavorLimits{
-						corev1.ResourceCPU: {{Name: "default", Guaranteed: 10000, Ceiling: pointer.Int64(20000)}},
+						corev1.ResourceCPU: {{Name: "default", Guaranteed: 10000, Ceiling: 20000}},
 					},
 					NamespaceSelector: labels.Nothing(),
 					LabelKeys:         map[corev1.ResourceName]sets.String{corev1.ResourceCPU: sets.NewString("cpuType")},
@@ -121,7 +116,7 @@ func TestCacheClusterQueueOperations(t *testing.T) {
 				"b": {
 					Name: "b",
 					RequestableResources: map[corev1.ResourceName][]FlavorLimits{
-						corev1.ResourceCPU: {{Name: "default", Guaranteed: 15000}},
+						corev1.ResourceCPU: {{Name: "default"}},
 					},
 					NamespaceSelector: labels.Nothing(),
 					UsedResources:     Resources{corev1.ResourceCPU: {"default": 0}},
@@ -146,7 +141,7 @@ func TestCacheClusterQueueOperations(t *testing.T) {
 			},
 		},
 		{
-			name: "add flavors after queue capacities",
+			name: "add flavors after queue capacties",
 			operation: func(cache *Cache) {
 				for _, c := range initialClusterQueues {
 					if err := cache.AddClusterQueue(context.Background(), &c); err != nil {
@@ -162,7 +157,7 @@ func TestCacheClusterQueueOperations(t *testing.T) {
 				"a": {
 					Name: "a",
 					RequestableResources: map[corev1.ResourceName][]FlavorLimits{
-						corev1.ResourceCPU: {{Name: "default", Guaranteed: 10000, Ceiling: pointer.Int64(20000)}},
+						corev1.ResourceCPU: {{Name: "default", Guaranteed: 10000, Ceiling: 20000}},
 					},
 					NamespaceSelector: labels.Nothing(),
 					LabelKeys:         map[corev1.ResourceName]sets.String{corev1.ResourceCPU: sets.NewString("cpuType")},
@@ -171,7 +166,7 @@ func TestCacheClusterQueueOperations(t *testing.T) {
 				"b": {
 					Name: "b",
 					RequestableResources: map[corev1.ResourceName][]FlavorLimits{
-						corev1.ResourceCPU: {{Name: "default", Guaranteed: 15000}},
+						corev1.ResourceCPU: {{Name: "default"}},
 					},
 					NamespaceSelector: labels.Nothing(),
 					UsedResources:     Resources{corev1.ResourceCPU: {"default": 0}},
@@ -211,7 +206,7 @@ func TestCacheClusterQueueOperations(t *testing.T) {
 											ResourceFlavor: "default",
 											Quota: kueue.Quota{
 												Guaranteed: resource.MustParse("5"),
-												Ceiling:    pointer.Quantity(resource.MustParse("10")),
+												Ceiling:    resource.MustParse("10"),
 											},
 										},
 									},
@@ -242,7 +237,7 @@ func TestCacheClusterQueueOperations(t *testing.T) {
 				"a": {
 					Name: "a",
 					RequestableResources: map[corev1.ResourceName][]FlavorLimits{
-						corev1.ResourceCPU: {{Name: "default", Guaranteed: 5000, Ceiling: pointer.Int64(10000)}},
+						corev1.ResourceCPU: {{Name: "default", Guaranteed: 5000, Ceiling: 10000}},
 					},
 					NamespaceSelector: labels.Nothing(),
 					LabelKeys:         map[corev1.ResourceName]sets.String{corev1.ResourceCPU: sets.NewString("cpuType", "region")},
@@ -288,7 +283,7 @@ func TestCacheClusterQueueOperations(t *testing.T) {
 				"b": {
 					Name: "b",
 					RequestableResources: map[corev1.ResourceName][]FlavorLimits{
-						corev1.ResourceCPU: {{Name: "default", Guaranteed: 15000}},
+						corev1.ResourceCPU: {{Name: "default"}},
 					},
 					NamespaceSelector: labels.Nothing(),
 					UsedResources:     Resources{corev1.ResourceCPU: {"default": 0}},
@@ -804,7 +799,7 @@ func TestClusterQueueUsage(t *testing.T) {
 							ResourceFlavor: "default",
 							Quota: kueue.Quota{
 								Guaranteed: resource.MustParse("10"),
-								Ceiling:    pointer.Quantity(resource.MustParse("20")),
+								Ceiling:    resource.MustParse("20"),
 							},
 						},
 					},
@@ -816,14 +811,14 @@ func TestClusterQueueUsage(t *testing.T) {
 							ResourceFlavor: "model_a",
 							Quota: kueue.Quota{
 								Guaranteed: resource.MustParse("5"),
-								Ceiling:    pointer.Quantity(resource.MustParse("10")),
+								Ceiling:    resource.MustParse("10"),
 							},
 						},
 						{
 							ResourceFlavor: "model_b",
 							Quota: kueue.Quota{
 								Guaranteed: resource.MustParse("5"),
-								// No ceiling.
+								Ceiling:    resource.MustParse("10"),
 							},
 						},
 					},
diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go
index 6f44f0a92..541235707 100644
--- a/pkg/scheduler/scheduler_test.go
+++ b/pkg/scheduler/scheduler_test.go
@@ -33,7 +33,6 @@ import (
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
-	"sigs.k8s.io/kueue/pkg/util/pointer"
 
 	kueue "sigs.k8s.io/kueue/api/v1alpha1"
 	"sigs.k8s.io/kueue/pkg/cache"
@@ -78,7 +77,7 @@ func TestSchedule(t *testing.T) {
 								ResourceFlavor: "default",
 								Quota: kueue.Quota{
 									Guaranteed: resource.MustParse("50"),
-									Ceiling:    pointer.Quantity(resource.MustParse("50")),
+									Ceiling:    resource.MustParse("50"),
 								},
 							},
 						},
@@ -108,14 +107,14 @@ func TestSchedule(t *testing.T) {
 								ResourceFlavor: "on-demand",
 								Quota: kueue.Quota{
 									Guaranteed: resource.MustParse("50"),
-									Ceiling:    pointer.Quantity(resource.MustParse("100")),
+									Ceiling:    resource.MustParse("100"),
 								},
 							},
 							{
 								ResourceFlavor: "spot",
 								Quota: kueue.Quota{
 									Guaranteed: resource.MustParse("100"),
-									Ceiling:    pointer.Quantity(resource.MustParse("100")),
+									Ceiling:    resource.MustParse("100"),
 								},
 							},
 						},
@@ -145,14 +144,14 @@ func TestSchedule(t *testing.T) {
 								ResourceFlavor: "on-demand",
 								Quota: kueue.Quota{
 									Guaranteed: resource.MustParse("50"),
-									Ceiling:    pointer.Quantity(resource.MustParse("60")),
+									Ceiling:    resource.MustParse("60"),
 								},
 							},
 							{
 								ResourceFlavor: "spot",
 								Quota: kueue.Quota{
 									Guaranteed: resource.MustParse("0"),
-									Ceiling:    pointer.Quantity(resource.MustParse("100")),
+									Ceiling:    resource.MustParse("100"),
 								},
 							},
 						},
@@ -164,7 +163,7 @@ func TestSchedule(t *testing.T) {
 								ResourceFlavor: "model-a",
 								Quota: kueue.Quota{
 									Guaranteed: resource.MustParse("20"),
-									Ceiling:    pointer.Quantity(resource.MustParse("20")),
+									Ceiling:    resource.MustParse("20"),
 								},
 							},
 						},
@@ -843,8 +842,8 @@ func TestEntryAssignFlavors(t *testing.T) {
 			},
 			clusterQueue: cache.ClusterQueue{
 				RequestableResources: map[corev1.ResourceName][]cache.FlavorLimits{
-					corev1.ResourceCPU:    {{Name: "default", Guaranteed: 1000}},
-					corev1.ResourceMemory: {{Name: "default", Guaranteed: 2 * utiltesting.Mi}},
+					corev1.ResourceCPU:    defaultFlavorNoBorrowing(1000),
+					corev1.ResourceMemory: defaultFlavorNoBorrowing(2 * utiltesting.Mi),
 				},
 			},
 			wantFits: true,
@@ -868,20 +867,18 @@ func TestEntryAssignFlavors(t *testing.T) {
 								},
 							},
 						},
-						Tolerations: []corev1.Toleration{{
+						Tolerations: []corev1.Toleration{corev1.Toleration{
 							Key:      "instance",
 							Operator: corev1.TolerationOpEqual,
 							Value:    "spot",
 							Effect:   corev1.TaintEffectNoSchedule,
 						}},
-					},
-				},
-			},
+					}}},
 			clusterQueue: cache.ClusterQueue{
 				RequestableResources: map[corev1.ResourceName][]cache.FlavorLimits{
-					corev1.ResourceCPU: {
+					corev1.ResourceCPU: noBorrowing([]cache.FlavorLimits{
 						{Name: "tainted", Guaranteed: 4000},
-					},
+					}),
 				},
 			},
 			wantFits: true,
@@ -903,7 +900,7 @@ func TestEntryAssignFlavors(t *testing.T) {
 			},
 			clusterQueue: cache.ClusterQueue{
 				RequestableResources: map[corev1.ResourceName][]cache.FlavorLimits{
-					corev1.ResourceCPU: {{Name: "default", Guaranteed: 4000}},
+					corev1.ResourceCPU: defaultFlavorNoBorrowing(4000),
 				},
 				UsedResources: cache.Resources{
 					corev1.ResourceCPU: {
@@ -925,14 +922,14 @@ func TestEntryAssignFlavors(t *testing.T) {
 			},
 			clusterQueue: cache.ClusterQueue{
 				RequestableResources: map[corev1.ResourceName][]cache.FlavorLimits{
-					corev1.ResourceCPU: {
+					corev1.ResourceCPU: noBorrowing([]cache.FlavorLimits{
 						{Name: "one", Guaranteed: 2000},
 						{Name: "two", Guaranteed: 4000},
-					},
-					corev1.ResourceMemory: {
+					}),
+					corev1.ResourceMemory: noBorrowing([]cache.FlavorLimits{
 						{Name: "one", Guaranteed: utiltesting.Gi},
 						{Name: "two", Guaranteed: 5 * utiltesting.Mi},
-					},
+					}),
 				},
 			},
 			wantFits: true,
@@ -956,14 +953,14 @@ func TestEntryAssignFlavors(t *testing.T) {
 			},
 			clusterQueue: cache.ClusterQueue{
 				RequestableResources: map[corev1.ResourceName][]cache.FlavorLimits{
-					corev1.ResourceCPU: {
+					corev1.ResourceCPU: noBorrowing([]cache.FlavorLimits{
 						{Name: "one", Guaranteed: 2000},
 						{Name: "two", Guaranteed: 4000},
-					},
-					corev1.ResourceMemory: {
+					}),
+					corev1.ResourceMemory: noBorrowing([]cache.FlavorLimits{
 						{Name: "one", Guaranteed: utiltesting.Gi},
 						{Name: "two", Guaranteed: 5 * utiltesting.Mi},
-					},
+					}),
 				},
 			},
 		},
@@ -979,10 +976,10 @@ func TestEntryAssignFlavors(t *testing.T) {
 			},
 			clusterQueue: cache.ClusterQueue{
 				RequestableResources: map[corev1.ResourceName][]cache.FlavorLimits{
-					corev1.ResourceCPU: {
+					corev1.ResourceCPU: noBorrowing([]cache.FlavorLimits{
 						{Name: "tainted", Guaranteed: 4000},
 						{Name: "two", Guaranteed: 4000},
-					},
+					}),
 				},
 			},
 			wantFits: true,
@@ -1004,10 +1001,10 @@ func TestEntryAssignFlavors(t *testing.T) {
 			},
 			clusterQueue: cache.ClusterQueue{
 				RequestableResources: map[corev1.ResourceName][]cache.FlavorLimits{
-					corev1.ResourceCPU: {
+					corev1.ResourceCPU: noBorrowing([]cache.FlavorLimits{
 						{Name: "non-existent", Guaranteed: 4000},
 						{Name: "two", Guaranteed: 4000},
-					},
+					}),
 				},
 			},
 			wantFits: true,
@@ -1053,10 +1050,10 @@ func TestEntryAssignFlavors(t *testing.T) {
 			},
 			clusterQueue: cache.ClusterQueue{
 				RequestableResources: map[corev1.ResourceName][]cache.FlavorLimits{
-					corev1.ResourceCPU: {
+					corev1.ResourceCPU: noBorrowing([]cache.FlavorLimits{
 						{Name: "one", Guaranteed: 4000},
 						{Name: "two", Guaranteed: 4000},
-					},
+					}),
 				},
 				LabelKeys: map[corev1.ResourceName]sets.String{corev1.ResourceCPU: sets.NewString("cpuType")},
 			},
@@ -1104,14 +1101,14 @@ func TestEntryAssignFlavors(t *testing.T) {
 			},
 			clusterQueue: cache.ClusterQueue{
 				RequestableResources: map[corev1.ResourceName][]cache.FlavorLimits{
-					corev1.ResourceCPU: {
+					corev1.ResourceCPU: noBorrowing([]cache.FlavorLimits{
 						{Name: "one", Guaranteed: 4000},
 						{Name: "two", Guaranteed: 4000},
-					},
-					corev1.ResourceMemory: {
+					}),
+					corev1.ResourceMemory: noBorrowing([]cache.FlavorLimits{
 						{Name: "one", Guaranteed: utiltesting.Gi},
 						{Name: "two", Guaranteed: utiltesting.Gi},
-					},
+					}),
 				},
 			},
 			wantFits: true,
@@ -1168,10 +1165,10 @@ func TestEntryAssignFlavors(t *testing.T) {
 			},
 			clusterQueue: cache.ClusterQueue{
 				RequestableResources: map[corev1.ResourceName][]cache.FlavorLimits{
-					corev1.ResourceCPU: {
+					corev1.ResourceCPU: noBorrowing([]cache.FlavorLimits{
 						{Name: "one", Guaranteed: 4000},
 						{Name: "two", Guaranteed: 4000},
-					},
+					}),
 				},
 			},
 			wantFits: true,
@@ -1214,10 +1211,10 @@ func TestEntryAssignFlavors(t *testing.T) {
 			},
 			clusterQueue: cache.ClusterQueue{
 				RequestableResources: map[corev1.ResourceName][]cache.FlavorLimits{
-					corev1.ResourceCPU: {
+					corev1.ResourceCPU: noBorrowing([]cache.FlavorLimits{
 						{Name: "one", Guaranteed: 4000},
 						{Name: "two", Guaranteed: 4000},
-					},
+					}),
 				},
 				LabelKeys: map[corev1.ResourceName]sets.String{corev1.ResourceCPU: sets.NewString("cpuType")},
 			},
@@ -1242,10 +1239,10 @@ func TestEntryAssignFlavors(t *testing.T) {
 			},
 			clusterQueue: cache.ClusterQueue{
 				RequestableResources: map[corev1.ResourceName][]cache.FlavorLimits{
-					corev1.ResourceCPU: {
+					corev1.ResourceCPU: noBorrowing([]cache.FlavorLimits{
 						{Name: "one", Guaranteed: 4000},
 						{Name: "two", Guaranteed: 10_000},
-					},
+					}),
 				},
 			},
 			wantFits: true,
@@ -1283,14 +1280,14 @@ func TestEntryAssignFlavors(t *testing.T) {
 						{
 							Name:       "default",
 							Guaranteed: 2000,
-							Ceiling:    pointer.Int64(100_000),
+							Ceiling:    100_000,
 						},
 					},
 					corev1.ResourceMemory: {
 						{
 							Name:       "default",
 							Guaranteed: 2 * utiltesting.Gi,
-							// No ceiling.
+							Ceiling:    100 * utiltesting.Gi,
 						},
 					},
 				},
@@ -1341,7 +1338,7 @@ func TestEntryAssignFlavors(t *testing.T) {
 						{
 							Name:       "one",
 							Guaranteed: 1000,
-							// No ceiling.
+							Ceiling:    10_000,
 						},
 					},
 				},
@@ -1371,7 +1368,7 @@ func TestEntryAssignFlavors(t *testing.T) {
 						{
 							Name:       "one",
 							Guaranteed: 1000,
-							Ceiling:    pointer.Int64(10_000),
+							Ceiling:    10_000,
 						},
 					},
 				},
@@ -1475,3 +1472,14 @@ func TestEntryOrdering(t *testing.T) {
 		t.Errorf("Unexpected order (-want,+got):\n%s", diff)
 	}
 }
+
+func defaultFlavorNoBorrowing(guaranteed int64) []cache.FlavorLimits {
+	return noBorrowing([]cache.FlavorLimits{{Name: "default", Guaranteed: guaranteed}})
+}
+
+func noBorrowing(flavors []cache.FlavorLimits) []cache.FlavorLimits {
+	for i := range flavors {
+		flavors[i].Ceiling = flavors[i].Guaranteed
+	}
+	return flavors
+}
diff --git a/pkg/util/testing/wrappers.go b/pkg/util/testing/wrappers.go
index bf48e11d2..2aa4ecedd 100644
--- a/pkg/util/testing/wrappers.go
+++ b/pkg/util/testing/wrappers.go
@@ -24,10 +24,10 @@ import (
 	schedulingv1 "k8s.io/api/scheduling/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/pointer"
 
 	kueue "sigs.k8s.io/kueue/api/v1alpha1"
 	"sigs.k8s.io/kueue/pkg/constants"
-	"sigs.k8s.io/kueue/pkg/util/pointer"
 )
 
 // JobWrapper wraps a Job.
@@ -42,8 +42,8 @@ func MakeJob(name, ns string) *JobWrapper {
 			Annotations: make(map[string]string, 1),
 		},
 		Spec: batchv1.JobSpec{
-			Parallelism: pointer.Int32(1),
-			Suspend:     pointer.Bool(true),
+			Parallelism: pointer.Int32Ptr(1),
+			Suspend:     pointer.BoolPtr(true),
 			Template: corev1.PodTemplateSpec{
 				Spec: corev1.PodSpec{
 					RestartPolicy: "Never",
@@ -69,13 +69,13 @@ func (j *JobWrapper) Obj() *batchv1.Job {
 
 // Suspend updates the suspend status of the job
 func (j *JobWrapper) Suspend(s bool) *JobWrapper {
-	j.Spec.Suspend = pointer.Bool(s)
+	j.Spec.Suspend = pointer.BoolPtr(s)
 	return j
 }
 
 // Parallelism updates job parallelism.
 func (j *JobWrapper) Parallelism(p int32) *JobWrapper {
-	j.Spec.Parallelism = pointer.Int32(p)
+	j.Spec.Parallelism = pointer.Int32Ptr(p)
 	return j
 }
 
@@ -315,6 +315,7 @@ func MakeFlavor(rf, guaranteed string) *FlavorWrapper {
 		ResourceFlavor: kueue.ResourceFlavorReference(rf),
 		Quota: kueue.Quota{
 			Guaranteed: resource.MustParse(guaranteed),
+			Ceiling:    resource.MustParse(guaranteed),
 		},
 	}}
 }
@@ -326,7 +327,7 @@ func (f *FlavorWrapper) Obj() *kueue.Flavor {
 
 // Ceiling updates the flavor ceiling.
 func (f *FlavorWrapper) Ceiling(c string) *FlavorWrapper {
-	f.Quota.Ceiling = pointer.Quantity(resource.MustParse(c))
+	f.Quota.Ceiling = resource.MustParse(c)
 	return f
 }
 
diff --git a/test/integration/controller/core/clusterqueue_controller_test.go b/test/integration/controller/core/clusterqueue_controller_test.go
index 8498675c3..e39b60ca8 100644
--- a/test/integration/controller/core/clusterqueue_controller_test.go
+++ b/test/integration/controller/core/clusterqueue_controller_test.go
@@ -43,10 +43,19 @@ const (
 
 var _ = ginkgo.Describe("ClusterQueue controller", func() {
 	var (
-		ns            *corev1.Namespace
-		clusterQueue  *kueue.ClusterQueue
-		queue         *kueue.Queue
-		emptyCqStatus kueue.ClusterQueueStatus
+		ns                 *corev1.Namespace
+		clusterQueue       *kueue.ClusterQueue
+		queue              *kueue.Queue
+		emptyUsedResources = kueue.UsedResources{
+			corev1.ResourceCPU: {
+				flavorOnDemand: {Total: pointer.Quantity(resource.MustParse("0"))},
+				flavorSpot:     {Total: pointer.Quantity(resource.MustParse("0"))},
+			},
+			resourceGPU: {
+				flavorModelA: {Total: pointer.Quantity(resource.MustParse("0"))},
+				flavorModelB: {Total: pointer.Quantity(resource.MustParse("0"))},
+			},
+		}
 	)
 
 	ginkgo.BeforeEach(func() {
@@ -73,19 +82,6 @@ var _ = ginkgo.Describe("ClusterQueue controller", func() {
 		gomega.Expect(k8sClient.Create(ctx, clusterQueue)).To(gomega.Succeed())
 		queue = testing.MakeQueue("queue", ns.Name).ClusterQueue(clusterQueue.Name).Obj()
 		gomega.Expect(k8sClient.Create(ctx, queue)).To(gomega.Succeed())
-		emptyCqStatus = kueue.ClusterQueueStatus{
-			AdmittedWorkloads: 0,
-			UsedResources: kueue.UsedResources{
-				corev1.ResourceCPU: {
-					flavorOnDemand: {Total: pointer.Quantity(resource.MustParse("0"))},
-					flavorSpot:     {Total: pointer.Quantity(resource.MustParse("0"))},
-				},
-				resourceGPU: {
-					flavorModelA: {Total: pointer.Quantity(resource.MustParse("0"))},
-					flavorModelB: {Total: pointer.Quantity(resource.MustParse("0"))},
-				},
-			},
-		}
 	})
 
 	ginkgo.AfterEach(func() {
@@ -102,7 +98,7 @@ var _ = ginkgo.Describe("ClusterQueue controller", func() {
 				Request(corev1.ResourceCPU, "1").Request(resourceGPU, "1").Obj(),
 			testing.MakeQueuedWorkload("four", ns.Name).Queue(queue.Name).
 				Request(corev1.ResourceCPU, "1").Request(resourceGPU, "1").Obj(),
-			testing.MakeQueuedWorkload("five", ns.Name).Queue(queue.Name).
+			testing.MakeQueuedWorkload("five", ns.Name).Queue("other").
 				Request(corev1.ResourceCPU, "1").Request(resourceGPU, "1").Obj(),
 			testing.MakeQueuedWorkload("six", ns.Name).Queue(queue.Name).
 				Request(corev1.ResourceCPU, "1").Request(resourceGPU, "1").Obj(),
@@ -116,7 +112,10 @@ var _ = ginkgo.Describe("ClusterQueue controller", func() {
 			var updatedCq kueue.ClusterQueue
 			gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterQueue), &updatedCq)).To(gomega.Succeed())
 			return updatedCq.Status
-		}, framework.Timeout, framework.Interval).Should(testing.Equal(emptyCqStatus))
+		}, framework.Timeout, framework.Interval).Should(testing.Equal(kueue.ClusterQueueStatus{
+			PendingWorkloads: 5,
+			UsedResources:    emptyUsedResources,
+		}))
 
 		ginkgo.By("Admitting workloads")
 		admissions := []*kueue.Admission{
@@ -184,6 +183,8 @@ var _ = ginkgo.Describe("ClusterQueue controller", func() {
 			var updatedCq kueue.ClusterQueue
 			gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterQueue), &updatedCq)).To(gomega.Succeed())
 			return updatedCq.Status
-		}, framework.Timeout, framework.Interval).Should(testing.Equal(emptyCqStatus))
+		}, framework.Timeout, framework.Interval).Should(testing.Equal(kueue.ClusterQueueStatus{
+			UsedResources: emptyUsedResources,
+		}))
 	})
 })
diff --git a/test/integration/controller/core/queuedworkload_controller_test.go b/test/integration/controller/core/queuedworkload_controller_test.go
index 846f0fcf2..071a660e6 100644
--- a/test/integration/controller/core/queuedworkload_controller_test.go
+++ b/test/integration/controller/core/queuedworkload_controller_test.go
@@ -57,7 +57,7 @@ var _ = ginkgo.Describe("QueuedWorkload controller", func() {
 		})
 		ginkgo.It("Should update status when workloads are created", func() {
 			wl = testing.MakeQueuedWorkload("one", ns.Name).Request(corev1.ResourceCPU, "1").Obj()
-			message = fmt.Sprintf("Queue %s for workload doesn't exist", "")
+			message = fmt.Sprintf("Queue %s doesn't exist", "")
 			gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed())
 			gomega.Eventually(func() int {
 				gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), &updatedQueueWorkload)).To(gomega.Succeed())
@@ -74,7 +74,7 @@ var _ = ginkgo.Describe("QueuedWorkload controller", func() {
 		})
 		ginkgo.It("Should update status when workloads are created", func() {
 			wl = testing.MakeQueuedWorkload("two", ns.Name).Queue("nonCreatedQueue").Request(corev1.ResourceCPU, "1").Obj()
-			message = fmt.Sprintf("Queue %s for workload doesn't exist", "nonCreatedQueue")
+			message = fmt.Sprintf("Queue %s doesn't exist", "nonCreatedQueue")
 			gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed())
 			gomega.Eventually(func() int {
 				gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), &updatedQueueWorkload)).To(gomega.Succeed())
diff --git a/test/integration/framework/framework.go b/test/integration/framework/framework.go
index 110b9e134..e6f0cc03c 100644
--- a/test/integration/framework/framework.go
+++ b/test/integration/framework/framework.go
@@ -24,6 +24,7 @@ import (
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -115,7 +116,8 @@ func DeleteNamespace(ctx context.Context, c client.Client, ns *corev1.Namespace)
 	if ns == nil {
 		return nil
 	}
-	if err := c.DeleteAllOf(ctx, &batchv1.Job{}, client.InNamespace(ns.Name)); err != nil && !apierrors.IsNotFound(err) {
+	err := c.DeleteAllOf(ctx, &batchv1.Job{}, client.InNamespace(ns.Name), client.PropagationPolicy(metav1.DeletePropagationBackground))
+	if err != nil && !apierrors.IsNotFound(err) {
 		return err
 	}
 	if err := c.DeleteAllOf(ctx, &kueue.Queue{}, client.InNamespace(ns.Name)); err != nil && !apierrors.IsNotFound(err) {
diff --git a/test/integration/scheduler/scheduler_test.go b/test/integration/scheduler/scheduler_test.go
index 996184c06..159aca133 100644
--- a/test/integration/scheduler/scheduler_test.go
+++ b/test/integration/scheduler/scheduler_test.go
@@ -46,7 +46,6 @@ var _ = ginkgo.Describe("Scheduler", func() {
 		onDemandFlavor      *kueue.ResourceFlavor
 		spotTaintedFlavor   *kueue.ResourceFlavor
 		spotUntaintedFlavor *kueue.ResourceFlavor
-		spotToleration      corev1.Toleration
 	)
 
 	ginkgo.BeforeEach(func() {
@@ -68,12 +67,6 @@ var _ = ginkgo.Describe("Scheduler", func() {
 				Effect: corev1.TaintEffectNoSchedule,
 			}).Obj()
 		gomega.Expect(k8sClient.Create(ctx, spotTaintedFlavor)).Should(gomega.Succeed())
-		spotToleration = corev1.Toleration{
-			Key:      instanceKey,
-			Operator: corev1.TolerationOpEqual,
-			Value:    spotTaintedFlavor.Name,
-			Effect:   corev1.TaintEffectNoSchedule,
-		}
 
 		spotUntaintedFlavor = testing.MakeResourceFlavor("spot-untainted").Label(instanceKey, "spot-untainted").Obj()
 		gomega.Expect(k8sClient.Create(ctx, spotUntaintedFlavor)).Should(gomega.Succeed())
@@ -81,8 +74,8 @@ var _ = ginkgo.Describe("Scheduler", func() {
 		prodClusterQ = testing.MakeClusterQueue("prod-cq").
 			Cohort("prod").
 			Resource(testing.MakeResource(corev1.ResourceCPU).
-				Flavor(testing.MakeFlavor(spotTaintedFlavor.Name, "5").Ceiling("5").Obj()).
-				Flavor(testing.MakeFlavor(onDemandFlavor.Name, "5").Obj()).
+				Flavor(testing.MakeFlavor(spotTaintedFlavor.Name, "5").Obj()).
+				Flavor(testing.MakeFlavor(onDemandFlavor.Name, "5").Ceiling("10").Obj()).
 				Obj()).
 			Obj()
 		gomega.Expect(k8sClient.Create(ctx, prodClusterQ)).Should(gomega.Succeed())
@@ -182,7 +175,12 @@ var _ = ginkgo.Describe("Scheduler", func() {
 
 		ginkgo.By("checking a third job with toleration starts")
 		job3 := testing.MakeJob("spot-job3", ns.Name).Queue(prodQueue.Name).
-			Toleration(spotToleration).
+			Toleration(corev1.Toleration{
+				Key:      instanceKey,
+				Operator: corev1.TolerationOpEqual,
+				Value:    spotTaintedFlavor.Name,
+				Effect:   corev1.TaintEffectNoSchedule,
+			}).
 			Request(corev1.ResourceCPU, "5").Obj()
 		gomega.Expect(k8sClient.Create(ctx, job3)).Should(gomega.Succeed())
 		createdJob3 := &batchv1.Job{}
@@ -195,8 +193,7 @@ var _ = ginkgo.Describe("Scheduler", func() {
 
 	ginkgo.It("Should schedule jobs using borrowed ClusterQueue", func() {
 		ginkgo.By("checking a no-fit job does not start")
-		job := testing.MakeJob("job", ns.Name).Queue(prodQueue.Name).
-			Request(corev1.ResourceCPU, "10").Toleration(spotToleration).Obj()
+		job := testing.MakeJob("job", ns.Name).Queue(prodQueue.Name).Request(corev1.ResourceCPU, "10").Obj()
 		gomega.Expect(k8sClient.Create(ctx, job)).Should(gomega.Succeed())
 		lookupKey := types.NamespacedName{Name: job.Name, Namespace: job.Namespace}
 		createdJob := &batchv1.Job{}
@@ -208,8 +205,7 @@ var _ = ginkgo.Describe("Scheduler", func() {
 		fallbackClusterQueue := testing.MakeClusterQueue("fallback-cq").
 			Cohort(prodClusterQ.Spec.Cohort).
 			Resource(testing.MakeResource(corev1.ResourceCPU).
-				Flavor(testing.MakeFlavor(spotTaintedFlavor.Name, "5").Obj()). // prod-cq can't borrow this flavor due to its ceiling.
-				Flavor(testing.MakeFlavor(onDemandFlavor.Name, "5").Obj()).
+				Flavor(testing.MakeFlavor(onDemandFlavor.Name, "5").Ceiling("10").Obj()).
 				Obj()).
 			Obj()
 		gomega.Expect(k8sClient.Create(ctx, fallbackClusterQueue)).Should(gomega.Succeed())