Repository: kubernetes-sigs/kueue

Issue:
[MultiKueue test] The connection to a worker cluster is unreliable makes cluster not working properly
**What happened**:

This test leaves the cluster not working properly, if there are any tests executing after it they flake.

**What you expected to happen**:

The test should leave the cluster in usable state.

**How to reproduce it (as minimally and precisely as possible)**:

see https://github.com/kubernetes-sigs/kueue/issues/6386

**Anything else we need to know?**:

Maybe we could call AwaitForAvailablity on both workers and manager clusters.

Hints from issue comments:
@mimowo: cc @mszadkow @mbobrovskyi it will be great to fix this@mszadkow: /assign@mszadkow: I believe I found the fix.
I tested back to back 10 times the mentioned test.
The problem was that kube-system components were still in CrashLoopBackoff.
The fix includes the need for checking that.
PR is on its way.

Solution patch:
diff --git a/cmd/kueueviz/frontend/package-lock.json b/cmd/kueueviz/frontend/package-lock.json
index 36f323028..6e1ab3a17 100644
--- a/cmd/kueueviz/frontend/package-lock.json
+++ b/cmd/kueueviz/frontend/package-lock.json
@@ -21,7 +21,7 @@
         "react-toastify": "^11.0.5"
       },
       "devDependencies": {
-        "@vitejs/plugin-react": "^4.7.0",
+        "@vitejs/plugin-react": "^5.0.0",
         "depcheck": "^1.4.7",
         "vite": "^7.1.1"
       }
@@ -1163,9 +1163,9 @@
       }
     },
     "node_modules/@rolldown/pluginutils": {
-      "version": "1.0.0-beta.27",
-      "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz",
-      "integrity": "sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==",
+      "version": "1.0.0-beta.30",
+      "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.30.tgz",
+      "integrity": "sha512-whXaSoNUFiyDAjkUF8OBpOm77Szdbk5lGNqFe6CbVbJFrhCCPinCbRA3NjawwlNHla1No7xvXXh+CpSxnPfUEw==",
       "dev": true,
       "license": "MIT"
     },
@@ -1551,21 +1551,21 @@
       }
     },
     "node_modules/@vitejs/plugin-react": {
-      "version": "4.7.0",
-      "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.7.0.tgz",
-      "integrity": "sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA==",
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-5.0.0.tgz",
+      "integrity": "sha512-Jx9JfsTa05bYkS9xo0hkofp2dCmp1blrKjw9JONs5BTHOvJCgLbaPSuZLGSVJW6u2qe0tc4eevY0+gSNNi0YCw==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "@babel/core": "^7.28.0",
         "@babel/plugin-transform-react-jsx-self": "^7.27.1",
         "@babel/plugin-transform-react-jsx-source": "^7.27.1",
-        "@rolldown/pluginutils": "1.0.0-beta.27",
+        "@rolldown/pluginutils": "1.0.0-beta.30",
         "@types/babel__core": "^7.20.5",
         "react-refresh": "^0.17.0"
       },
       "engines": {
-        "node": "^14.18.0 || >=16.0.0"
+        "node": "^20.19.0 || >=22.12.0"
       },
       "peerDependencies": {
         "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0"
diff --git a/cmd/kueueviz/frontend/package.json b/cmd/kueueviz/frontend/package.json
index 4b76755e1..927dbbe74 100644
--- a/cmd/kueueviz/frontend/package.json
+++ b/cmd/kueueviz/frontend/package.json
@@ -25,7 +25,7 @@
     "react-toastify": "^11.0.5"
   },
   "devDependencies": {
-    "@vitejs/plugin-react": "^4.7.0",
+    "@vitejs/plugin-react": "^5.0.0",
     "depcheck": "^1.4.7",
     "vite": "^7.1.1"
   },
diff --git a/pkg/workloadslicing/workloadslicing.go b/pkg/workloadslicing/workloadslicing.go
index e91094197..9e61cd502 100644
--- a/pkg/workloadslicing/workloadslicing.go
+++ b/pkg/workloadslicing/workloadslicing.go
@@ -28,7 +28,6 @@ import (
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apiserver/pkg/storage"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1"
@@ -112,21 +111,6 @@ func Finish(ctx context.Context, clnt client.Client, workloadSlice *kueue.Worklo
 	return nil
 }
 
-// versionHelper is used in the FindNotFinishedWorkloads sort routine as a tiebreaker
-// for workloads with identical creationTimestamp values.
-//
-// Note: Kubernetes documentation explicitly states:
-//
-//	"Clients must not assume that resource versions are numeric or collatable."
-//
-// To comply with this guidance, we rely on the Kubernetes apiserver/storage library
-// (storage.APIObjectVersioner) to perform resourceVersion comparisons.
-//
-// This ensures that if Kubernetes changes its internal resourceVersion format or semantics,
-// the helper will either be updated accordingly or removed entirely, in which case,
-// such changes will surface at compile time, prompting this code to be updated as needed.
-var versionHelper = storage.APIObjectVersioner{}
-
 // FindNotFinishedWorkloads returns a sorted list of workloads "owned by" the provided job object/gvk combination and
 // without "Finished" condition with status = "True".
 func FindNotFinishedWorkloads(ctx context.Context, clnt client.Client, jobObject client.Object, jobObjectGVK schema.GroupVersionKind) ([]kueue.Workload, error) {
@@ -137,14 +121,14 @@ func FindNotFinishedWorkloads(ctx context.Context, clnt client.Client, jobObject
 
 	// Sort workloads by creation timestamp, oldest first.
 	// In the rare case that two workload slices have identical creationTimestamp values
-	// (due to RFC3339 second-level precision), fall back to resourceVersion comparison
+	// (due to RFC3339 second-level precision), use WorkloadSliceReplacementFor
 	// as a tiebreaker. This edge case is uncommon in production but can occur in
 	// integration or e2e tests where the original and scaled-up workloads are created
 	// in rapid succession.
 	sort.Slice(list.Items, func(i, j int) bool {
 		a, b := list.Items[i], list.Items[j]
 		if a.CreationTimestamp.Equal(&b.CreationTimestamp) {
-			return versionHelper.CompareResourceVersion(&a, &b) < 0
+			return b.Annotations[WorkloadSliceReplacementFor] == string(workload.Key(&a))
 		}
 		return a.CreationTimestamp.Before(&b.CreationTimestamp)
 	})
@@ -217,12 +201,12 @@ func EnsureWorkloadSlices(ctx context.Context, clnt client.Client, jobPodSets []
 		// This transient state typically occurs when a new slice has been created,
 		// and the old slice is pending deactivation. The system should resolve this
 		// by deactivating the old slice, after which processing will continue under "case #1".
-
-		// There is also an edge-case when the old slice was preempted/evicted by the scheduler
-		// to make the room for other (than new slice) workload, which would result in two
-		// "pending" workloads. In such case - it is safe to deactivate the old slice.
 		oldWorkload := workloads[0]
-		if !workload.HasQuotaReservation(&oldWorkload) {
+
+		// Finish the old workload slice if it lost its quota reservation or if it was
+		// explicitly evicted.
+		if evictedCondition := apimeta.FindStatusCondition(oldWorkload.Status.Conditions, kueue.WorkloadEvicted); !workload.HasQuotaReservation(&oldWorkload) || evictedCondition != nil {
+			// Finish the old workload slice as out of sync.
 			if err := Finish(ctx, clnt, &oldWorkload, kueue.WorkloadFinishedReasonOutOfSync, "The workload slice is out of sync with its parent job"); err != nil {
 				return nil, true, err
 			}



Test patch:
diff --git a/pkg/workloadslicing/workloadslicing_test.go b/pkg/workloadslicing/workloadslicing_test.go
index 1c8f596fd..7f8b03521 100644
--- a/pkg/workloadslicing/workloadslicing_test.go
+++ b/pkg/workloadslicing/workloadslicing_test.go
@@ -772,7 +772,7 @@ func TestEnsureWorkloadSlices(t *testing.T) {
 				compatible: true,
 			},
 		},
-		"TwoWorkloadSlices_NewIsIncompatible": {
+		"TwoWorkloads_NewIsIncompatible": {
 			args: args{
 				ctx: t.Context(),
 				clnt: testWorkloadClientBuilder().WithObjects(
@@ -823,6 +823,34 @@ func TestEnsureWorkloadSlices(t *testing.T) {
 				compatible: true,
 			},
 		},
+		"TwoWorkloads_OldWithReservedQuotaAndEvicted_NewWithoutQuotaReservation": {
+			args: args{
+				ctx: t.Context(),
+				clnt: testWorkloadClientBuilder().WithObjects(
+					utiltesting.MakeWorkload(testJobObject.Name+"-1", testJobObject.Namespace).
+						OwnerReference(testJobGVK, testJobObject.Name, string(testJobObject.UID)).
+						ResourceVersion("1").
+						Creation(fiveMinutesAgo).
+						PodSets(*utiltesting.MakePodSet(kueue.DefaultPodSetName, 1).Request(corev1.ResourceCPU, "1").Obj()).
+						ReserveQuota(utiltesting.MakeAdmission("default", kueue.DefaultPodSetName).Assignment(corev1.ResourceCPU, "default", "1").AssignmentPodCount(1).Obj()).
+						Evicted().
+						Obj(),
+					utiltesting.MakeWorkload(testJobObject.Name+"-2", testJobObject.Namespace).
+						OwnerReference(testJobGVK, testJobObject.Name, string(testJobObject.UID)).
+						ResourceVersion("1").
+						Creation(now).
+						PodSets(*utiltesting.MakePodSet(kueue.DefaultPodSetName, 3).Request(corev1.ResourceCPU, "1").Obj()).
+						Obj()).
+					Build(),
+				jobPodSets:   []kueue.PodSet{*utiltesting.MakePodSet(kueue.DefaultPodSetName, 5).Request(corev1.ResourceCPU, "1").Obj()},
+				jobObject:    testJobObject,
+				jobObjectGVK: testJobGVK,
+			},
+			want: want{
+				error:      true,
+				compatible: true,
+			},
+		},
 		"TwoWorkloadSlices_NewIsUnreservedAndCurrent": {
 			args: args{
 				ctx: t.Context(),
diff --git a/test/e2e/multikueue/e2e_test.go b/test/e2e/multikueue/e2e_test.go
index f85e849c4..6d3072c74 100644
--- a/test/e2e/multikueue/e2e_test.go
+++ b/test/e2e/multikueue/e2e_test.go
@@ -171,11 +171,11 @@ var _ = ginkgo.Describe("MultiKueue", func() {
 		gomega.Expect(util.DeleteNamespace(ctx, k8sWorker1Client, worker1Ns)).To(gomega.Succeed())
 		gomega.Expect(util.DeleteNamespace(ctx, k8sWorker2Client, worker2Ns)).To(gomega.Succeed())
 
-		util.ExpectObjectToBeDeleted(ctx, k8sWorker1Client, worker1Cq, true)
-		util.ExpectObjectToBeDeleted(ctx, k8sWorker1Client, worker1Flavor, true)
+		util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sWorker1Client, worker1Cq, true, util.LongTimeout)
+		util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sWorker1Client, worker1Flavor, true, util.LongTimeout)
 
-		util.ExpectObjectToBeDeleted(ctx, k8sWorker2Client, worker2Cq, true)
-		util.ExpectObjectToBeDeleted(ctx, k8sWorker2Client, worker2Flavor, true)
+		util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sWorker2Client, worker2Cq, true, util.LongTimeout)
+		util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sWorker2Client, worker2Flavor, true, util.LongTimeout)
 
 		util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sManagerClient, managerCq, true, util.LongTimeout)
 		util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sManagerClient, managerFlavor, true, util.LongTimeout)
diff --git a/test/integration/singlecluster/controller/jobs/job/job_controller_test.go b/test/integration/singlecluster/controller/jobs/job/job_controller_test.go
index 632b9545d..bcf154db5 100644
--- a/test/integration/singlecluster/controller/jobs/job/job_controller_test.go
+++ b/test/integration/singlecluster/controller/jobs/job/job_controller_test.go
@@ -19,6 +19,7 @@ package job
 import (
 	"fmt"
 	"maps"
+	"strconv"
 	"time"
 
 	"github.com/google/go-cmp/cmp/cmpopts"
@@ -3095,6 +3096,7 @@ var _ = ginkgo.Describe("Job with elastic jobs via workload-slices support", gin
 		clusterQueue   *kueue.ClusterQueue
 		localQueue     *kueue.LocalQueue
 	)
+	cpuNominalQuota := 5
 
 	ginkgo.BeforeAll(func() {
 		gomega.Expect(utilfeature.DefaultMutableFeatureGate.SetFromMap(map[string]bool{string(features.ElasticJobsViaWorkloadSlices): true})).Should(gomega.Succeed())
@@ -3111,14 +3113,16 @@ var _ = ginkgo.Describe("Job with elastic jobs via workload-slices support", gin
 		util.MustCreate(ctx, k8sClient, resourceFlavor)
 
 		clusterQueue = testing.MakeClusterQueue("default").
-			ResourceGroup(*testing.MakeFlavorQuotas("default").Resource(corev1.ResourceCPU, "5").Obj()).
+			ResourceGroup(*testing.MakeFlavorQuotas(resourceFlavor.Name).Resource(corev1.ResourceCPU, strconv.Itoa(cpuNominalQuota)).Obj()).
+			Preemption(kueue.ClusterQueuePreemption{
+				WithinClusterQueue: kueue.PreemptionPolicyLowerPriority,
+			}).
 			Obj()
 		util.MustCreate(ctx, k8sClient, clusterQueue)
 
 		localQueue = testing.MakeLocalQueue("default", ns.Name).ClusterQueue(clusterQueue.Name).Obj()
 		util.MustCreate(ctx, k8sClient, localQueue)
 	})
-
 	ginkgo.AfterEach(func() {
 		gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed())
 		util.ExpectObjectToBeDeleted(ctx, k8sClient, clusterQueue, true)
@@ -3126,180 +3130,249 @@ var _ = ginkgo.Describe("Job with elastic jobs via workload-slices support", gin
 		util.ExpectObjectToBeDeleted(ctx, k8sClient, resourceFlavor, true)
 	})
 
-	ginkgo.When("ElasticJobsViaWorkloadSlices is enabled", func() {
-		ginkgo.When("Job is opted in", func() {
-			ginkgo.It("Should support job scale-down and scale-up", func() {
-				testJob := testingjob.MakeJob("job1", ns.Name).
-					SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue).
-					Queue(kueue.LocalQueueName(localQueue.Name)).
-					Request(corev1.ResourceCPU, "100m").
-					Parallelism(3).
-					Completions(3).
-					Obj()
+	ginkgo.It("Should support job scale-down and scale-up", func() {
+		testJob := testingjob.MakeJob("job1", ns.Name).
+			SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue).
+			Queue(kueue.LocalQueueName(localQueue.Name)).
+			Request(corev1.ResourceCPU, "100m").
+			Parallelism(3).
+			Completions(3).
+			Obj()
 
-				var testJobWorkload *kueue.Workload
+		var testJobWorkload *kueue.Workload
 
-				ginkgo.By("creating a job")
-				util.MustCreate(ctx, k8sClient, testJob)
+		ginkgo.By("creating a job")
+		util.MustCreate(ctx, k8sClient, testJob)
 
-				ginkgo.By("admitting the job's workload")
-				gomega.Eventually(func(g gomega.Gomega) {
-					workloads := &kueue.WorkloadList{}
-					g.Expect(k8sClient.List(ctx, workloads, client.InNamespace(testJob.Namespace))).Should(gomega.Succeed())
-					g.Expect(workloads.Items).Should(gomega.HaveLen(1))
-					testJobWorkload = &workloads.Items[0]
-					util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, testJobWorkload)
-				}, util.Timeout, util.Interval).Should(gomega.Succeed())
+		ginkgo.By("admitting the job's workload")
+		gomega.Eventually(func(g gomega.Gomega) {
+			workloads := &kueue.WorkloadList{}
+			g.Expect(k8sClient.List(ctx, workloads, client.InNamespace(testJob.Namespace))).Should(gomega.Succeed())
+			g.Expect(workloads.Items).Should(gomega.HaveLen(1))
+			testJobWorkload = &workloads.Items[0]
+			util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, testJobWorkload)
+		}, util.Timeout, util.Interval).Should(gomega.Succeed())
 
-				ginkgo.By("the job is unsuspended")
-				gomega.Eventually(func(g gomega.Gomega) {
-					g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(testJob), testJob)).Should(gomega.Succeed())
-					g.Expect(ptr.Deref(testJob.Spec.Suspend, false)).Should(gomega.BeFalse())
-				}, util.Timeout, util.Interval).Should(gomega.Succeed())
+		ginkgo.By("the job is unsuspended")
+		gomega.Eventually(func(g gomega.Gomega) {
+			g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(testJob), testJob)).Should(gomega.Succeed())
+			g.Expect(ptr.Deref(testJob.Spec.Suspend, false)).Should(gomega.BeFalse())
+		}, util.Timeout, util.Interval).Should(gomega.Succeed())
 
-				ginkgo.By("resource flavor utilization is correctly recorded")
-				gomega.Eventually(func(g gomega.Gomega) {
-					cq := &kueue.ClusterQueue{}
-					g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterQueue), cq)).Should(gomega.Succeed())
-					g.Expect(len(cq.Status.FlavorsUsage)).Should(gomega.BeEquivalentTo(1))
-					g.Expect(len(cq.Status.FlavorsUsage[0].Resources)).Should(gomega.BeEquivalentTo(1))
-					g.Expect(cq.Status.FlavorsUsage[0].Resources[0].Total).Should(gomega.BeEquivalentTo(resource.MustParse("300m")))
-				}, util.Timeout, util.Interval).Should(gomega.Succeed())
+		ginkgo.By("resource flavor utilization is correctly recorded")
+		gomega.Eventually(func(g gomega.Gomega) {
+			cq := &kueue.ClusterQueue{}
+			g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterQueue), cq)).Should(gomega.Succeed())
+			g.Expect(len(cq.Status.FlavorsUsage)).Should(gomega.BeEquivalentTo(1))
+			g.Expect(len(cq.Status.FlavorsUsage[0].Resources)).Should(gomega.BeEquivalentTo(1))
+			g.Expect(cq.Status.FlavorsUsage[0].Resources[0].Total).Should(gomega.BeEquivalentTo(resource.MustParse("300m")))
+		}, util.Timeout, util.Interval).Should(gomega.Succeed())
 
-				ginkgo.By("reducing the job's parallelism to emulate scale-down operation")
-				gomega.Eventually(func(g gomega.Gomega) {
-					testJob.Spec.Parallelism = ptr.To(int32(1))
-					g.Expect(k8sClient.Update(ctx, testJob)).Should(gomega.Succeed())
-				}, util.Timeout, util.Interval).Should(gomega.Succeed())
+		ginkgo.By("reducing the job's parallelism to emulate scale-down operation")
+		gomega.Eventually(func(g gomega.Gomega) {
+			testJob.Spec.Parallelism = ptr.To(int32(1))
+			g.Expect(k8sClient.Update(ctx, testJob)).Should(gomega.Succeed())
+		}, util.Timeout, util.Interval).Should(gomega.Succeed())
 
-				ginkgo.By("resource flavor utilization is correctly updated")
-				gomega.Eventually(func(g gomega.Gomega) {
-					cq := &kueue.ClusterQueue{}
-					g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterQueue), cq)).Should(gomega.Succeed())
-					g.Expect(len(cq.Status.FlavorsUsage)).Should(gomega.BeEquivalentTo(1))
-					g.Expect(len(cq.Status.FlavorsUsage[0].Resources)).Should(gomega.BeEquivalentTo(1))
-					g.Expect(cq.Status.FlavorsUsage[0].Resources[0].Total).Should(gomega.BeEquivalentTo(resource.MustParse("100m")))
-				}, util.Timeout, util.Interval).Should(gomega.Succeed())
+		ginkgo.By("resource flavor utilization is correctly updated")
+		gomega.Eventually(func(g gomega.Gomega) {
+			cq := &kueue.ClusterQueue{}
+			g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterQueue), cq)).Should(gomega.Succeed())
+			g.Expect(len(cq.Status.FlavorsUsage)).Should(gomega.BeEquivalentTo(1))
+			g.Expect(len(cq.Status.FlavorsUsage[0].Resources)).Should(gomega.BeEquivalentTo(1))
+			g.Expect(cq.Status.FlavorsUsage[0].Resources[0].Total).Should(gomega.BeEquivalentTo(resource.MustParse("100m")))
+		}, util.Timeout, util.Interval).Should(gomega.Succeed())
 
-				ginkgo.By("assert the job's workload is updated and still admitted")
-				gomega.Eventually(func(g gomega.Gomega) {
-					workloads := &kueue.WorkloadList{}
-					g.Expect(k8sClient.List(ctx, workloads, client.InNamespace(testJob.Namespace))).Should(gomega.Succeed())
-					g.Expect(workloads.Items).Should(gomega.HaveLen(1))
-					util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, testJobWorkload)
-					g.Expect(workloads.Items[0].Spec.PodSets[0].Count).Should(gomega.BeEquivalentTo(int32(1)))
-					g.Expect(workloads.Items[0].UID).Should(gomega.BeEquivalentTo(testJobWorkload.UID))
-					testJobWorkload = &workloads.Items[0]
-				}, util.Timeout, util.Interval).Should(gomega.Succeed())
+		ginkgo.By("assert the job's workload is updated and still admitted")
+		gomega.Eventually(func(g gomega.Gomega) {
+			workloads := &kueue.WorkloadList{}
+			g.Expect(k8sClient.List(ctx, workloads, client.InNamespace(testJob.Namespace))).Should(gomega.Succeed())
+			g.Expect(workloads.Items).Should(gomega.HaveLen(1))
+			util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, testJobWorkload)
+			g.Expect(workloads.Items[0].Spec.PodSets[0].Count).Should(gomega.BeEquivalentTo(int32(1)))
+			g.Expect(workloads.Items[0].UID).Should(gomega.BeEquivalentTo(testJobWorkload.UID))
+			testJobWorkload = &workloads.Items[0]
+		}, util.Timeout, util.Interval).Should(gomega.Succeed())
 
-				ginkgo.By("increasing the job's parallelism to emulate scale-up operation")
-				gomega.Eventually(func(g gomega.Gomega) {
-					g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(testJob), testJob)).Should(gomega.Succeed())
-					testJob.Spec.Parallelism = ptr.To(int32(2))
-					g.Expect(k8sClient.Update(ctx, testJob)).Should(gomega.Succeed())
-				}, util.Timeout, util.Interval).Should(gomega.Succeed())
+		ginkgo.By("increasing the job's parallelism to emulate scale-up operation")
+		gomega.Eventually(func(g gomega.Gomega) {
+			g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(testJob), testJob)).Should(gomega.Succeed())
+			testJob.Spec.Parallelism = ptr.To(int32(2))
+			g.Expect(k8sClient.Update(ctx, testJob)).Should(gomega.Succeed())
+		}, util.Timeout, util.Interval).Should(gomega.Succeed())
 
-				ginkgo.By("resource flavor utilization is correctly updated")
-				gomega.Eventually(func(g gomega.Gomega) {
-					cq := &kueue.ClusterQueue{}
-					g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterQueue), cq)).Should(gomega.Succeed())
-					g.Expect(len(cq.Status.FlavorsUsage)).Should(gomega.BeEquivalentTo(1))
-					g.Expect(len(cq.Status.FlavorsUsage[0].Resources)).Should(gomega.BeEquivalentTo(1))
-					g.Expect(cq.Status.FlavorsUsage[0].Resources[0].Total).Should(gomega.BeEquivalentTo(resource.MustParse("200m")))
-				}, util.Timeout, util.Interval).Should(gomega.Succeed())
+		ginkgo.By("resource flavor utilization is correctly updated")
+		gomega.Eventually(func(g gomega.Gomega) {
+			cq := &kueue.ClusterQueue{}
+			g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterQueue), cq)).Should(gomega.Succeed())
+			g.Expect(len(cq.Status.FlavorsUsage)).Should(gomega.BeEquivalentTo(1))
+			g.Expect(len(cq.Status.FlavorsUsage[0].Resources)).Should(gomega.BeEquivalentTo(1))
+			g.Expect(cq.Status.FlavorsUsage[0].Resources[0].Total).Should(gomega.BeEquivalentTo(resource.MustParse("200m")))
+		}, util.Timeout, util.Interval).Should(gomega.Succeed())
 
-				ginkgo.By("old workload is finished and new workload is admitted")
-				gomega.Eventually(func(g gomega.Gomega) {
-					workloads := &kueue.WorkloadList{}
-					g.Expect(k8sClient.List(ctx, workloads, client.InNamespace(testJob.Namespace))).Should(gomega.Succeed())
-					g.Expect(workloads.Items).Should(gomega.HaveLen(2))
-					for i := range workloads.Items {
-						if workloads.Items[i].Name == testJobWorkload.Name {
-							g.Expect(workload.IsFinished(&workloads.Items[i])).Should(gomega.BeTrue())
-							continue
-						}
-						testJobWorkload = &workloads.Items[i]
-						util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, testJobWorkload)
-					}
-				}, util.Timeout, util.Interval).Should(gomega.Succeed())
-			})
+		ginkgo.By("old workload is finished and new workload is admitted")
+		gomega.Eventually(func(g gomega.Gomega) {
+			workloads := &kueue.WorkloadList{}
+			g.Expect(k8sClient.List(ctx, workloads, client.InNamespace(testJob.Namespace))).Should(gomega.Succeed())
+			g.Expect(workloads.Items).Should(gomega.HaveLen(2))
+			for i := range workloads.Items {
+				if workloads.Items[i].Name == testJobWorkload.Name {
+					g.Expect(workload.IsFinished(&workloads.Items[i])).Should(gomega.BeTrue())
+					continue
+				}
+				testJobWorkload = &workloads.Items[i]
+				util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, testJobWorkload)
+			}
+		}, util.Timeout, util.Interval).Should(gomega.Succeed())
+	})
 
-			ginkgo.It("Should support scheduling pending workload after freeing capacity on scale-down", func() {
-				var (
-					testJobAWorkload *kueue.Workload
-					testJobBWorkload *kueue.Workload
-				)
-
-				testJobA := testingjob.MakeJob("job-a", ns.Name).
-					SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue).
-					Queue(kueue.LocalQueueName(localQueue.Name)).
-					Request(corev1.ResourceCPU, "1").
-					Parallelism(3).
-					Completions(3).
-					Obj()
+	ginkgo.It("Should support scheduling pending workload after freeing capacity on scale-down", func() {
+		var (
+			testJobAWorkload *kueue.Workload
+			testJobBWorkload *kueue.Workload
+		)
 
-				ginkgo.By("creating a job-a")
-				util.MustCreate(ctx, k8sClient, testJobA)
+		testJobA := testingjob.MakeJob("job-a", ns.Name).
+			SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue).
+			Queue(kueue.LocalQueueName(localQueue.Name)).
+			Request(corev1.ResourceCPU, "1").
+			Parallelism(3).
+			Completions(3).
+			Obj()
 
-				ginkgo.By("admitting the job-a's workload")
-				gomega.Eventually(func(g gomega.Gomega) {
-					workloads := &kueue.WorkloadList{}
-					g.Expect(k8sClient.List(ctx, workloads, client.InNamespace(testJobA.Namespace))).Should(gomega.Succeed())
-					g.Expect(workloads.Items).Should(gomega.HaveLen(1))
-					testJobAWorkload = &workloads.Items[0]
-					util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, testJobAWorkload)
-				}, util.Timeout, util.Interval).Should(gomega.Succeed())
+		ginkgo.By("creating a job-a")
+		util.MustCreate(ctx, k8sClient, testJobA)
 
-				ginkgo.By("the job-a is unsuspended")
-				gomega.Eventually(func(g gomega.Gomega) {
-					g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(testJobA), testJobA)).Should(gomega.Succeed())
-					g.Expect(ptr.Deref(testJobA.Spec.Suspend, false)).Should(gomega.BeFalse())
-				}, util.Timeout, util.Interval).Should(gomega.Succeed())
+		ginkgo.By("admitting the job-a's workload")
+		gomega.Eventually(func(g gomega.Gomega) {
+			workloads := &kueue.WorkloadList{}
+			g.Expect(k8sClient.List(ctx, workloads, client.InNamespace(testJobA.Namespace))).Should(gomega.Succeed())
+			g.Expect(workloads.Items).Should(gomega.HaveLen(1))
+			testJobAWorkload = &workloads.Items[0]
+			util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, testJobAWorkload)
+		}, util.Timeout, util.Interval).Should(gomega.Succeed())
 
-				testJobB := testingjob.MakeJob("job-b", ns.Name).
-					SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue).
-					Queue(kueue.LocalQueueName(localQueue.Name)).
-					Request(corev1.ResourceCPU, "1").
-					Parallelism(3).
-					Completions(3).
-					Obj()
+		ginkgo.By("the job-a is unsuspended")
+		gomega.Eventually(func(g gomega.Gomega) {
+			g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(testJobA), testJobA)).Should(gomega.Succeed())
+			g.Expect(ptr.Deref(testJobA.Spec.Suspend, false)).Should(gomega.BeFalse())
+		}, util.Timeout, util.Interval).Should(gomega.Succeed())
 
-				ginkgo.By("creating a job-b")
-				util.MustCreate(ctx, k8sClient, testJobB)
+		testJobB := testingjob.MakeJob("job-b", ns.Name).
+			SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue).
+			Queue(kueue.LocalQueueName(localQueue.Name)).
+			Request(corev1.ResourceCPU, "1").
+			Parallelism(3).
+			Completions(3).
+			Obj()
 
-				ginkgo.By("the job-b remains suspended")
-				gomega.Eventually(func(g gomega.Gomega) {
-					g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(testJobB), testJobB)).Should(gomega.Succeed())
-					g.Expect(ptr.Deref(testJobB.Spec.Suspend, false)).Should(gomega.BeTrue())
-				}, util.Timeout, util.Interval).Should(gomega.Succeed())
+		ginkgo.By("creating a job-b")
+		util.MustCreate(ctx, k8sClient, testJobB)
 
-				ginkgo.By("job-b's workload remains pending with unreserved quota")
-				gomega.Eventually(func(g gomega.Gomega) {
-					workloads := &kueue.WorkloadList{}
-					g.Expect(k8sClient.List(ctx, workloads, client.InNamespace(testJobA.Namespace), client.MatchingLabels{constants.JobUIDLabel: string(testJobB.UID)})).Should(gomega.Succeed())
-					g.Expect(workloads.Items).Should(gomega.HaveLen(1))
-					testJobBWorkload = &workloads.Items[0]
-					util.ExpectWorkloadsToBePending(ctx, k8sClient, testJobBWorkload)
-				}, util.Timeout, util.Interval).Should(gomega.Succeed())
+		ginkgo.By("the job-b remains suspended")
+		gomega.Eventually(func(g gomega.Gomega) {
+			g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(testJobB), testJobB)).Should(gomega.Succeed())
+			g.Expect(ptr.Deref(testJobB.Spec.Suspend, false)).Should(gomega.BeTrue())
+		}, util.Timeout, util.Interval).Should(gomega.Succeed())
 
-				ginkgo.By("scale-down job-a to make room for job-b")
-				gomega.Eventually(func(g gomega.Gomega) {
-					testJobA.Spec.Parallelism = ptr.To(int32(1))
-					g.Expect(k8sClient.Update(ctx, testJobA)).Should(gomega.Succeed())
-				}, util.Timeout, util.Interval).Should(gomega.Succeed())
+		ginkgo.By("job-b's workload remains pending with unreserved quota")
+		gomega.Eventually(func(g gomega.Gomega) {
+			workloads := &kueue.WorkloadList{}
+			g.Expect(k8sClient.List(ctx, workloads, client.InNamespace(testJobA.Namespace), client.MatchingLabels{constants.JobUIDLabel: string(testJobB.UID)})).Should(gomega.Succeed())
+			g.Expect(workloads.Items).Should(gomega.HaveLen(1))
+			testJobBWorkload = &workloads.Items[0]
+			util.ExpectWorkloadsToBePending(ctx, k8sClient, testJobBWorkload)
+		}, util.Timeout, util.Interval).Should(gomega.Succeed())
 
-				ginkgo.By("admitting the job-b workload")
-				gomega.Eventually(func(g gomega.Gomega) {
-					g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(testJobBWorkload), testJobBWorkload)).Should(gomega.Succeed())
-					util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, testJobBWorkload)
-				}, util.Timeout, util.Interval).Should(gomega.Succeed())
+		ginkgo.By("scale-down job-a to make room for job-b")
+		gomega.Eventually(func(g gomega.Gomega) {
+			testJobA.Spec.Parallelism = ptr.To(int32(1))
+			g.Expect(k8sClient.Update(ctx, testJobA)).Should(gomega.Succeed())
+		}, util.Timeout, util.Interval).Should(gomega.Succeed())
 
-				ginkgo.By("the job-b is unsuspended")
-				gomega.Eventually(func(g gomega.Gomega) {
-					g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(testJobB), testJobB)).Should(gomega.Succeed())
-					g.Expect(ptr.Deref(testJobB.Spec.Suspend, false)).Should(gomega.BeFalse())
-				}, util.Timeout, util.Interval).Should(gomega.Succeed())
-			})
+		ginkgo.By("admitting the job-b workload")
+		gomega.Eventually(func(g gomega.Gomega) {
+			g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(testJobBWorkload), testJobBWorkload)).Should(gomega.Succeed())
+			util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, testJobBWorkload)
+		}, util.Timeout, util.Interval).Should(gomega.Succeed())
+
+		ginkgo.By("the job-b is unsuspended")
+		gomega.Eventually(func(g gomega.Gomega) {
+			g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(testJobB), testJobB)).Should(gomega.Succeed())
+			g.Expect(ptr.Deref(testJobB.Spec.Suspend, false)).Should(gomega.BeFalse())
+		}, util.Timeout, util.Interval).Should(gomega.Succeed())
+	})
+
+	ginkgo.It("Should mark old pending workload-slice evicted by scheduler as finished", func() {
+		features.SetFeatureGateDuringTest(ginkgo.GinkgoTB(), features.ElasticJobsViaWorkloadSlices, true)
+
+		ginkgo.By("create low priority class")
+		lowPriorityClass := testing.MakeWorkloadPriorityClass("low").PriorityValue(int32(priorityValue)).Obj()
+		util.MustCreate(ctx, k8sClient, lowPriorityClass)
+		ginkgo.DeferCleanup(func() {
+			gomega.Expect(k8sClient.Delete(ctx, lowPriorityClass)).To(gomega.Succeed())
 		})
+
+		ginkgo.By("create high priority class")
+		highPriorityClass := testing.MakeWorkloadPriorityClass("high").PriorityValue(int32(highPriorityValue)).Obj()
+		util.MustCreate(ctx, k8sClient, highPriorityClass)
+		ginkgo.DeferCleanup(func() {
+			gomega.Expect(k8sClient.Delete(ctx, highPriorityClass)).To(gomega.Succeed())
+		})
+
+		lowPriorityJob := testingjob.MakeJob("job-low", ns.Name).
+			SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue).
+			Queue(kueue.LocalQueueName(localQueue.Name)).
+			Request(corev1.ResourceCPU, "1000m").
+			Parallelism(3).
+			Completions(int32(cpuNominalQuota + 1)).
+			WorkloadPriorityClass(lowPriorityClass.Name).
+			Obj()
+
+		ginkgo.By("creating a low-priority job")
+		util.MustCreate(ctx, k8sClient, lowPriorityJob)
+
+		ginkgo.By("the low-priority job is unsuspended")
+		util.ExpectJobUnsuspendedWithNodeSelectors(ctx, k8sClient, client.ObjectKeyFromObject(lowPriorityJob), nil)
+		gomega.Eventually(func(g gomega.Gomega) {
+			g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lowPriorityJob), lowPriorityJob)).Should(gomega.Succeed())
+			g.Expect(ptr.Deref(lowPriorityJob.Spec.Suspend, false)).Should(gomega.BeFalse())
+		}, util.Timeout, util.Interval).Should(gomega.Succeed())
+
+		var lowPriorityWorkloadSlice *kueue.Workload
+		ginkgo.By("the low-priority workload is admitted")
+		workloads := util.ExpectWorkloadsInNamespace(ctx, k8sClient, lowPriorityJob.Namespace, 1)
+		lowPriorityWorkloadSlice = &workloads[0]
+		util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, lowPriorityWorkloadSlice)
+
+		ginkgo.By("scale-up low-priority job beyond the queue's nominal capacity")
+		gomega.Eventually(func(g gomega.Gomega) {
+			g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lowPriorityJob), lowPriorityJob)).Should(gomega.Succeed())
+			lowPriorityJob.Spec.Parallelism = ptr.To(int32(cpuNominalQuota + 1))
+			g.Expect(k8sClient.Update(ctx, lowPriorityJob)).Should(gomega.Succeed())
+		}, util.Timeout, util.Interval).Should(gomega.Succeed())
+
+		ginkgo.By("low priority new workload slice is pending")
+		lowPriorityNewWorkloadSlice := util.ExpectNewWorkloadSlice(ctx, k8sClient, lowPriorityWorkloadSlice)
+		gomega.Expect(lowPriorityNewWorkloadSlice).Should(gomega.Not(gomega.BeNil()))
+		util.ExpectWorkloadsToBePending(ctx, k8sClient, lowPriorityNewWorkloadSlice)
+
+		highPriorityJob := testingjob.MakeJob("high", ns.Name).
+			SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue).
+			Queue(kueue.LocalQueueName(localQueue.Name)).
+			Request(corev1.ResourceCPU, "1000m").
+			Parallelism(3).
+			Completions(3).
+			WorkloadPriorityClass(highPriorityClass.Name).
+			Obj()
+
+		ginkgo.By("creating a high priority job")
+		util.MustCreate(ctx, k8sClient, highPriorityJob)
+
+		ginkgo.By("the high priority job is unsuspended")
+		util.ExpectJobUnsuspendedWithNodeSelectors(ctx, k8sClient, client.ObjectKeyFromObject(highPriorityJob), nil)
+
+		ginkgo.By("the low priority old workload slice is finished")
+		util.ExpectWorkloadToFinish(ctx, k8sClient, client.ObjectKeyFromObject(lowPriorityWorkloadSlice))
 	})
 })
diff --git a/test/util/util.go b/test/util/util.go
index adf8fa11e..4cdc154a8 100644
--- a/test/util/util.go
+++ b/test/util/util.go
@@ -71,6 +71,7 @@ import (
 	utiltas "sigs.k8s.io/kueue/pkg/util/tas"
 	"sigs.k8s.io/kueue/pkg/util/testing"
 	"sigs.k8s.io/kueue/pkg/workload"
+	"sigs.k8s.io/kueue/pkg/workloadslicing"
 )
 
 const (
@@ -1121,3 +1122,51 @@ func ExpectLocalQueueFairSharingUsageToBe(ctx context.Context, k8sClient client.
 		g.Expect(usage.MilliValue()).To(gomega.BeNumerically(comparator, compareTo))
 	}, Timeout, Interval).Should(gomega.Succeed())
 }
+
+// ExpectWorkloadsInNamespace waits until the specified number of kueue.Workload
+// objects exist in the given namespace, then returns the list observed at that moment.
+//
+// It repeatedly lists Workloads using the provided client until the count of
+// items matches the expected value. The poll frequency and maximum wait time are
+// controlled by the test-scoped Interval and Timeout variables. If the expected
+// count is not reached before Timeout, the test fails.
+//
+// Returns:
+//
+//	The slice of Workloads present in the namespace when the expectation is met.
+func ExpectWorkloadsInNamespace(ctx context.Context, k8sClient client.Client, namespace string, count int) []kueue.Workload {
+	list := &kueue.WorkloadList{}
+	gomega.Eventually(func(g gomega.Gomega) {
+		g.Expect(k8sClient.List(ctx, list, client.InNamespace(namespace))).To(gomega.Succeed())
+		g.Expect(list.Items).Should(gomega.HaveLen(count))
+	}, Timeout, Interval).Should(gomega.Succeed())
+	return list.Items
+}
+
+// ExpectNewWorkloadSlice waits until a new kueue.Workload is created in the same
+// namespace as the given oldWorkload, and whose replacement annotation points to
+// the oldWorkload's key.
+//
+// This helper repeatedly lists Workloads in the oldWorkload's namespace until it
+// finds one where workloadslicing.ReplacementForKey matches the key of the given
+// oldWorkload. The search is retried until the test-scoped Timeout expires, polling
+// at the configured Interval. If no such workload is found within the Timeout,
+// the test fails.
+//
+// Returns:
+//   - newWorkload: A pointer to the discovered replacement Workload. Guaranteed
+//     non-nil if the function succeeds; otherwise, the test fails before returning.
+func ExpectNewWorkloadSlice(ctx context.Context, k8sClient client.Client, oldWorkload *kueue.Workload) (newWorkload *kueue.Workload) {
+	gomega.Eventually(func(g gomega.Gomega) {
+		wlList := &kueue.WorkloadList{}
+		g.Expect(k8sClient.List(ctx, wlList, client.InNamespace(oldWorkload.Namespace))).To(gomega.Succeed())
+		for i := range wlList.Items {
+			wl := &wlList.Items[i]
+			if key := workloadslicing.ReplacementForKey(wl); key != nil && *key == workload.Key(oldWorkload) {
+				newWorkload = wl
+				break
+			}
+		}
+	}, Timeout, Interval).Should(gomega.Succeed())
+	return newWorkload
+}