Repository: kubernetes-sigs/kueue

Issue:
Flaky Test: Race detection on MultiKueue AdmissionCheck controller
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!

If the matter is security related, please disclose it privately via https://kubernetes.io/security/
-->


**What happened**:

The CI detects race conditions in the following:

- `sigs.k8s.io/kueue/pkg/controller/admissionchecks/multikueue: TestListMultiKueueClustersUsingKubeConfig/multiple_clusters,_single_match`

```shell
{Failed  === RUN   TestListMultiKueueClustersUsingKubeConfig/multiple_clusters,_single_match
==================
WARNING: DATA RACE
Read at 0x00c000438043 by goroutine 203:
  testing.(*common).logDepth()
      /usr/local/go/src/testing/testing.go:1053 +0xca
  testing.(*common).log()
      /usr/local/go/src/testing/testing.go:1046 +0x7b
  testing.(*common).Log()
      /usr/local/go/src/testing/testing.go:1087 +0x53
  github.com/go-logr/logr/testr.logInfo()
      /home/prow/go/src/sigs.k8s.io/kueue/vendor/github.com/go-logr/logr/testr/testr.go:101 +0x193
  github.com/go-logr/logr/testr.testloggerInterface.Info()
      /home/prow/go/src/sigs.k8s.io/kueue/vendor/github.com/go-logr/logr/testr/testr.go:148 +0x13c
  github.com/go-logr/logr/testr.(*testloggerInterface).Info()
      <autogenerated>:1 +0x147
  github.com/go-logr/logr.Logger.Info()
      /home/prow/go/src/sigs.k8s.io/kueue/vendor/github.com/go-logr/logr/logr.go:280 +0x107
  sigs.k8s.io/kueue/pkg/controller/admissionchecks/multikueue.(*KubeConfigFSWatcher).Start.func1()
      /home/prow/go/src/sigs.k8s.io/kueue/pkg/controller/admissionchecks/multikueue/fswatch.go:92 +0x904

Previous write at 0x00c000438043 by goroutine 197:
  testing.tRunner.func1()
      /usr/local/go/src/testing/testing.go:1779 +0x8b9
  runtime.deferreturn()
      /usr/local/go/src/runtime/panic.go:610 +0x5d
  testing.(*T).Run.gowrap1()
      /usr/local/go/src/testing/testing.go:1851 +0x44

Goroutine 203 (running) created at:
  sigs.k8s.io/kueue/pkg/controller/admissionchecks/multikueue.(*KubeConfigFSWatcher).Start()
      /home/prow/go/src/sigs.k8s.io/kueue/pkg/controller/admissionchecks/multikueue/fswatch.go:70 +0x245
  sigs.k8s.io/kueue/pkg/controller/admissionchecks/multikueue.TestFSWatchAddRm.func2()
      /home/prow/go/src/sigs.k8s.io/kueue/pkg/controller/admissionchecks/multikueue/fswatch_test.go:227 +0x44
  sigs.k8s.io/kueue/pkg/controller/admissionchecks/multikueue.TestFSWatchAddRm.func13()
      /home/prow/go/src/sigs.k8s.io/kueue/pkg/controller/admissionchecks/multikueue/fswatch_test.go:452 +0x5a
  testing.tRunner()
      /usr/local/go/src/testing/testing.go:1792 +0x225
  testing.(*T).Run.gowrap1()
      /usr/local/go/src/testing/testing.go:1851 +0x44

Goroutine 197 (finished) created at:
  testing.(*T).Run()
      /usr/local/go/src/testing/testing.go:1851 +0x8f2
  testing.runTests.func1()
      /usr/local/go/src/testing/testing.go:2279 +0x85
  testing.tRunner()
      /usr/local/go/src/testing/testing.go:1792 +0x225
  testing.runTests()
      /usr/local/go/src/testing/testing.go:2277 +0x96c
  testing.(*M).Run()
      /usr/local/go/src/testing/testing.go:2142 +0xeea
  main.main()
      _testmain.go:71 +0x164
==================
    testing.go:1490: race detected during execution of test
--- FAIL: TestListMultiKueueClustersUsingKubeConfig/multiple_clusters,_single_match (0.04s)
}
```

- `sigs.k8s.io/kueue/pkg/controller/admissionchecks/multikueue: TestListMultiKueueClustersUsingKubeConfig`

```shell
{Failed  === RUN   TestListMultiKueueClustersUsingKubeConfig
--- FAIL: TestListMultiKueueClustersUsingKubeConfig (0.13s)
}
```

**What you expected to happen**:

No race condition

**How to reproduce it (as minimally and precisely as possible)**:

https://prow.k8s.io/view/gs/kubernetes-ci-logs/pr-logs/pull/kubernetes-sigs_kueue/6673/pull-kueue-test-unit-main/1960302981727391744

**Anything else we need to know?**:

**Environment**:
- Kubernetes version (use `kubectl version`):
- Kueue version (use `git describe --tags --dirty --always`):
- Cloud provider or hardware configuration:
- OS (e.g: `cat /etc/os-release`):
- Kernel (e.g. `uname -a`):
- Install tools:
- Others:

Discussion:
@tenzen-y: /kind flake@tenzen-y: Can anyone take a look at this one? It seems that this occurs frequently.
@tenzen-y: We observed this issue in the periodic test as well: https://prow.k8s.io/view/gs/kubernetes-ci-logs/logs/periodic-kueue-test-unit-main/1960409512091324416
@mbobrovskyi: /assign

Solution patch:
diff --git a/pkg/controller/core/workload_controller.go b/pkg/controller/core/workload_controller.go
index b8bc0afcc..8917a3f41 100644
--- a/pkg/controller/core/workload_controller.go
+++ b/pkg/controller/core/workload_controller.go
@@ -233,12 +233,13 @@ func (r *WorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 		}
 	} else {
 		var updated, evicted bool
+		var underlyingCause string
 		reason := kueue.WorkloadDeactivated
 		message := "The workload is deactivated"
 		dtCond := apimeta.FindStatusCondition(wl.Status.Conditions, kueue.WorkloadDeactivationTarget)
 		if !apimeta.IsStatusConditionTrue(wl.Status.Conditions, kueue.WorkloadEvicted) {
 			if dtCond != nil {
-				reason = fmt.Sprintf("%sDueTo%s", reason, dtCond.Reason)
+				underlyingCause = dtCond.Reason
 				message = fmt.Sprintf("%s due to %s", message, dtCond.Message)
 			}
 			updated = true
@@ -253,7 +254,7 @@ func (r *WorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 		}
 		if updated {
 			if evicted {
-				if err := workload.Evict(ctx, r.client, r.recorder, &wl, reason, "", message, r.clock); err != nil {
+				if err := workload.Evict(ctx, r.client, r.recorder, &wl, reason, underlyingCause, message, r.clock); err != nil {
 					if !apierrors.IsNotFound(err) {
 						return ctrl.Result{}, fmt.Errorf("setting eviction: %w", err)
 					}
diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go
index 14617d093..507d167bd 100644
--- a/pkg/metrics/metrics.go
+++ b/pkg/metrics/metrics.go
@@ -167,11 +167,13 @@ The label 'reason' can have the following values:
 - "LocalQueueStopped" means that the workload was evicted because the LocalQueue is stopped.
 - "NodeFailures" means that the workload was evicted due to node failures when using TopologyAwareScheduling.
 - "Deactivated" means that the workload was evicted because spec.active is set to false.
-- "DeactivatedDueToAdmissionCheck" means that the workload was evicted and deactivated by Kueue due to a rejected admission check.
-- "DeactivatedDueToMaximumExecutionTimeExceeded" means that the workload was evicted and deactivated by Kueue due to maximum execution time exceeded.
-- "DeactivatedDueToRequeuingLimitExceeded" means that the workload was evicted and deactivated by Kueue due to requeuing limit exceeded.`,
+The label 'underlying_cause' can have the following values:
+- "" means that the value in 'reason' label is the root cause for eviction.
+- "AdmissionCheck" means that the workload was evicted by Kueue due to a rejected admission check.
+- "MaximumExecutionTimeExceeded" means that the workload was evicted by Kueue due to maximum execution time exceeded.
+- "RequeuingLimitExceeded" means that the workload was evicted by Kueue due to requeuing limit exceeded.`,
 			Buckets: generateExponentialBuckets(14),
-		}, []string{"cluster_queue", "reason"},
+		}, []string{"cluster_queue", "reason", "underlying_cause"},
 	)
 
 	localQueueQuotaReservedWaitTime = prometheus.NewHistogramVec(
@@ -284,10 +286,12 @@ The label 'reason' can have the following values:
 - "LocalQueueStopped" means that the workload was evicted because the LocalQueue is stopped.
 - "NodeFailures" means that the workload was evicted due to node failures when using TopologyAwareScheduling.
 - "Deactivated" means that the workload was evicted because spec.active is set to false.
-- "DeactivatedDueToAdmissionCheck" means that the workload was evicted and deactivated by Kueue due to a rejected admission check.
-- "DeactivatedDueToMaximumExecutionTimeExceeded" means that the workload was evicted and deactivated by Kueue due to maximum execution time exceeded.
-- "DeactivatedDueToRequeuingLimitExceeded" means that the workload was evicted and deactivated by Kueue due to requeuing limit exceeded.`,
-		}, []string{"cluster_queue", "reason"},
+The label 'underlying_cause' can have the following values:
+- "" means that the value in 'reason' label is the root cause for eviction.
+- "AdmissionCheck" means that the workload was evicted by Kueue due to a rejected admission check.
+- "MaximumExecutionTimeExceeded" means that the workload was evicted by Kueue due to maximum execution time exceeded.
+- "RequeuingLimitExceeded" means that the workload was evicted by Kueue due to requeuing limit exceeded.`,
+		}, []string{"cluster_queue", "reason", "underlying_cause"},
 	)
 
 	ReplacedWorkloadSlicesTotal = prometheus.NewCounterVec(
@@ -311,10 +315,12 @@ The label 'reason' can have the following values:
 - "LocalQueueStopped" means that the workload was evicted because the LocalQueue is stopped.
 - "NodeFailures" means that the workload was evicted due to node failures when using TopologyAwareScheduling.
 - "Deactivated" means that the workload was evicted because spec.active is set to false.
-- "DeactivatedDueToAdmissionCheck" means that the workload was evicted and deactivated by Kueue due to a rejected admission check.
-- "DeactivatedDueToMaximumExecutionTimeExceeded" means that the workload was evicted and deactivated by Kueue due to maximum execution time exceeded.
-- "DeactivatedDueToRequeuingLimitExceeded" means that the workload was evicted and deactivated by Kueue due to requeuing limit exceeded.`,
-		}, []string{"name", "namespace", "reason"},
+The label 'underlying_cause' can have the following values:
+- "" means that the value in 'reason' label is the root cause for eviction.
+- "AdmissionCheck" means that the workload was evicted by Kueue due to a rejected admission check.
+- "MaximumExecutionTimeExceeded" means that the workload was evicted by Kueue due to maximum execution time exceeded.
+- "RequeuingLimitExceeded" means that the workload was evicted by Kueue due to requeuing limit exceeded.`,
+		}, []string{"name", "namespace", "reason", "underlying_cause"},
 	)
 
 	EvictedWorkloadsOnceTotal = prometheus.NewCounterVec(
@@ -330,9 +336,13 @@ The label 'reason' can have the following values:
 - "LocalQueueStopped" means that the workload was evicted because the LocalQueue is stopped.
 - "NodeFailures" means that the workload was evicted due to node failures when using TopologyAwareScheduling.
 - "Deactivated" means that the workload was evicted because spec.active is set to false.
-- "DeactivatedDueToAdmissionCheck" means that the workload was evicted and deactivated by Kueue due to a rejected admission check.
-- "DeactivatedDueToMaximumExecutionTimeExceeded" means that the workload was evicted and deactivated by Kueue due to maximum execution time exceeded.
-- "DeactivatedDueToRequeuingLimitExceeded" means that the workload was evicted and deactivated by Kueue due to requeuing limit exceeded.`,
+The label 'detailed_reason' can have the following values:
+- "" means that the value in 'reason' label is the root cause for eviction.
+- "WaitForStart" means that the pods have not been ready since admission, or the workload is not admitted.
+- "WaitForRecovery" means that the Pods were ready since the workload admission, but some pod has failed.
+- "AdmissionCheck" means that the workload was evicted by Kueue due to a rejected admission check.
+- "MaximumExecutionTimeExceeded" means that the workload was evicted by Kueue due to maximum execution time exceeded.
+- "RequeuingLimitExceeded" means that the workload was evicted by Kueue due to requeuing limit exceeded.`,
 		}, []string{"cluster_queue", "reason", "detailed_reason"},
 	)
 
@@ -554,16 +564,16 @@ func ReportLocalQueuePendingWorkloads(lq LocalQueueReference, active, inadmissib
 	LocalQueuePendingWorkloads.WithLabelValues(string(lq.Name), lq.Namespace, PendingStatusInadmissible).Set(float64(inadmissible))
 }
 
-func ReportEvictedWorkloads(cqName kueue.ClusterQueueReference, evictionReason string) {
-	EvictedWorkloadsTotal.WithLabelValues(string(cqName), evictionReason).Inc()
+func ReportEvictedWorkloads(cqName kueue.ClusterQueueReference, evictionReason, underlyingCause string) {
+	EvictedWorkloadsTotal.WithLabelValues(string(cqName), evictionReason, underlyingCause).Inc()
 }
 
 func ReportReplacedWorkloadSlices(cqName kueue.ClusterQueueReference) {
 	ReplacedWorkloadSlicesTotal.WithLabelValues(string(cqName)).Inc()
 }
 
-func ReportLocalQueueEvictedWorkloads(lq LocalQueueReference, reason string) {
-	LocalQueueEvictedWorkloadsTotal.WithLabelValues(string(lq.Name), lq.Namespace, reason).Inc()
+func ReportLocalQueueEvictedWorkloads(lq LocalQueueReference, reason, underlyingCause string) {
+	LocalQueueEvictedWorkloadsTotal.WithLabelValues(string(lq.Name), lq.Namespace, reason, underlyingCause).Inc()
 }
 
 func ReportEvictedWorkloadsOnce(cqName kueue.ClusterQueueReference, reason, underlyingCause string) {
diff --git a/pkg/workload/workload.go b/pkg/workload/workload.go
index 893c90020..a10c58369 100644
--- a/pkg/workload/workload.go
+++ b/pkg/workload/workload.go
@@ -1082,7 +1082,11 @@ func AdmissionChecksForWorkload(log logr.Logger, wl *kueue.Workload, admissionCh
 }
 
 func Evict(ctx context.Context, c client.Client, recorder record.EventRecorder, wl *kueue.Workload, reason, underlyingCause, msg string, clock clock.Clock) error {
-	prepareForEviction(wl, clock.Now(), reason, msg)
+	evictionReason := reason
+	if reason == kueue.WorkloadDeactivated && underlyingCause != "" {
+		evictionReason = fmt.Sprintf("%sDueTo%s", evictionReason, underlyingCause)
+	}
+	prepareForEviction(wl, clock.Now(), evictionReason, msg)
 	reportWorkloadEvictedOnce := workloadEvictionStateInc(wl, reason, underlyingCause)
 	if err := ApplyAdmissionStatus(ctx, c, wl, true, clock); err != nil {
 		return err
@@ -1095,7 +1099,7 @@ func Evict(ctx context.Context, c client.Client, recorder record.EventRecorder,
 		log.V(3).Info("WARNING: unexpected eviction of workload without status.Admission", "workload", klog.KObj(wl))
 		return nil
 	}
-	reportEvictedWorkload(recorder, wl, wl.Status.Admission.ClusterQueue, reason, msg)
+	reportEvictedWorkload(recorder, wl, wl.Status.Admission.ClusterQueue, reason, underlyingCause, msg)
 	if reportWorkloadEvictedOnce {
 		metrics.ReportEvictedWorkloadsOnce(wl.Status.Admission.ClusterQueue, reason, underlyingCause)
 	}
@@ -1113,15 +1117,19 @@ func resetClusterNomination(w *kueue.Workload) {
 	w.Status.NominatedClusterNames = nil
 }
 
-func reportEvictedWorkload(recorder record.EventRecorder, wl *kueue.Workload, cqName kueue.ClusterQueueReference, reason, message string) {
-	metrics.ReportEvictedWorkloads(cqName, reason)
+func reportEvictedWorkload(recorder record.EventRecorder, wl *kueue.Workload, cqName kueue.ClusterQueueReference, reason, underlyingCause, message string) {
+	metrics.ReportEvictedWorkloads(cqName, reason, underlyingCause)
 	if podsReadyToEvictionTime := workloadsWithPodsReadyToEvictedTime(wl); podsReadyToEvictionTime != nil {
-		metrics.PodsReadyToEvictedTimeSeconds.WithLabelValues(string(cqName), reason).Observe(podsReadyToEvictionTime.Seconds())
+		metrics.PodsReadyToEvictedTimeSeconds.WithLabelValues(string(cqName), reason, underlyingCause).Observe(podsReadyToEvictionTime.Seconds())
 	}
 	if features.Enabled(features.LocalQueueMetrics) {
-		metrics.ReportLocalQueueEvictedWorkloads(metrics.LQRefFromWorkload(wl), reason)
+		metrics.ReportLocalQueueEvictedWorkloads(metrics.LQRefFromWorkload(wl), reason, underlyingCause)
+	}
+	eventReason := fmt.Sprintf("%sDueTo%s", kueue.WorkloadEvicted, reason)
+	if reason == kueue.WorkloadDeactivated && underlyingCause != "" {
+		eventReason = fmt.Sprintf("%sDueTo%s", eventReason, underlyingCause)
 	}
-	recorder.Event(wl, corev1.EventTypeNormal, fmt.Sprintf("%sDueTo%s", kueue.WorkloadEvicted, reason), message)
+	recorder.Event(wl, corev1.EventTypeNormal, eventReason, message)
 }
 
 func ReportPreemption(preemptingCqName kueue.ClusterQueueReference, preemptingReason string, targetCqName kueue.ClusterQueueReference) {



Test patch:
diff --git a/pkg/controller/core/workload_controller_test.go b/pkg/controller/core/workload_controller_test.go
index b69709c23..f7d546e23 100644
--- a/pkg/controller/core/workload_controller_test.go
+++ b/pkg/controller/core/workload_controller_test.go
@@ -677,8 +677,9 @@ func TestReconcile(t *testing.T) {
 				).
 				SchedulingStatsEviction(
 					kueue.WorkloadSchedulingStatsEviction{
-						Reason: "DeactivatedDueToAdmissionCheck",
-						Count:  1,
+						Reason:          "Deactivated",
+						UnderlyingCause: "AdmissionCheck",
+						Count:           1,
 					},
 				).
 				Obj(),
@@ -1344,7 +1345,11 @@ func TestReconcile(t *testing.T) {
 					Message: "exceeding the maximum number of re-queuing retries",
 				}).
 				SchedulingStatsEviction(
-					kueue.WorkloadSchedulingStatsEviction{Reason: "DeactivatedDueToRequeuingLimitExceeded", Count: 1},
+					kueue.WorkloadSchedulingStatsEviction{
+						Reason:          "Deactivated",
+						UnderlyingCause: "RequeuingLimitExceeded",
+						Count:           1,
+					},
 				).
 				Obj(),
 			wantEvents: []utiltesting.EventRecord{
@@ -1408,8 +1413,9 @@ func TestReconcile(t *testing.T) {
 				}).
 				SchedulingStatsEviction(
 					kueue.WorkloadSchedulingStatsEviction{
-						Reason: "DeactivatedDueToRequeuingLimitExceeded",
-						Count:  1,
+						Reason:          "Deactivated",
+						UnderlyingCause: "RequeuingLimitExceeded",
+						Count:           1,
 					},
 				).
 				Obj(),
@@ -1477,8 +1483,9 @@ func TestReconcile(t *testing.T) {
 				RequeueState(ptr.To[int32](100), nil).
 				SchedulingStatsEviction(
 					kueue.WorkloadSchedulingStatsEviction{
-						Reason: "DeactivatedDueToRequeuingLimitExceeded",
-						Count:  1,
+						Reason:          "Deactivated",
+						UnderlyingCause: "RequeuingLimitExceeded",
+						Count:           1,
 					},
 				).
 				Obj(),
@@ -1546,8 +1553,9 @@ func TestReconcile(t *testing.T) {
 				RequeueState(ptr.To[int32](100), nil).
 				SchedulingStatsEviction(
 					kueue.WorkloadSchedulingStatsEviction{
-						Reason: "DeactivatedDueToRequeuingLimitExceeded",
-						Count:  1,
+						Reason:          "Deactivated",
+						UnderlyingCause: "RequeuingLimitExceeded",
+						Count:           1,
 					},
 				).
 				Obj(),
diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go
index 8cb5470e7..15a8ce3f7 100644
--- a/pkg/metrics/metrics_test.go
+++ b/pkg/metrics/metrics_test.go
@@ -146,8 +146,8 @@ func TestReportAndCleanupClusterQueueUsage(t *testing.T) {
 }
 
 func TestReportAndCleanupClusterQueueEvictedNumber(t *testing.T) {
-	ReportEvictedWorkloads("cluster_queue1", "Preempted")
-	ReportEvictedWorkloads("cluster_queue1", "Evicted")
+	ReportEvictedWorkloads("cluster_queue1", "Preempted", "")
+	ReportEvictedWorkloads("cluster_queue1", "Evicted", "")
 
 	expectFilteredMetricsCount(t, EvictedWorkloadsTotal, 2, "cluster_queue", "cluster_queue1")
 	expectFilteredMetricsCount(t, EvictedWorkloadsTotal, 1, "cluster_queue", "cluster_queue1", "reason", "Preempted")
diff --git a/test/integration/singlecluster/controller/admissionchecks/provisioning/provisioning_test.go b/test/integration/singlecluster/controller/admissionchecks/provisioning/provisioning_test.go
index fcb9e7900..ee9b3aa73 100644
--- a/test/integration/singlecluster/controller/admissionchecks/provisioning/provisioning_test.go
+++ b/test/integration/singlecluster/controller/admissionchecks/provisioning/provisioning_test.go
@@ -359,7 +359,7 @@ var _ = ginkgo.Describe("Provisioning", ginkgo.Ordered, ginkgo.ContinueOnFailure
 				gomega.Eventually(func(g gomega.Gomega) {
 					g.Expect(k8sClient.Get(ctx, wlKey, &updatedWl)).To(gomega.Succeed())
 					g.Expect(workload.IsEvictedByDeactivation(&updatedWl)).To(gomega.BeTrue())
-					util.ExpectEvictedWorkloadsTotalMetric(cq.Name, "DeactivatedDueToAdmissionCheck", 1)
+					util.ExpectEvictedWorkloadsTotalMetric(cq.Name, "Deactivated", "AdmissionCheck", 1)
 				}, util.Timeout, util.Interval).Should(gomega.Succeed())
 			})
 		})
@@ -425,7 +425,7 @@ var _ = ginkgo.Describe("Provisioning", ginkgo.Ordered, ginkgo.ContinueOnFailure
 					g.Expect(workload.IsEvictedByDeactivation(&updatedWl)).To(gomega.BeTrue())
 				}, util.Timeout, util.Interval).Should(gomega.Succeed())
 
-				util.ExpectEvictedWorkloadsTotalMetric(cq.Name, "DeactivatedDueToAdmissionCheck", 1)
+				util.ExpectEvictedWorkloadsTotalMetric(cq.Name, "Deactivated", "AdmissionCheck", 1)
 			})
 		})
 
@@ -477,7 +477,7 @@ var _ = ginkgo.Describe("Provisioning", ginkgo.Ordered, ginkgo.ContinueOnFailure
 					g.Expect(workload.IsEvictedByDeactivation(&updatedWl)).To(gomega.BeTrue())
 				}, util.Timeout, util.Interval).Should(gomega.Succeed())
 
-				util.ExpectEvictedWorkloadsTotalMetric(cq.Name, "DeactivatedDueToAdmissionCheck", 1)
+				util.ExpectEvictedWorkloadsTotalMetric(cq.Name, "Deactivated", "AdmissionCheck", 1)
 			})
 		})
 
@@ -549,7 +549,7 @@ var _ = ginkgo.Describe("Provisioning", ginkgo.Ordered, ginkgo.ContinueOnFailure
 					g.Expect(workload.IsEvictedByDeactivation(&updatedWl)).To(gomega.BeFalse())
 				}, util.Timeout, util.Interval).Should(gomega.Succeed())
 
-				util.ExpectEvictedWorkloadsTotalMetric(cq.Name, kueue.WorkloadDeactivated, 0)
+				util.ExpectEvictedWorkloadsTotalMetric(cq.Name, kueue.WorkloadDeactivated, "", 0)
 			})
 		})
 
@@ -1130,7 +1130,7 @@ var _ = ginkgo.Describe("Provisioning", ginkgo.Ordered, ginkgo.ContinueOnFailure
 					g.Expect(k8sClient.Get(ctx, wlKey, &updatedWl)).To(gomega.Succeed())
 
 					g.Expect(workload.IsEvictedByDeactivation(&updatedWl)).To(gomega.BeTrue())
-					util.ExpectEvictedWorkloadsTotalMetric(cq.Name, "DeactivatedDueToAdmissionCheck", 1)
+					util.ExpectEvictedWorkloadsTotalMetric(cq.Name, "Deactivated", "AdmissionCheck", 1)
 				}, util.Timeout, util.Interval).Should(gomega.Succeed())
 			})
 		})
diff --git a/test/integration/singlecluster/controller/core/workload_controller_test.go b/test/integration/singlecluster/controller/core/workload_controller_test.go
index 47e01b4c6..ee1a9ddce 100644
--- a/test/integration/singlecluster/controller/core/workload_controller_test.go
+++ b/test/integration/singlecluster/controller/core/workload_controller_test.go
@@ -310,8 +310,8 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn
 				gomega.Eventually(func(g gomega.Gomega) {
 					g.Expect(k8sClient.Get(ctx, wlKey, updatedWl)).To(gomega.Succeed())
 					g.Expect(workload.IsEvictedByDeactivation(updatedWl)).To(gomega.BeTrue())
-					util.ExpectEvictedWorkloadsTotalMetric(clusterQueue.Name, "DeactivatedDueToAdmissionCheck", 1)
-					util.ExpectEvictedWorkloadsOnceTotalMetric(clusterQueue.Name, "DeactivatedDueToAdmissionCheck", "", 1)
+					util.ExpectEvictedWorkloadsTotalMetric(clusterQueue.Name, "Deactivated", "AdmissionCheck", 1)
+					util.ExpectEvictedWorkloadsOnceTotalMetric(clusterQueue.Name, "Deactivated", "AdmissionCheck", 1)
 				}, util.Timeout, util.Interval).Should(gomega.Succeed())
 			})
 		})
@@ -395,8 +395,8 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn
 					g.Expect(k8sClient.Get(ctx, wlKey, updatedWl)).To(gomega.Succeed())
 
 					g.Expect(workload.IsEvictedByDeactivation(updatedWl)).To(gomega.BeTrue())
-					util.ExpectEvictedWorkloadsTotalMetric(clusterQueue.Name, "DeactivatedDueToAdmissionCheck", 1)
-					util.ExpectEvictedWorkloadsOnceTotalMetric(clusterQueue.Name, "DeactivatedDueToAdmissionCheck", "", 1)
+					util.ExpectEvictedWorkloadsTotalMetric(clusterQueue.Name, "Deactivated", "AdmissionCheck", 1)
+					util.ExpectEvictedWorkloadsOnceTotalMetric(clusterQueue.Name, "Deactivated", "AdmissionCheck", 1)
 				}, util.Timeout, util.Interval).Should(gomega.Succeed())
 			})
 		})
diff --git a/test/integration/singlecluster/scheduler/podsready/scheduler_test.go b/test/integration/singlecluster/scheduler/podsready/scheduler_test.go
index b4532b7b9..4fb160a3a 100644
--- a/test/integration/singlecluster/scheduler/podsready/scheduler_test.go
+++ b/test/integration/singlecluster/scheduler/podsready/scheduler_test.go
@@ -196,7 +196,7 @@ var _ = ginkgo.Describe("SchedulerWithWaitForPodsReady", func() {
 			util.FinishEvictionForWorkloads(ctx, k8sClient, prodWl)
 
 			ginkgo.By("check for PodsReadyToEvictedTimeSeconds metric existence")
-			util.ExpectPodsReadyToEvictedTimeSeconds(prodClusterQ.Name, kueue.WorkloadDeactivated, 1)
+			util.ExpectPodsReadyToEvictedTimeSeconds(prodClusterQ.Name, kueue.WorkloadDeactivated, "", 1)
 		})
 
 		ginkgo.It("Should unblock admission of new workloads once the admitted workload is deleted", func() {
@@ -283,7 +283,7 @@ var _ = ginkgo.Describe("SchedulerWithWaitForPodsReady", func() {
 			}, util.Timeout, util.Interval).Should(gomega.Succeed(), "the workload should be evicted after the timeout expires")
 
 			util.FinishEvictionForWorkloads(ctx, k8sClient, prodWl1)
-			util.ExpectEvictedWorkloadsTotalMetric(prodClusterQ.Name, kueue.WorkloadEvictedByPodsReadyTimeout, 1)
+			util.ExpectEvictedWorkloadsTotalMetric(prodClusterQ.Name, kueue.WorkloadEvictedByPodsReadyTimeout, kueue.WorkloadWaitForStart, 1)
 			util.ExpectEvictedWorkloadsOnceTotalMetric(prodClusterQ.Name, kueue.WorkloadEvictedByPodsReadyTimeout, kueue.WorkloadWaitForStart, 1)
 
 			ginkgo.By("verify the 'prod2' workload gets admitted and the 'prod1' is pending by backoff")
@@ -335,7 +335,7 @@ var _ = ginkgo.Describe("SchedulerWithWaitForPodsReady", func() {
 			}, util.Timeout, util.Interval).Should(gomega.Succeed())
 			util.FinishEvictionForWorkloads(ctx, k8sClient, prodWl)
 			// should observe a metrics of WorkloadEvictedByDeactivation
-			util.ExpectEvictedWorkloadsTotalMetric(prodClusterQ.Name, "DeactivatedDueToRequeuingLimitExceeded", 1)
+			util.ExpectEvictedWorkloadsTotalMetric(prodClusterQ.Name, "Deactivated", "RequeuingLimitExceeded", 1)
 			util.ExpectEvictedWorkloadsOnceTotalMetric(prodClusterQ.Name, kueue.WorkloadEvictedByPodsReadyTimeout, kueue.WorkloadWaitForStart, 1)
 
 			ginkgo.By("the reactivated workload should not be deactivated by the scheduler unless exceeding the backoffLimitCount")
diff --git a/test/integration/singlecluster/scheduler/preemption_test.go b/test/integration/singlecluster/scheduler/preemption_test.go
index 0ca164387..28b565e8b 100644
--- a/test/integration/singlecluster/scheduler/preemption_test.go
+++ b/test/integration/singlecluster/scheduler/preemption_test.go
@@ -132,7 +132,7 @@ var _ = ginkgo.Describe("Preemption", func() {
 			util.MustCreate(ctx, k8sClient, highWl2)
 
 			util.FinishEvictionForWorkloads(ctx, k8sClient, lowWl1, lowWl2)
-			util.ExpectEvictedWorkloadsTotalMetric(cq.Name, kueue.WorkloadEvictedByPreemption, 2)
+			util.ExpectEvictedWorkloadsTotalMetric(cq.Name, kueue.WorkloadEvictedByPreemption, "", 2)
 			util.ExpectPreemptedWorkloadsTotalMetric(cq.Name, kueue.InClusterQueueReason, 2)
 			util.ExpectEvictedWorkloadsOnceTotalMetric(cq.Name, kueue.WorkloadEvictedByPreemption, "", 2)
 
diff --git a/test/integration/singlecluster/scheduler/scheduler_test.go b/test/integration/singlecluster/scheduler/scheduler_test.go
index 2563adeb2..337f540a1 100644
--- a/test/integration/singlecluster/scheduler/scheduler_test.go
+++ b/test/integration/singlecluster/scheduler/scheduler_test.go
@@ -1966,7 +1966,7 @@ var _ = ginkgo.Describe("Scheduler", func() {
 			}, util.Timeout, util.Interval).Should(gomega.Succeed())
 
 			util.ExpectClusterQueueStatusMetric(cq, metrics.CQStatusPending)
-			util.ExpectEvictedWorkloadsTotalMetric(clusterQueue.Name, kueue.WorkloadEvictedByClusterQueueStopped, 1)
+			util.ExpectEvictedWorkloadsTotalMetric(clusterQueue.Name, kueue.WorkloadEvictedByClusterQueueStopped, "", 1)
 			util.ExpectEvictedWorkloadsOnceTotalMetric(cq.Name, kueue.WorkloadEvictedByClusterQueueStopped, "", 1)
 
 			ginkgo.By("Checking the condition of workload is evicted", func() {
@@ -2066,7 +2066,7 @@ var _ = ginkgo.Describe("Scheduler", func() {
 				g.Expect(k8sClient.Update(ctx, createdLq)).Should(gomega.Succeed())
 			}, util.Timeout, util.Interval).Should(gomega.Succeed())
 
-			util.ExpectEvictedWorkloadsTotalMetric(cq.Name, kueue.WorkloadEvictedByLocalQueueStopped, 1)
+			util.ExpectEvictedWorkloadsTotalMetric(cq.Name, kueue.WorkloadEvictedByLocalQueueStopped, "", 1)
 			util.ExpectEvictedWorkloadsOnceTotalMetric(cq.Name, kueue.WorkloadEvictedByLocalQueueStopped, "", 1)
 
 			createdWl := kueue.Workload{}
diff --git a/test/integration/singlecluster/tas/tas_test.go b/test/integration/singlecluster/tas/tas_test.go
index 5fe44f97e..e9e89708d 100644
--- a/test/integration/singlecluster/tas/tas_test.go
+++ b/test/integration/singlecluster/tas/tas_test.go
@@ -1485,7 +1485,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() {
 
 				ginkgo.By("verify the workload is evicted due to no replacement possible", func() {
 					util.FinishEvictionForWorkloads(ctx, k8sClient, wl1)
-					util.ExpectEvictedWorkloadsTotalMetric(clusterQueue.Name, kueue.WorkloadEvictedDueToNodeFailures, 1)
+					util.ExpectEvictedWorkloadsTotalMetric(clusterQueue.Name, kueue.WorkloadEvictedDueToNodeFailures, "", 1)
 					gomega.Eventually(func(g gomega.Gomega) {
 						updatedWl := &kueue.Workload{}
 						g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl1), updatedWl)).To(gomega.Succeed())
diff --git a/test/util/util.go b/test/util/util.go
index 4cdc154a8..70c922479 100644
--- a/test/util/util.go
+++ b/test/util/util.go
@@ -566,14 +566,14 @@ func ExpectAdmittedWorkloadsTotalMetric(cq *kueue.ClusterQueue, v int) {
 	expectCounterMetric(metric, v)
 }
 
-func ExpectEvictedWorkloadsTotalMetric(cqName, reason string, v int) {
-	metric := metrics.EvictedWorkloadsTotal.WithLabelValues(cqName, reason)
+func ExpectEvictedWorkloadsTotalMetric(cqName, reason, underlyingCause string, v int) {
+	metric := metrics.EvictedWorkloadsTotal.WithLabelValues(cqName, reason, underlyingCause)
 	expectCounterMetric(metric, v)
 }
 
-func ExpectPodsReadyToEvictedTimeSeconds(cqName, reason string, v int) {
+func ExpectPodsReadyToEvictedTimeSeconds(cqName, reason, underlyingCause string, v int) {
 	metric := metrics.PodsReadyToEvictedTimeSeconds
-	expectHistogramMetric(metric, cqName, reason, v)
+	expectHistogramMetric(metric, cqName, reason, underlyingCause, v)
 }
 
 func ExpectEvictedWorkloadsOnceTotalMetric(cqName string, reason, underlyingCause string, v int) {
@@ -599,9 +599,9 @@ func expectCounterMetric(metric prometheus.Counter, count int) {
 	}, Timeout, Interval).Should(gomega.Succeed())
 }
 
-func expectHistogramMetric(metric *prometheus.HistogramVec, cqName, preemptionReason string, count int) {
+func expectHistogramMetric(metric *prometheus.HistogramVec, cqName, preemptionReason, underlyingCause string, count int) {
 	gomega.EventuallyWithOffset(2, func(g gomega.Gomega) {
-		v, err := testutil.GetHistogramMetricCount(metric.WithLabelValues(cqName, preemptionReason))
+		v, err := testutil.GetHistogramMetricCount(metric.WithLabelValues(cqName, preemptionReason, underlyingCause))
 		g.Expect(err).ToNot(gomega.HaveOccurred())
 		g.Expect(int(v)).Should(gomega.Equal(count))
 	}, Timeout, Interval).Should(gomega.Succeed())