Repository: kubernetes-sigs/kueue

Issue:
Commonize the code for workload eviction
<!-- Please only use this template for submitting clean up requests -->

**What would you like to be cleaned**:

Commonize workload eviction by using the common workload.EvictWorkload function (or just workload.Evict).

**Why is this needed**:

Currently many different places handles workload eviction in slightly different ways, we should commonize to make bugs less likely.

Discussion:
@mimowo: /assign 
tentatively

Solution patch:
diff --git a/pkg/controller/core/workload_controller.go b/pkg/controller/core/workload_controller.go
index 639cbc443..d7ab5f4bd 100644
--- a/pkg/controller/core/workload_controller.go
+++ b/pkg/controller/core/workload_controller.go
@@ -255,7 +255,7 @@ func (r *WorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 		// exception due to complicated logic, not using PrepareForEviction (temporarily)
 		workload.ResetClusterNomination(&wl)
 		updated = workload.ResetChecksOnEviction(&wl, r.clock.Now()) || updated
-		reportWorkloadEvictedOnce := workload.WorkloadEvictionStateInc(&wl, kueue.WorkloadDeactivated, "")
+		reportWorkloadEvictedOnce := workload.WorkloadEvictionStateInc(&wl, reason, "")
 		if updated {
 			if err := workload.ApplyAdmissionStatus(ctx, r.client, &wl, true, r.clock); err != nil {
 				if apierrors.IsNotFound(err) {
@@ -266,7 +266,7 @@ func (r *WorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 			if evicted && wl.Status.Admission != nil {
 				workload.ReportEvictedWorkload(r.recorder, &wl, wl.Status.Admission.ClusterQueue, reason, message)
 				if reportWorkloadEvictedOnce {
-					metrics.ReportEvictedWorkloadsOnce(wl.Status.Admission.ClusterQueue, kueue.WorkloadDeactivated, "")
+					metrics.ReportEvictedWorkloadsOnce(wl.Status.Admission.ClusterQueue, reason, "")
 				}
 			}
 			return ctrl.Result{}, nil
@@ -488,12 +488,16 @@ func (r *WorkloadReconciler) reconcileOnLocalQueueActiveState(ctx context.Contex
 		}
 		log.V(3).Info("Workload is evicted because the LocalQueue is stopped", "localQueue", klog.KRef(wl.Namespace, string(wl.Spec.QueueName)))
 		workload.PrepareForEviction(wl, r.clock.Now(), kueue.WorkloadEvictedByLocalQueueStopped, "The LocalQueue is stopped")
+		reportWorkloadEvictedOnce := workload.WorkloadEvictionStateInc(wl, kueue.WorkloadEvictedByLocalQueueStopped, "")
 		err := workload.ApplyAdmissionStatus(ctx, r.client, wl, true, r.clock)
 		if err == nil {
 			cqName := lq.Spec.ClusterQueue
 			if slices.Contains(r.queues.GetClusterQueueNames(), cqName) {
 				workload.ReportEvictedWorkload(r.recorder, wl, cqName, kueue.WorkloadEvictedByLocalQueueStopped, "The LocalQueue is stopped")
 			}
+			if reportWorkloadEvictedOnce {
+				metrics.ReportEvictedWorkloadsOnce(cqName, kueue.WorkloadEvictedByLocalQueueStopped, "")
+			}
 		}
 		return true, err
 	}
diff --git a/pkg/controller/tas/node_failure_controller.go b/pkg/controller/tas/node_failure_controller.go
index 09db8ea29..fbbcfb61e 100644
--- a/pkg/controller/tas/node_failure_controller.go
+++ b/pkg/controller/tas/node_failure_controller.go
@@ -300,12 +300,7 @@ func (r *nodeFailureReconciler) patchWorkloadsForNodeToReplace(ctx context.Conte
 }
 
 func (r *nodeFailureReconciler) startEviction(ctx context.Context, wl *kueue.Workload, evictionMessage string) error {
-	workload.PrepareForEviction(wl, r.clock.Now(), kueue.WorkloadEvictedDueToNodeFailures, evictionMessage)
-	if err := workload.ApplyAdmissionStatus(ctx, r.client, wl, true, r.clock); err != nil {
-		return err
-	}
-	workload.ReportEvictedWorkload(r.recorder, wl, wl.Status.Admission.ClusterQueue, kueue.WorkloadEvictedDueToNodeFailures, evictionMessage)
-	return nil
+	return workload.EvictWorkload(ctx, r.client, r.recorder, wl, kueue.WorkloadEvictedDueToNodeFailures, evictionMessage, r.clock)
 }
 
 func (r *nodeFailureReconciler) reconcileForReplaceNodeOnPodTermination(ctx context.Context, nodeName string) (ctrl.Result, error) {
diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go
index 948de6a87..9e16e9087 100644
--- a/pkg/metrics/metrics.go
+++ b/pkg/metrics/metrics.go
@@ -164,7 +164,12 @@ The label 'reason' can have the following values:
 - "PodsReadyTimeout" means that the eviction took place due to a PodsReady timeout.
 - "AdmissionCheck" means that the workload was evicted because at least one admission check transitioned to False.
 - "ClusterQueueStopped" means that the workload was evicted because the ClusterQueue is stopped.
-- "Deactivated" means that the workload was evicted because spec.active is set to false`,
+- "LocalQueueStopped" means that the workload was evicted because the LocalQueue is stopped.
+- "NodeFailures" means that the workload was evicted due to node failures when using TopologyAwareScheduling.
+- "Deactivated" means that the workload was evicted because spec.active is set to false.
+- "DeactivatedDueToAdmissionCheck" means that the workload was evicted and deactivated by Kueue due to a rejected admission check.
+- "DeactivatedDueToMaximumExecutionTimeExceeded" means that the workload was evicted and deactivated by Kueue due to maximum execution time exceeded.
+- "DeactivatedDueToRequeuingLimitExceeded" means that the workload was evicted and deactivated by Kueue due to requeuing limit exceeded.`,
 			Buckets: generateExponentialBuckets(14),
 		}, []string{"cluster_queue", "reason"},
 	)
@@ -276,7 +281,12 @@ The label 'reason' can have the following values:
 - "PodsReadyTimeout" means that the eviction took place due to a PodsReady timeout.
 - "AdmissionCheck" means that the workload was evicted because at least one admission check transitioned to False.
 - "ClusterQueueStopped" means that the workload was evicted because the ClusterQueue is stopped.
-- "Deactivated" means that the workload was evicted because spec.active is set to false`,
+- "LocalQueueStopped" means that the workload was evicted because the LocalQueue is stopped.
+- "NodeFailures" means that the workload was evicted due to node failures when using TopologyAwareScheduling.
+- "Deactivated" means that the workload was evicted because spec.active is set to false.
+- "DeactivatedDueToAdmissionCheck" means that the workload was evicted and deactivated by Kueue due to a rejected admission check.
+- "DeactivatedDueToMaximumExecutionTimeExceeded" means that the workload was evicted and deactivated by Kueue due to maximum execution time exceeded.
+- "DeactivatedDueToRequeuingLimitExceeded" means that the workload was evicted and deactivated by Kueue due to requeuing limit exceeded.`,
 		}, []string{"cluster_queue", "reason"},
 	)
 
@@ -298,7 +308,12 @@ The label 'reason' can have the following values:
 - "PodsReadyTimeout" means that the eviction took place due to a PodsReady timeout.
 - "AdmissionCheck" means that the workload was evicted because at least one admission check transitioned to False.
 - "ClusterQueueStopped" means that the workload was evicted because the ClusterQueue is stopped.
-- "Deactivated" means that the workload was evicted because spec.active is set to false`,
+- "LocalQueueStopped" means that the workload was evicted because the LocalQueue is stopped.
+- "NodeFailures" means that the workload was evicted due to node failures when using TopologyAwareScheduling.
+- "Deactivated" means that the workload was evicted because spec.active is set to false.
+- "DeactivatedDueToAdmissionCheck" means that the workload was evicted and deactivated by Kueue due to a rejected admission check.
+- "DeactivatedDueToMaximumExecutionTimeExceeded" means that the workload was evicted and deactivated by Kueue due to maximum execution time exceeded.
+- "DeactivatedDueToRequeuingLimitExceeded" means that the workload was evicted and deactivated by Kueue due to requeuing limit exceeded.`,
 		}, []string{"name", "namespace", "reason"},
 	)
 
@@ -312,7 +327,12 @@ The label 'reason' can have the following values:
 - "PodsReadyTimeout" means that the eviction took place due to a PodsReady timeout.
 - "AdmissionCheck" means that the workload was evicted because at least one admission check transitioned to False.
 - "ClusterQueueStopped" means that the workload was evicted because the ClusterQueue is stopped.
-- "Deactivated" means that the workload was evicted because spec.active is set to false`,
+- "LocalQueueStopped" means that the workload was evicted because the LocalQueue is stopped.
+- "NodeFailures" means that the workload was evicted due to node failures when using TopologyAwareScheduling.
+- "Deactivated" means that the workload was evicted because spec.active is set to false.
+- "DeactivatedDueToAdmissionCheck" means that the workload was evicted and deactivated by Kueue due to a rejected admission check.
+- "DeactivatedDueToMaximumExecutionTimeExceeded" means that the workload was evicted and deactivated by Kueue due to maximum execution time exceeded.
+- "DeactivatedDueToRequeuingLimitExceeded" means that the workload was evicted and deactivated by Kueue due to requeuing limit exceeded.`,
 		}, []string{"cluster_queue", "reason", "detailed_reason"},
 	)
 
@@ -443,8 +463,8 @@ For a LocalQueue, the metric only reports a value of 1 for one of the statuses.`
 		prometheus.GaugeOpts{
 			Subsystem: constants.KueueName,
 			Name:      "cluster_queue_weighted_share",
-			Help: `Reports a value that representing the maximum of the ratios of usage above nominal 
-quota to the lendable resources in the cohort, among all the resources provided by 
+			Help: `Reports a value that representing the maximum of the ratios of usage above nominal
+quota to the lendable resources in the cohort, among all the resources provided by
 the ClusterQueue, and divided by the weight.
 If zero, it means that the usage of the ClusterQueue is below the nominal quota.
 If the ClusterQueue has a weight of zero and is borrowing, this will return 9223372036854775807,
@@ -456,8 +476,8 @@ the maximum possible share value.`,
 		prometheus.GaugeOpts{
 			Subsystem: constants.KueueName,
 			Name:      "cohort_weighted_share",
-			Help: `Reports a value that representing the maximum of the ratios of usage above nominal 
-quota to the lendable resources in the Cohort, among all the resources provided by 
+			Help: `Reports a value that representing the maximum of the ratios of usage above nominal
+quota to the lendable resources in the Cohort, among all the resources provided by
 the Cohort, and divided by the weight.
 If zero, it means that the usage of the Cohort is below the nominal quota.
 If the Cohort has a weight of zero and is borrowing, this will return 9223372036854775807,
diff --git a/pkg/scheduler/preemption/preemption.go b/pkg/scheduler/preemption/preemption.go
index 978dd9d01..ef92ed7c3 100644
--- a/pkg/scheduler/preemption/preemption.go
+++ b/pkg/scheduler/preemption/preemption.go
@@ -188,10 +188,13 @@ func (p *Preemptor) applyPreemptionWithSSA(ctx context.Context, w *kueue.Workloa
 	workload.PrepareForEviction(w, p.clock.Now(), kueue.WorkloadEvictedByPreemption, message)
 	reportWorkloadEvictedOnce := workload.WorkloadEvictionStateInc(w, kueue.WorkloadEvictedByPreemption, "")
 	workload.SetPreemptedCondition(w, reason, message)
-	if reportWorkloadEvictedOnce {
-		metrics.ReportEvictedWorkloadsOnce(w.Status.Admission.ClusterQueue, kueue.WorkloadEvictedByPreemption, "")
+	err := workload.ApplyAdmissionStatus(ctx, p.client, w, true, p.clock)
+	if err == nil {
+		if reportWorkloadEvictedOnce {
+			metrics.ReportEvictedWorkloadsOnce(w.Status.Admission.ClusterQueue, kueue.WorkloadEvictedByPreemption, "")
+		}
 	}
-	return workload.ApplyAdmissionStatus(ctx, p.client, w, true, p.clock)
+	return err
 }
 
 type preemptionAttemptOpts struct {
diff --git a/site/content/zh-CN/docs/tasks/run/multikueue/deployment.md b/site/content/zh-CN/docs/tasks/run/multikueue/deployment.md
index c3a166359..6c699cf07 100644
--- a/site/content/zh-CN/docs/tasks/run/multikueue/deployment.md
+++ b/site/content/zh-CN/docs/tasks/run/multikueue/deployment.md
@@ -1,29 +1,27 @@
 ---
-title: "Run Deployment in Multi-Cluster"
+title: "在多集群环境中运行 Deployment"
 linkTitle: "Deployment"
 weight: 2
 date: 2025-02-17
-description: >
-  Run a MultiKueue scheduled Deployment.
+description: 运行 MultiKueue 调度的 Deployment。
 ---
 
-## Before you begin
+## 开始之前 {#before-you-begin}
 
-1. Check the [MultiKueue installation guide](/docs/tasks/manage/setup_multikueue) on how to properly setup MultiKueue clusters.
+1. 请查阅 [MultiKueue 安装指南](/zh-CN/docs/tasks/manage/setup_multikueue)了解如何正确设置 MultiKueue 集群。
 
-2. Follow steps in [Run Plain Pods](/docs/tasks/run/plain_pods/#before-you-begin)
-to learn how to enable and configure the `pod` integration which is required for enabling the `deployment` integration.
+2. 按照[运行普通 Pod](/zh-CN/docs/tasks/run/plain_pods/#before-you-begin)的步骤，了解如何启用和配置 Pod 集成，这对于启用 Deployment 集成是必需的。
 
-Deployments receive live status updates through the status from the remote Pods created on the worker cluster.
+Deployment 会实时接收在工作集群创建的远程 Pod 的状态，并更新。
 
 {{< feature-state state="beta" for_version="v0.11.0" >}}
 
-{{% alert title="Note" color="primary" %}}
-In this current implementation, when Deployments are created in environments with multiple worker clusters, Pods are allocated to any worker.
+{{% alert title="注意" color="primary" %}}
+在当前实现中，当在有多个工作集群的环境中创建 Deployment 时，Pod 会被分配到任意工作集群。
 {{% /alert %}}
 
-## Example
+## 示例 {#examples}
 
-Once the setup is complete you can test it by running the example below:
+完成设置后，你可以通过运行以下示例进行测试：
 
 {{< include "examples/serving-workloads/sample-deployment.yaml" "yaml" >}}
diff --git a/site/content/zh-CN/docs/tasks/run/multikueue/job.md b/site/content/zh-CN/docs/tasks/run/multikueue/job.md
index a1a7e7f8e..ac40d8d67 100644
--- a/site/content/zh-CN/docs/tasks/run/multikueue/job.md
+++ b/site/content/zh-CN/docs/tasks/run/multikueue/job.md
@@ -1,47 +1,46 @@
 ---
-title: "Run Kubernetes Job in Multi-Cluster"
+title: "在多集群环境中运行 Job"
 linkTitle: "Kubernetes Job"
 weight: 2
 date: 2024-11-05
-description: >
-  Run a MultiKueue scheduled Kubernetes Job.
+description: 运行 MultiKueue 调度的 Kubernetes Job。
 ---
 
-## Before you begin
+## 开始之前 {#before-you-begin}
 
-Check the [MultiKueue installation guide](/docs/tasks/manage/setup_multikueue) on how to properly setup MultiKueue clusters.
+请查阅 [MultiKueue 安装指南](/zh-CN/docs/tasks/manage/setup_multikueue)了解如何正确设置 MultiKueue 集群。
 
-For the ease of setup and use we recommend using at least Kueue v0.8.1.
+为方便安装和使用，建议使用 Kueue v0.8.1 以上版本。
 
-The recommended way of running MultiKueue depends on the configuration of the `JobManagedBy` feature gate in your cluster. 
+运行 MultiKueue 的推荐方式取决于集群中 `JobManagedBy` 特性门控的配置。
 
-{{% alert title="Note" color="primary" %}}
-The `JobManagedBy` feature gate is disabled in 1.30 and 1.31 by default, and will be enabled in 1.32 by default.
+{{% alert title="注意" color="primary" %}}
+`JobManagedBy` 特性门控在 Kubernetes v1.30 和 v1.31 版本中默认禁用，在 v1.32 版本中默认启用。
 {{% /alert %}}
 
-### Cluster with JobManagedBy enabled
+### 集群启用 JobManagedBy {#cluster-with-jobmanagedby-enabled}
 
-When `JobManagedBy` is enabled in your cluster we recommend configuring Kueue to enable the `MultiKueueBatchJobWithManagedBy` feature gate. 
+当集群启用 `JobManagedBy` 时，建议配置 Kueue 启用 `MultiKueueBatchJobWithManagedBy` 特性门控。
 
-When `MultiKueueBatchJobWithManagedBy` is enabled, the current status of a Job being executed by MultiKueue on a worker cluster is live-updated on the management cluster.
+启用后，MultiKueue 在工作集群上执行的 Job 的当前状态会实时同步到管理集群。
 
-This gives the users and automation tools the ability to track the progress of Job status (.status) without lookup to the
-worker cluster, making MultiKueue transparent from that perspective.
+这样，用户和自动化工具可以在不访问工作集群的情况下，直接跟踪 Job 的状态（`.status`），
+从而实现 MultiKueue 的透明化。
 
-### Cluster with JobManagedBy disabled
+### 集群未启用 JobManagedBy {#cluster-with-jobmanagedby-disabled}
 
-When `JobManagedBy` is disabled in your cluster you should make sure `MultiKueueBatchJobWithManagedBy` is also disabled in Kueue. 
+当集群未启用 `JobManagedBy` 时，应确保 Kueue 也未启用 `MultiKueueBatchJobWithManagedBy`。
 
-This is important so that MultiKueue does not conflict with the build-in Job controller on the management cluster. 
+这样可以避免 MultiKueue 与管理集群上的内置 Job 控制器发生冲突。
 
-As a limitation of this deployment mode, to get the actual status of the job, you need to access the worker cluster.
+这种部署模式有一个缺点，你需要访问工作集群才能获取 Job 的实际状态。
 
-You can identify the worker cluster running the job by checking the AC status message of the workload object in the management cluster.
+你可以通过检查管理集群中工作负载对象的 AC 状态消息，辨别正在运行 Job 的工作集群。
 
-Also, the job is suspended from the perspective of the management cluster until it is `Finished`.
+此外，从管理集群的角度看，Job 在达到 `Finished` 之前，会一直处于挂起状态。
 
-## Example
+## 示例 {#examples}
 
-Once the setup is complete you can test it by running the example below:
+完成设置后，你可以通过运行以下示例进行测试：
 
 {{< include "examples/jobs/sample-job.yaml" "yaml" >}}



Test patch:
diff --git a/pkg/controller/core/workload_controller_test.go b/pkg/controller/core/workload_controller_test.go
index 13130afa6..b69709c23 100644
--- a/pkg/controller/core/workload_controller_test.go
+++ b/pkg/controller/core/workload_controller_test.go
@@ -677,7 +677,7 @@ func TestReconcile(t *testing.T) {
 				).
 				SchedulingStatsEviction(
 					kueue.WorkloadSchedulingStatsEviction{
-						Reason: kueue.WorkloadDeactivated,
+						Reason: "DeactivatedDueToAdmissionCheck",
 						Count:  1,
 					},
 				).
@@ -1344,7 +1344,7 @@ func TestReconcile(t *testing.T) {
 					Message: "exceeding the maximum number of re-queuing retries",
 				}).
 				SchedulingStatsEviction(
-					kueue.WorkloadSchedulingStatsEviction{Reason: kueue.WorkloadDeactivated, Count: 1},
+					kueue.WorkloadSchedulingStatsEviction{Reason: "DeactivatedDueToRequeuingLimitExceeded", Count: 1},
 				).
 				Obj(),
 			wantEvents: []utiltesting.EventRecord{
@@ -1408,7 +1408,7 @@ func TestReconcile(t *testing.T) {
 				}).
 				SchedulingStatsEviction(
 					kueue.WorkloadSchedulingStatsEviction{
-						Reason: kueue.WorkloadDeactivated,
+						Reason: "DeactivatedDueToRequeuingLimitExceeded",
 						Count:  1,
 					},
 				).
@@ -1477,7 +1477,7 @@ func TestReconcile(t *testing.T) {
 				RequeueState(ptr.To[int32](100), nil).
 				SchedulingStatsEviction(
 					kueue.WorkloadSchedulingStatsEviction{
-						Reason: kueue.WorkloadDeactivated,
+						Reason: "DeactivatedDueToRequeuingLimitExceeded",
 						Count:  1,
 					},
 				).
@@ -1546,7 +1546,7 @@ func TestReconcile(t *testing.T) {
 				RequeueState(ptr.To[int32](100), nil).
 				SchedulingStatsEviction(
 					kueue.WorkloadSchedulingStatsEviction{
-						Reason: kueue.WorkloadDeactivated,
+						Reason: "DeactivatedDueToRequeuingLimitExceeded",
 						Count:  1,
 					},
 				).
@@ -1640,6 +1640,12 @@ func TestReconcile(t *testing.T) {
 					Reason:  kueue.WorkloadEvictedByLocalQueueStopped,
 					Message: "The LocalQueue is stopped",
 				}).
+				SchedulingStatsEviction(
+					kueue.WorkloadSchedulingStatsEviction{
+						Reason: kueue.WorkloadEvictedByLocalQueueStopped,
+						Count:  1,
+					},
+				).
 				Obj(),
 			wantEvents: []utiltesting.EventRecord{
 				{
diff --git a/test/integration/singlecluster/controller/core/workload_controller_test.go b/test/integration/singlecluster/controller/core/workload_controller_test.go
index 0b6c06f10..47e01b4c6 100644
--- a/test/integration/singlecluster/controller/core/workload_controller_test.go
+++ b/test/integration/singlecluster/controller/core/workload_controller_test.go
@@ -311,7 +311,7 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn
 					g.Expect(k8sClient.Get(ctx, wlKey, updatedWl)).To(gomega.Succeed())
 					g.Expect(workload.IsEvictedByDeactivation(updatedWl)).To(gomega.BeTrue())
 					util.ExpectEvictedWorkloadsTotalMetric(clusterQueue.Name, "DeactivatedDueToAdmissionCheck", 1)
-					util.ExpectEvictedWorkloadsOnceTotalMetric(clusterQueue.Name, kueue.WorkloadDeactivated, "", 1)
+					util.ExpectEvictedWorkloadsOnceTotalMetric(clusterQueue.Name, "DeactivatedDueToAdmissionCheck", "", 1)
 				}, util.Timeout, util.Interval).Should(gomega.Succeed())
 			})
 		})
@@ -396,7 +396,7 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn
 
 					g.Expect(workload.IsEvictedByDeactivation(updatedWl)).To(gomega.BeTrue())
 					util.ExpectEvictedWorkloadsTotalMetric(clusterQueue.Name, "DeactivatedDueToAdmissionCheck", 1)
-					util.ExpectEvictedWorkloadsOnceTotalMetric(clusterQueue.Name, kueue.WorkloadDeactivated, "", 1)
+					util.ExpectEvictedWorkloadsOnceTotalMetric(clusterQueue.Name, "DeactivatedDueToAdmissionCheck", "", 1)
 				}, util.Timeout, util.Interval).Should(gomega.Succeed())
 			})
 		})
diff --git a/test/integration/singlecluster/scheduler/scheduler_test.go b/test/integration/singlecluster/scheduler/scheduler_test.go
index 71eee5088..9848bb864 100644
--- a/test/integration/singlecluster/scheduler/scheduler_test.go
+++ b/test/integration/singlecluster/scheduler/scheduler_test.go
@@ -2067,6 +2067,7 @@ var _ = ginkgo.Describe("Scheduler", func() {
 			}, util.Timeout, util.Interval).Should(gomega.Succeed())
 
 			util.ExpectEvictedWorkloadsTotalMetric(cq.Name, kueue.WorkloadEvictedByLocalQueueStopped, 1)
+			util.ExpectEvictedWorkloadsOnceTotalMetric(cq.Name, kueue.WorkloadEvictedByLocalQueueStopped, "", 1)
 
 			createdWl := kueue.Workload{}
 			ginkgo.By("Checking the condition of workload is evicted")