Repository: kubernetes-sigs/kueue
Pull request #4502: Automated cherry pick of #4341: TAS: fix topology assignment for the RayJob's submitter Job
#4498: Add check to verify RayJob tas pods were deleted
Author: login: mszadkow
Labels: name: kind/bug
description: Categorizes issue or PR as related to a bug.
name: release-note
description: Denotes a PR that will be considered when it comes time to generate release notes.
name: lgtm
description: "Looks good to me", indicates that a PR is ready to be merged.
name: size/XL
description: Denotes a PR that changes 500-999 lines, ignoring generated files.
name: approved
description: Indicates a PR has been approved by an approver from all required OWNERS files.
name: cncf-cla: yes
description: Indicates the PR's author has signed the CNCF CLA.
Created: 2025-03-05T11:29:09Z | Merged: 2025-03-06T15:59:47Z

Cherry pick of #4341 #4498 on release-0.10.

#4341: TAS: fix topology assignment for the RayJob's submitter Job
#4498: Add check to verify RayJob tas pods were deleted

For details on the cherry pick process, see the [cherry pick requests](https://git.k8s.io/community/contributors/devel/sig-release/cherry-picks.md) page.

```release-note
TAS: Do not ignore the TAS annotation if set on the template for the Ray submitter Job.
```


Diff:
diff --git a/Makefile-deps.mk b/Makefile-deps.mk
index aeaf48389b3..3d892ace194 100644
--- a/Makefile-deps.mk
+++ b/Makefile-deps.mk
@@ -128,8 +128,21 @@ kf-training-operator-manifests: ## Copy whole manifests folder from the training
 RAY_ROOT = $(shell $(GO_CMD) list -m -mod=readonly -f "{{.Dir}}" github.com/ray-project/kuberay/ray-operator)
 .PHONY: ray-operator-crd
 ray-operator-crd: ## Copy the CRDs from the ray-operator to the dep-crds directory.
-	mkdir -p $(EXTERNAL_CRDS_DIR)/ray-operator/
-	cp -f $(RAY_ROOT)/config/crd/bases/* $(EXTERNAL_CRDS_DIR)/ray-operator/
+	mkdir -p $(EXTERNAL_CRDS_DIR)/ray-operator-crds/
+	cp -f $(RAY_ROOT)/config/crd/bases/* $(EXTERNAL_CRDS_DIR)/ray-operator-crds/
+
+.PHONY: ray-operator-manifests
+ray-operator-manifests: ## Copy the whole manifests content from the ray-operator to the dep-crds directory.
+	## Full version of the manifest is required for e2e multikueue tests.
+	if [ -d "$(EXTERNAL_CRDS_DIR)/ray-operator" ]; then \
+		chmod -R u+w "$(EXTERNAL_CRDS_DIR)/ray-operator" && \
+		rm -rf "$(EXTERNAL_CRDS_DIR)/ray-operator"; \
+	fi
+	mkdir -p "$(EXTERNAL_CRDS_DIR)/ray-operator"; \
+	cp -rf "$(RAY_ROOT)/config/crd" "$(EXTERNAL_CRDS_DIR)/ray-operator"
+	cp -rf "$(RAY_ROOT)/config/default" "$(EXTERNAL_CRDS_DIR)/ray-operator"
+	cp -rf "$(RAY_ROOT)/config/rbac" "$(EXTERNAL_CRDS_DIR)/ray-operator"
+	cp -rf "$(RAY_ROOT)/config/manager" "$(EXTERNAL_CRDS_DIR)/ray-operator"
 
 JOBSET_ROOT = $(shell $(GO_CMD) list -m -mod=readonly -f "{{.Dir}}" sigs.k8s.io/jobset)
 .PHONY: jobset-operator-crd
@@ -144,7 +157,7 @@ cluster-autoscaler-crd: ## Copy the CRDs from the cluster-autoscaler to the dep-
 	cp -f $(CLUSTER_AUTOSCALER_ROOT)/config/crd/* $(EXTERNAL_CRDS_DIR)/cluster-autoscaler/
 
 .PHONY: dep-crds
-dep-crds: mpi-operator-crd kf-training-operator-crd ray-operator-crd jobset-operator-crd cluster-autoscaler-crd kf-training-operator-manifests ## Copy the CRDs from the external operators to the dep-crds directory.
+dep-crds: mpi-operator-crd kf-training-operator-crd ray-operator-crd jobset-operator-crd cluster-autoscaler-crd kf-training-operator-manifests ray-operator-manifests## Copy the CRDs from the external operators to the dep-crds directory.
 	@echo "Copying CRDs from external operators to dep-crds directory"
 
 .PHONY: kueuectl-docs
diff --git a/Makefile-test.mk b/Makefile-test.mk
index 14781bb0a58..e6129829991 100644
--- a/Makefile-test.mk
+++ b/Makefile-test.mk
@@ -70,6 +70,7 @@ IMAGE_TAG ?= $(IMAGE_REPO):$(GIT_TAG)
 JOBSET_VERSION = $(shell $(GO_CMD) list -m -f "{{.Version}}" sigs.k8s.io/jobset)
 KUBEFLOW_VERSION = $(shell $(GO_CMD) list -m -f "{{.Version}}" github.com/kubeflow/training-operator)
 KUBEFLOW_MPI_VERSION = $(shell $(GO_CMD) list -m -f "{{.Version}}" github.com/kubeflow/mpi-operator)
+KUBERAY_VERSION = $(shell $(GO_CMD) list -m -f "{{.Version}}" github.com/ray-project/kuberay/ray-operator)
 
 ##@ Tests
 
@@ -128,6 +129,7 @@ run-test-tas-e2e-%: FORCE
 	E2E_KIND_VERSION="kindest/node:v$(K8S_VERSION)" KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) CREATE_KIND_CLUSTER=$(CREATE_KIND_CLUSTER) \
 		ARTIFACTS="$(ARTIFACTS)/$@" IMAGE_TAG=$(IMAGE_TAG) GINKGO_ARGS="$(GINKGO_ARGS)" \
 		JOBSET_VERSION=$(JOBSET_VERSION) KUBEFLOW_VERSION=$(KUBEFLOW_VERSION) KUBEFLOW_MPI_VERSION=$(KUBEFLOW_MPI_VERSION) \
+		KUBERAY_VERSION=$(KUBERAY_VERSION) \
 		KIND_CLUSTER_FILE="tas-kind-cluster.yaml" E2E_TARGET_FOLDER="tas" \
 		./hack/e2e-test.sh
 	$(PROJECT_DIR)/bin/ginkgo-top -i $(ARTIFACTS)/$@/e2e.json > $(ARTIFACTS)/$@/e2e-top.yaml
diff --git a/hack/e2e-common.sh b/hack/e2e-common.sh
index 4234b09563e..13b9fdd7948 100644
--- a/hack/e2e-common.sh
+++ b/hack/e2e-common.sh
@@ -40,6 +40,14 @@ if [[ -n ${KUBEFLOW_MPI_VERSION:-} ]]; then
     export KUBEFLOW_MPI_IMAGE=mpioperator/mpi-operator:${KUBEFLOW_MPI_VERSION/#v}
 fi
 
+if [[ -n ${KUBERAY_VERSION:-} ]]; then
+    export KUBERAY_MANIFEST="${ROOT_DIR}/dep-crds/ray-operator/default/"
+    export KUBERAY_IMAGE=bitnami/kuberay-operator:${KUBERAY_VERSION/#v}
+    export KUBERAY_RAY_IMAGE=rayproject/ray:2.9.0
+    export KUBERAY_RAY_IMAGE_ARM=rayproject/ray:2.9.0-aarch64
+    export KUBERAY_CRDS=${ROOT_DIR}/dep-crds/ray-operator/crd/bases
+fi
+
 # agnhost image to use for testing.
 export E2E_TEST_AGNHOST_IMAGE_OLD=registry.k8s.io/e2e-test-images/agnhost:2.52@sha256:b173c7d0ffe3d805d49f4dfe48375169b7b8d2e1feb81783efd61eb9d08042e6
 E2E_TEST_AGNHOST_IMAGE_OLD_WITHOUT_SHA=${E2E_TEST_AGNHOST_IMAGE_OLD%%@*}
@@ -85,6 +93,18 @@ function prepare_docker_images {
     if [[ -n ${KUBEFLOW_MPI_VERSION:-} ]]; then
         docker pull "${KUBEFLOW_MPI_IMAGE}"
     fi
+
+    if [[ -n ${KUBERAY_VERSION:-} ]]; then
+        docker pull "${KUBERAY_IMAGE}"
+
+        # Extra e2e images required for Kuberay
+        unamestr=$(uname)
+        if [[ "$unamestr" == 'Linux' ]]; then
+            docker pull "${KUBERAY_RAY_IMAGE}"
+        elif [[ "$unamestr" == 'Darwin' ]]; then
+            docker pull "${KUBERAY_RAY_IMAGE_ARM}"
+        fi
+    fi
 }
 
 # $1 cluster
@@ -131,6 +151,22 @@ function install_mpi {
     kubectl apply --server-side -f "${KUBEFLOW_MPI_MANIFEST}"
 }
 
+#$1 - cluster name
+function install_kuberay {
+    # Extra e2e images required for Kuberay
+    unamestr=$(uname)
+    if [[ "$unamestr" == 'Linux' ]]; then
+        cluster_kind_load_image "${1}" "${KUBERAY_RAY_IMAGE}"
+    elif [[ "$unamestr" == 'Darwin' ]]; then
+        cluster_kind_load_image "${1}" "${KUBERAY_RAY_IMAGE_ARM}"
+    fi 
+
+    cluster_kind_load_image "${1}" "${KUBERAY_IMAGE}"
+    kubectl config use-context "kind-${1}"
+    # create used instead of apply - https://github.com/ray-project/kuberay/issues/504
+    kubectl create -k "${KUBERAY_MANIFEST}"
+}
+
 INITIAL_IMAGE=$($YQ '.images[] | select(.name == "controller") | [.newName, .newTag] | join(":")' config/components/manager/kustomization.yaml)
 export INITIAL_IMAGE
 
diff --git a/hack/e2e-test.sh b/hack/e2e-test.sh
index a2236dfd458..7606260d017 100755
--- a/hack/e2e-test.sh
+++ b/hack/e2e-test.sh
@@ -62,6 +62,9 @@ function kind_load {
     if [[ -n ${KUBEFLOW_MPI_VERSION:-} ]]; then
         install_mpi "$KIND_CLUSTER_NAME"
     fi
+    if [[ -n ${KUBERAY_VERSION:-} ]]; then
+        install_kuberay "$KIND_CLUSTER_NAME"
+    fi
 }
 
 function kueue_deploy {
diff --git a/pkg/controller/jobs/rayjob/rayjob_controller.go b/pkg/controller/jobs/rayjob/rayjob_controller.go
index 16488da894e..cfc58fa64cc 100644
--- a/pkg/controller/jobs/rayjob/rayjob_controller.go
+++ b/pkg/controller/jobs/rayjob/rayjob_controller.go
@@ -137,11 +137,14 @@ func (j *RayJob) PodSets() []kueue.PodSet {
 	// submitter Job
 	if j.Spec.SubmissionMode == rayv1.K8sJobMode {
 		submitterJobPodSet := kueue.PodSet{
-			Name:  submitterJobPodSetName,
-			Count: 1,
+			Name:     submitterJobPodSetName,
+			Count:    1,
+			Template: *getSubmitterTemplate(j),
 		}
 
-		submitterJobPodSet.Template = *getSubmitterTemplate(j)
+		// Create the TopologyRequest for the Submitter Job PodSet, based on the annotations
+		// in rayJob.Spec.SubmitterPodTemplate, which can be specified by the user.
+		submitterJobPodSet.TopologyRequest = jobframework.PodSetTopologyRequest(&submitterJobPodSet.Template.ObjectMeta, nil, nil, nil)
 		podSets = append(podSets, submitterJobPodSet)
 	}
 
diff --git a/pkg/controller/jobs/rayjob/rayjob_controller_test.go b/pkg/controller/jobs/rayjob/rayjob_controller_test.go
index 00b65bc6c89..dc21236bd3d 100644
--- a/pkg/controller/jobs/rayjob/rayjob_controller_test.go
+++ b/pkg/controller/jobs/rayjob/rayjob_controller_test.go
@@ -197,6 +197,126 @@ func TestPodSets(t *testing.T) {
 				}
 			},
 		},
+		"with submitter topology annotation not specified": {
+			rayJob: (*RayJob)(testingrayutil.MakeJob("rayjob", "ns").
+				WithSubmissionMode(rayv1.K8sJobMode).
+				WithHeadGroupSpec(
+					rayv1.HeadGroupSpec{
+						Template: corev1.PodTemplateSpec{
+							ObjectMeta: metav1.ObjectMeta{
+								Annotations: map[string]string{
+									kueuealpha.PodSetRequiredTopologyAnnotation: "cloud.com/rack",
+								},
+							},
+							Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "head_c"}}},
+						},
+					},
+				).
+				WithWorkerGroups(
+					rayv1.WorkerGroupSpec{
+						GroupName: "group1",
+						Replicas:  ptr.To[int32](3),
+						Template: corev1.PodTemplateSpec{
+							ObjectMeta: metav1.ObjectMeta{
+								Annotations: map[string]string{
+									kueuealpha.PodSetPreferredTopologyAnnotation: "cloud.com/block",
+								},
+							},
+							Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "group1_c"}}},
+						},
+					},
+				).
+				Obj()),
+			wantPodSets: func(rayJob *RayJob) []kueue.PodSet {
+				return []kueue.PodSet{
+					{
+						Name:            headGroupPodSetName,
+						Count:           1,
+						Template:        *rayJob.Spec.RayClusterSpec.HeadGroupSpec.Template.DeepCopy(),
+						TopologyRequest: &kueue.PodSetTopologyRequest{Required: ptr.To("cloud.com/rack")},
+					},
+					{
+						Name:            rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].GroupName,
+						Count:           3,
+						Template:        *rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Template.DeepCopy(),
+						TopologyRequest: &kueue.PodSetTopologyRequest{Preferred: ptr.To("cloud.com/block")},
+					},
+					{
+						Name:     "submitter",
+						Count:    1,
+						Template: *getSubmitterTemplate(rayJob),
+					},
+				}
+			},
+		},
+		"with submitter topology annotation specified": {
+			rayJob: (*RayJob)(testingrayutil.MakeJob("rayjob", "ns").
+				WithSubmissionMode(rayv1.K8sJobMode).
+				WithHeadGroupSpec(
+					rayv1.HeadGroupSpec{
+						Template: corev1.PodTemplateSpec{
+							ObjectMeta: metav1.ObjectMeta{
+								Annotations: map[string]string{
+									kueuealpha.PodSetRequiredTopologyAnnotation: "cloud.com/rack",
+								},
+							},
+							Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "head_c"}}},
+						},
+					},
+				).
+				WithWorkerGroups(
+					rayv1.WorkerGroupSpec{
+						GroupName: "group1",
+						Replicas:  ptr.To[int32](3),
+						Template: corev1.PodTemplateSpec{
+							ObjectMeta: metav1.ObjectMeta{
+								Annotations: map[string]string{
+									kueuealpha.PodSetPreferredTopologyAnnotation: "cloud.com/block",
+								},
+							},
+							Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "group1_c"}}},
+						},
+					},
+				).
+				WithSubmitterPodTemplate(corev1.PodTemplateSpec{
+					Spec: corev1.PodSpec{
+						Containers: []corev1.Container{
+							{
+								Name: "submitter_c",
+							},
+						},
+						RestartPolicy: corev1.RestartPolicyNever,
+					},
+					ObjectMeta: metav1.ObjectMeta{
+						Annotations: map[string]string{
+							kueuealpha.PodSetRequiredTopologyAnnotation: "cloud.com/block",
+						},
+					},
+				}).
+				Obj()),
+			wantPodSets: func(rayJob *RayJob) []kueue.PodSet {
+				return []kueue.PodSet{
+					{
+						Name:            headGroupPodSetName,
+						Count:           1,
+						Template:        *rayJob.Spec.RayClusterSpec.HeadGroupSpec.Template.DeepCopy(),
+						TopologyRequest: &kueue.PodSetTopologyRequest{Required: ptr.To("cloud.com/rack")},
+					},
+					{
+						Name:            rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].GroupName,
+						Count:           3,
+						Template:        *rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Template.DeepCopy(),
+						TopologyRequest: &kueue.PodSetTopologyRequest{Preferred: ptr.To("cloud.com/block")},
+					},
+					{
+						Name:            "submitter",
+						Count:           1,
+						Template:        *getSubmitterTemplate(rayJob),
+						TopologyRequest: &kueue.PodSetTopologyRequest{Required: ptr.To("cloud.com/block")},
+					},
+				}
+			},
+		},
 		"with NumOfHosts > 1": {
 			rayJob: (*RayJob)(testingrayutil.MakeJob("rayjob", "ns").
 				WithHeadGroupSpec(
diff --git a/pkg/util/testingjobs/raycluster/wrappers.go b/pkg/util/testingjobs/raycluster/wrappers.go
index db05f1e1b10..2d76f2e1669 100644
--- a/pkg/util/testingjobs/raycluster/wrappers.go
+++ b/pkg/util/testingjobs/raycluster/wrappers.go
@@ -167,3 +167,14 @@ func (j *ClusterWrapper) WorkloadPriorityClass(wpc string) *ClusterWrapper {
 	j.Labels[constants.WorkloadPriorityClassLabel] = wpc
 	return j
 }
+
+func (j *ClusterWrapper) Image(rayType rayv1.RayNodeType, image string, args []string) *ClusterWrapper {
+	if rayType == rayv1.HeadNode {
+		j.Spec.HeadGroupSpec.Template.Spec.Containers[0].Image = image
+		j.Spec.HeadGroupSpec.Template.Spec.Containers[0].Args = args
+	} else if rayType == rayv1.WorkerNode {
+		j.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Image = image
+		j.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Args = args
+	}
+	return j
+}
diff --git a/pkg/util/testingjobs/rayjob/wrappers.go b/pkg/util/testingjobs/rayjob/wrappers.go
index 4718f7cfaeb..3ec0cd0f2b3 100644
--- a/pkg/util/testingjobs/rayjob/wrappers.go
+++ b/pkg/util/testingjobs/rayjob/wrappers.go
@@ -180,3 +180,21 @@ func (j *JobWrapper) Generation(num int64) *JobWrapper {
 func (j *JobWrapper) Clone() *JobWrapper {
 	return &JobWrapper{*j.DeepCopy()}
 }
+
+func (j *JobWrapper) Image(rayType rayv1.RayNodeType, image string, args []string) *JobWrapper {
+	if rayType == rayv1.HeadNode {
+		j.Spec.RayClusterSpec.HeadGroupSpec.Template.Spec.Containers[0].Image = image
+		j.Spec.RayClusterSpec.HeadGroupSpec.Template.Spec.Containers[0].Args = args
+		j.Spec.RayClusterSpec.HeadGroupSpec.Template.Spec.Containers[0].ImagePullPolicy = corev1.PullIfNotPresent
+	} else if rayType == rayv1.WorkerNode {
+		j.Spec.RayClusterSpec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Image = image
+		j.Spec.RayClusterSpec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Args = args
+		j.Spec.RayClusterSpec.WorkerGroupSpecs[0].Template.Spec.Containers[0].ImagePullPolicy = corev1.PullIfNotPresent
+	}
+	return j
+}
+
+func (j *JobWrapper) Entrypoint(e string) *JobWrapper {
+	j.Spec.Entrypoint = e
+	return j
+}
diff --git a/test/e2e/singlecluster/jobset_test.go b/test/e2e/singlecluster/jobset_test.go
index 15d5efb1e56..7eb5466b6db 100644
--- a/test/e2e/singlecluster/jobset_test.go
+++ b/test/e2e/singlecluster/jobset_test.go
@@ -67,7 +67,7 @@ var _ = ginkgo.Describe("JobSet", func() {
 			gomega.Expect(k8sClient.Create(ctx, localQueue)).Should(gomega.Succeed())
 		})
 		ginkgo.AfterEach(func() {
-			gomega.Expect(util.DeleteAllJobsetsInNamespace(ctx, k8sClient, ns)).Should(gomega.Succeed())
+			gomega.Expect(util.DeleteAllJobSetsInNamespace(ctx, k8sClient, ns)).Should(gomega.Succeed())
 			// Force remove workloads to be sure that cluster queue can be removed.
 			gomega.Expect(util.DeleteWorkloadsInNamespace(ctx, k8sClient, ns)).Should(gomega.Succeed())
 			gomega.Expect(util.DeleteObject(ctx, k8sClient, localQueue)).Should(gomega.Succeed())
@@ -147,7 +147,7 @@ var _ = ginkgo.Describe("JobSet", func() {
 			gomega.Expect(k8sClient.Create(ctx, localQueue)).Should(gomega.Succeed())
 		})
 		ginkgo.AfterEach(func() {
-			gomega.Expect(util.DeleteAllJobsetsInNamespace(ctx, k8sClient, ns)).Should(gomega.Succeed())
+			gomega.Expect(util.DeleteAllJobSetsInNamespace(ctx, k8sClient, ns)).Should(gomega.Succeed())
 			// Force remove workloads to be sure that cluster queue can be removed.
 			gomega.Expect(util.DeleteWorkloadsInNamespace(ctx, k8sClient, ns)).Should(gomega.Succeed())
 			gomega.Expect(util.DeleteObject(ctx, k8sClient, localQueue)).Should(gomega.Succeed())
diff --git a/test/e2e/tas/jobset_test.go b/test/e2e/tas/jobset_test.go
index 7dd90614a50..cf622c5ccfe 100644
--- a/test/e2e/tas/jobset_test.go
+++ b/test/e2e/tas/jobset_test.go
@@ -78,7 +78,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for JobSet", func() {
 			gomega.Expect(k8sClient.Create(ctx, localQueue)).Should(gomega.Succeed())
 		})
 		ginkgo.AfterEach(func() {
-			gomega.Expect(util.DeleteAllJobsetsInNamespace(ctx, k8sClient, ns)).Should(gomega.Succeed())
+			gomega.Expect(util.DeleteAllJobSetsInNamespace(ctx, k8sClient, ns)).Should(gomega.Succeed())
 			// Force remove workloads to be sure that cluster queue can be removed.
 			gomega.Expect(util.DeleteWorkloadsInNamespace(ctx, k8sClient, ns)).Should(gomega.Succeed())
 			util.ExpectObjectToBeDeleted(ctx, k8sClient, localQueue, true)
diff --git a/test/e2e/tas/rayjob_test.go b/test/e2e/tas/rayjob_test.go
new file mode 100644
index 00000000000..b23eedcdf49
--- /dev/null
+++ b/test/e2e/tas/rayjob_test.go
@@ -0,0 +1,237 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tase2e
+
+import (
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/utils/ptr"
+	"k8s.io/utils/set"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	kueuealpha "sigs.k8s.io/kueue/apis/kueue/v1alpha1"
+	kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1"
+	"sigs.k8s.io/kueue/pkg/util/testing"
+	testingrayjob "sigs.k8s.io/kueue/pkg/util/testingjobs/rayjob"
+	"sigs.k8s.io/kueue/test/util"
+)
+
+var _ = ginkgo.Describe("TopologyAwareScheduling for RayJob", ginkgo.Ordered, func() {
+	var (
+		ns           *corev1.Namespace
+		topology     *kueuealpha.Topology
+		tasFlavor    *kueue.ResourceFlavor
+		clusterQueue *kueue.ClusterQueue
+		localQueue   *kueue.LocalQueue
+	)
+
+	ginkgo.BeforeEach(func() {
+		ns = &corev1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				GenerateName: "e2e-tas-rayjob-",
+			},
+		}
+		gomega.Expect(k8sClient.Create(ctx, ns)).To(gomega.Succeed())
+
+		topology = testing.MakeDefaultThreeLevelTopology("datacenter")
+		gomega.Expect(k8sClient.Create(ctx, topology)).Should(gomega.Succeed())
+
+		tasFlavor = testing.MakeResourceFlavor("tas-flavor").
+			NodeLabel(tasNodeGroupLabel, instanceType).
+			TopologyName(topology.Name).
+			Obj()
+		gomega.Expect(k8sClient.Create(ctx, tasFlavor)).Should(gomega.Succeed())
+
+		clusterQueue = testing.MakeClusterQueue("cluster-queue").
+			ResourceGroup(
+				*testing.MakeFlavorQuotas(tasFlavor.Name).
+					Resource(corev1.ResourceCPU, "1").
+					Resource(corev1.ResourceMemory, "8Gi").
+					Resource(extraResource, "8").
+					Obj(),
+			).
+			Obj()
+		gomega.Expect(k8sClient.Create(ctx, clusterQueue)).Should(gomega.Succeed())
+		util.ExpectClusterQueuesToBeActive(ctx, k8sClient, clusterQueue)
+
+		localQueue = testing.MakeLocalQueue("local-queue", ns.Name).ClusterQueue(clusterQueue.Name).Obj()
+		gomega.Expect(k8sClient.Create(ctx, localQueue)).Should(gomega.Succeed())
+		util.ExpectLocalQueuesToBeActive(ctx, k8sClient, localQueue)
+	})
+	ginkgo.AfterEach(func() {
+		gomega.Expect(util.DeleteAllRayJobsInNamespace(ctx, k8sClient, ns)).To(gomega.Succeed())
+		gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed())
+		util.ExpectObjectToBeDeleted(ctx, k8sClient, clusterQueue, true)
+		util.ExpectObjectToBeDeleted(ctx, k8sClient, tasFlavor, true)
+		util.ExpectObjectToBeDeleted(ctx, k8sClient, topology, true)
+		util.ExpectAllPodsInNamespaceDeleted(ctx, k8sClient, ns)
+	})
+
+	ginkgo.When("Creating a RayJob", func() {
+		ginkgo.It("Should place pods based on the ranks-ordering", func() {
+			const (
+				headReplicas     = 1
+				workerReplicas   = 3
+				submitterReplica = 1
+			)
+			numPods := headReplicas + workerReplicas + submitterReplica
+			kuberayTestImage := util.GetKuberayTestImage()
+			rayjob := testingrayjob.MakeJob("ranks-ray", ns.Name).
+				Queue(localQueue.Name).
+				WithSubmissionMode(rayv1.K8sJobMode).
+				Entrypoint("python -c \"import ray; ray.init(); print(ray.cluster_resources())\"").
+				WithHeadGroupSpec(
+					rayv1.HeadGroupSpec{
+						RayStartParams: map[string]string{},
+						Template: corev1.PodTemplateSpec{
+							Spec: corev1.PodSpec{
+								RestartPolicy: corev1.RestartPolicyOnFailure,
+								Containers: []corev1.Container{
+									{
+										Name: "head-container",
+										Resources: corev1.ResourceRequirements{
+											Requests: corev1.ResourceList{
+												corev1.ResourceCPU: resource.MustParse("100m"),
+											},
+											Limits: corev1.ResourceList{
+												corev1.ResourceCPU: resource.MustParse("100m"),
+											},
+										},
+									},
+								},
+							},
+							ObjectMeta: metav1.ObjectMeta{
+								Annotations: map[string]string{
+									kueuealpha.PodSetPreferredTopologyAnnotation: testing.DefaultRackTopologyLevel,
+								},
+							},
+						},
+					},
+				).
+				WithWorkerGroups(
+					rayv1.WorkerGroupSpec{
+						GroupName:      "workers-group-0",
+						Replicas:       ptr.To[int32](workerReplicas),
+						MinReplicas:    ptr.To[int32](workerReplicas),
+						MaxReplicas:    ptr.To[int32](10),
+						RayStartParams: map[string]string{},
+						Template: corev1.PodTemplateSpec{
+							Spec: corev1.PodSpec{
+								RestartPolicy: corev1.RestartPolicyOnFailure,
+								Containers: []corev1.Container{
+									{
+										Name: "head-container",
+										Resources: corev1.ResourceRequirements{
+											Requests: corev1.ResourceList{
+												extraResource: resource.MustParse("1"),
+											},
+											Limits: corev1.ResourceList{
+												extraResource: resource.MustParse("1"),
+											},
+										},
+									},
+								},
+							},
+							ObjectMeta: metav1.ObjectMeta{
+								Annotations: map[string]string{
+									kueuealpha.PodSetPreferredTopologyAnnotation: testing.DefaultBlockTopologyLevel,
+								},
+							},
+						},
+					},
+				).
+				WithSubmitterPodTemplate(corev1.PodTemplateSpec{
+					Spec: corev1.PodSpec{
+						Containers: []corev1.Container{
+							{
+								Name:  "ranks-ray-submitter",
+								Image: kuberayTestImage,
+								Resources: corev1.ResourceRequirements{
+									Requests: corev1.ResourceList{
+										corev1.ResourceCPU: resource.MustParse("50m"),
+									},
+									Limits: corev1.ResourceList{
+										corev1.ResourceCPU: resource.MustParse("50m"),
+									},
+								},
+							},
+						},
+						RestartPolicy: corev1.RestartPolicyNever,
+					},
+					ObjectMeta: metav1.ObjectMeta{
+						Annotations: map[string]string{
+							kueuealpha.PodSetRequiredTopologyAnnotation: testing.DefaultBlockTopologyLevel,
+						},
+					},
+				}).
+				Image(rayv1.HeadNode, kuberayTestImage, []string{}).
+				Image(rayv1.WorkerNode, kuberayTestImage, []string{}).
+				Obj()
+
+			gomega.Expect(k8sClient.Create(ctx, rayjob)).Should(gomega.Succeed())
+
+			ginkgo.By("RayJob is unsuspended", func() {
+				gomega.Eventually(func(g gomega.Gomega) {
+					g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(rayjob), rayjob)).To(gomega.Succeed())
+					g.Expect(rayjob.Spec.Suspend).Should(gomega.BeFalse())
+				}, util.LongTimeout, util.Interval).Should(gomega.Succeed())
+			})
+
+			pods := &corev1.PodList{}
+			ginkgo.By("ensure all pods are created", func() {
+				gomega.Eventually(func(g gomega.Gomega) {
+					g.Expect(k8sClient.List(ctx, pods, client.InNamespace(rayjob.Namespace))).To(gomega.Succeed())
+					g.Expect(pods.Items).Should(gomega.HaveLen(numPods))
+					// The timeout is long to ensure all cluster pods are up and running.
+					// This is because the Ray image takes long time (around 170s on the CI)
+					// to load and then to sync with head.
+				}, util.VeryLongTimeout, util.Interval).Should(gomega.Succeed())
+			})
+
+			ginkgo.By("ensure all pods are scheduled", func() {
+				listOpts := &client.ListOptions{
+					FieldSelector: fields.OneTermNotEqualSelector("spec.nodeName", ""),
+				}
+				gomega.Eventually(func(g gomega.Gomega) {
+					g.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Name), listOpts)).To(gomega.Succeed())
+					g.Expect(pods.Items).Should(gomega.HaveLen(numPods))
+				}, util.LongTimeout, util.Interval).Should(gomega.Succeed())
+			})
+
+			ginkgo.By("verify the assignment of pods are as expected with rank-based ordering", func() {
+				gomega.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Name))).To(gomega.Succeed())
+				gotAssignment := readAssignedNodes(pods.Items)
+				gomega.Expect(gotAssignment).Should(gomega.HaveLen(workerReplicas))
+			})
+		})
+	})
+})
+
+func readAssignedNodes(pods []corev1.Pod) set.Set[string] {
+	assignment := set.Set[string]{}
+	for _, pod := range pods {
+		if role := pod.Labels["ray.io/node-type"]; role == "worker" {
+			assignment = assignment.Insert(pod.Spec.NodeName)
+		}
+	}
+	return assignment
+}
diff --git a/test/e2e/tas/suite_test.go b/test/e2e/tas/suite_test.go
index 885e7a2a786..cea97f35a28 100644
--- a/test/e2e/tas/suite_test.go
+++ b/test/e2e/tas/suite_test.go
@@ -61,6 +61,7 @@ var _ = ginkgo.BeforeSuite(func() {
 	util.WaitForJobSetAvailability(ctx, k8sClient)
 	util.WaitForKubeFlowTrainingOperatorAvailability(ctx, k8sClient)
 	util.WaitForKubeFlowMPIOperatorAvailability(ctx, k8sClient)
+	util.WaitForKubeRayOperatorAvailability(ctx, k8sClient)
 	ginkgo.GinkgoLogr.Info(
 		"Kueue, JobSet, KubeFlow Training and KubeFlow MPI operators are available in the cluster",
 		"waitingTime", time.Since(waitForAvailableStart),
diff --git a/test/integration/controller/jobs/raycluster/suite_test.go b/test/integration/controller/jobs/raycluster/suite_test.go
index 6dd6ab1f7e2..8c9d7a0918c 100644
--- a/test/integration/controller/jobs/raycluster/suite_test.go
+++ b/test/integration/controller/jobs/raycluster/suite_test.go
@@ -46,7 +46,7 @@ var (
 	ctx        context.Context
 	fwk        *framework.Framework
 	crdPath    = filepath.Join("..", "..", "..", "..", "..", "config", "components", "crd", "bases")
-	rayCrdPath = filepath.Join("..", "..", "..", "..", "..", "dep-crds", "ray-operator")
+	rayCrdPath = filepath.Join("..", "..", "..", "..", "..", "dep-crds", "ray-operator-crds")
 )
 
 func TestAPIs(t *testing.T) {
diff --git a/test/integration/controller/jobs/rayjob/suite_test.go b/test/integration/controller/jobs/rayjob/suite_test.go
index c59ac16141e..773965b347b 100644
--- a/test/integration/controller/jobs/rayjob/suite_test.go
+++ b/test/integration/controller/jobs/rayjob/suite_test.go
@@ -45,7 +45,7 @@ var (
 	ctx        context.Context
 	fwk        *framework.Framework
 	crdPath    = filepath.Join("..", "..", "..", "..", "..", "config", "components", "crd", "bases")
-	rayCrdPath = filepath.Join("..", "..", "..", "..", "..", "dep-crds", "ray-operator")
+	rayCrdPath = filepath.Join("..", "..", "..", "..", "..", "dep-crds", "ray-operator-crds")
 )
 
 func TestAPIs(t *testing.T) {
diff --git a/test/integration/webhook/jobs/suite_test.go b/test/integration/webhook/jobs/suite_test.go
index 8260b74a963..dda6f432d6b 100644
--- a/test/integration/webhook/jobs/suite_test.go
+++ b/test/integration/webhook/jobs/suite_test.go
@@ -46,7 +46,7 @@ var (
 	webhookPath          = filepath.Join("..", "..", "..", "..", "config", "components", "webhook")
 	mpiCrdPath           = filepath.Join("..", "..", "..", "..", "dep-crds", "mpi-operator")
 	jobsetCrdPath        = filepath.Join("..", "..", "..", "..", "dep-crds", "jobset-operator")
-	rayCrdPath           = filepath.Join("..", "..", "..", "..", "dep-crds", "ray-operator")
+	rayCrdPath           = filepath.Join("..", "..", "..", "..", "dep-crds", "ray-operator-crds")
 	kubeflowCrdPath      = filepath.Join("..", "..", "..", "..", "dep-crds", "training-operator-crds")
 )
 
diff --git a/test/util/constants.go b/test/util/constants.go
index b763954f8a5..397ad0dc485 100644
--- a/test/util/constants.go
+++ b/test/util/constants.go
@@ -30,6 +30,9 @@ const (
 	// LongTimeout is meant for E2E tests when waiting for complex operations
 	// such as running pods to completion.
 	LongTimeout = 45 * time.Second
+	// VeryLongTimeout is meant for E2E tests involving Ray which starts ray-project images (over 2GB)
+	// and also synchronizes the cluster before it can be used
+	VeryLongTimeout = 5 * time.Minute
 	// StartUpTimeout is meant to be used for waiting for Kueue to startup, given
 	// that cert updates can take up to 3 minutes to propagate to the filesystem.
 	// Taken into account that after the certificates are ready, all Kueue's components
diff --git a/test/util/e2e.go b/test/util/e2e.go
index 6ece7904f09..2456dbb05a1 100644
--- a/test/util/e2e.go
+++ b/test/util/e2e.go
@@ -4,11 +4,13 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"runtime"
 
 	"github.com/google/go-cmp/cmp/cmpopts"
 	kfmpi "github.com/kubeflow/mpi-operator/pkg/apis/kubeflow/v2beta1"
 	kftraining "github.com/kubeflow/training-operator/pkg/apis/kubeflow.org/v1"
 	"github.com/onsi/gomega"
+	rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -59,6 +61,9 @@ func CreateClientUsingCluster(kContext string) (client.WithWatch, *rest.Config)
 	err = kfmpi.AddToScheme(scheme.Scheme)
 	gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
 
+	err = rayv1.AddToScheme(scheme.Scheme)
+	gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
+
 	cfg.APIPath = "/api"
 	cfg.ContentConfig.GroupVersion = &schema.GroupVersion{Group: "", Version: "v1"}
 	cfg.ContentConfig.NegotiatedSerializer = scheme.Codecs.WithoutConversion()
@@ -143,6 +148,11 @@ func WaitForKubeFlowMPIOperatorAvailability(ctx context.Context, k8sClient clien
 	waitForOperatorAvailability(ctx, k8sClient, kftoKey)
 }
 
+func WaitForKubeRayOperatorAvailability(ctx context.Context, k8sClient client.Client) {
+	kroKey := types.NamespacedName{Namespace: "ray-system", Name: "kuberay-operator"}
+	waitForOperatorAvailability(ctx, k8sClient, kroKey)
+}
+
 func WaitForActivePodsAndTerminate(ctx context.Context, k8sClient client.Client, restClient *rest.RESTClient, cfg *rest.Config, namespace string, activePodsCount, exitCode int) {
 	var activePods []corev1.Pod
 	pods := corev1.PodList{}
@@ -170,3 +180,17 @@ func WaitForActivePodsAndTerminate(ctx context.Context, k8sClient client.Client,
 		}
 	}
 }
+
+func GetKuberayTestImage() string {
+	var (
+		kuberayTestImage string
+		found            bool
+	)
+	if runtime.GOARCH == "arm64" {
+		kuberayTestImage, found = os.LookupEnv("KUBERAY_RAY_IMAGE_ARM")
+	} else {
+		kuberayTestImage, found = os.LookupEnv("KUBERAY_RAY_IMAGE")
+	}
+	gomega.Expect(found).To(gomega.BeTrue())
+	return kuberayTestImage
+}
diff --git a/test/util/util.go b/test/util/util.go
index 28adcf1f181..1433c37ab9c 100644
--- a/test/util/util.go
+++ b/test/util/util.go
@@ -31,6 +31,7 @@ import (
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
 	"github.com/prometheus/client_golang/prometheus"
+	rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
 	zaplog "go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	appsv1 "k8s.io/api/apps/v1"
@@ -104,7 +105,7 @@ func DeleteNamespace(ctx context.Context, c client.Client, ns *corev1.Namespace)
 	if ns == nil {
 		return nil
 	}
-	if err := DeleteAllJobsetsInNamespace(ctx, c, ns); err != nil {
+	if err := DeleteAllJobSetsInNamespace(ctx, c, ns); err != nil {
 		return err
 	}
 	if err := DeleteAllJobsInNamespace(ctx, c, ns); err != nil {
@@ -133,56 +134,62 @@ func DeleteNamespace(ctx context.Context, c client.Client, ns *corev1.Namespace)
 }
 
 func DeleteAllJobsInNamespace(ctx context.Context, c client.Client, ns *corev1.Namespace) error {
-	err := c.DeleteAllOf(ctx, &batchv1.Job{}, client.InNamespace(ns.Name), client.PropagationPolicy(metav1.DeletePropagationBackground))
-	if err != nil && !apierrors.IsNotFound(err) {
-		return err
-	}
-	return nil
+	return deleteAllObjectsInNamespace(ctx, c, ns, &batchv1.Job{})
 }
 
-func DeleteAllJobsetsInNamespace(ctx context.Context, c client.Client, ns *corev1.Namespace) error {
-	err := c.DeleteAllOf(ctx, &jobset.JobSet{}, client.InNamespace(ns.Name), client.PropagationPolicy(metav1.DeletePropagationBackground))
-	if err != nil && !apierrors.IsNotFound(err) && !errors.Is(err, &apimeta.NoKindMatchError{}) {
-		return err
-	}
-	return nil
+func DeleteAllJobSetsInNamespace(ctx context.Context, c client.Client, ns *corev1.Namespace) error {
+	return deleteAllObjectsInNamespace(ctx, c, ns, &jobset.JobSet{})
 }
 
 func DeleteAllMPIJobsInNamespace(ctx context.Context, c client.Client, ns *corev1.Namespace) error {
-	err := c.DeleteAllOf(ctx, &kfmpi.MPIJob{}, client.InNamespace(ns.Name), client.PropagationPolicy(metav1.DeletePropagationBackground))
-	if err != nil && !apierrors.IsNotFound(err) && !errors.Is(err, &apimeta.NoKindMatchError{}) {
-		return err
-	}
-	return nil
+	return deleteAllObjectsInNamespace(ctx, c, ns, &kfmpi.MPIJob{})
 }
 
 func DeleteAllPyTorchJobsInNamespace(ctx context.Context, c client.Client, ns *corev1.Namespace) error {
-	err := c.DeleteAllOf(ctx, &kftraining.PyTorchJob{}, client.InNamespace(ns.Name), client.PropagationPolicy(metav1.DeletePropagationBackground))
+	return deleteAllObjectsInNamespace(ctx, c, ns, &kftraining.PyTorchJob{})
+}
+
+func DeleteAllRayJobsInNamespace(ctx context.Context, c client.Client, ns *corev1.Namespace) error {
+	return deleteAllObjectsInNamespace(ctx, c, ns, &rayv1.RayJob{})
+}
+
+func DeleteAllPodsInNamespace(ctx context.Context, c client.Client, ns *corev1.Namespace) error {
+	return deleteAllPodsInNamespace(ctx, c, ns, 2)
+}
+
+func deleteAllObjectsInNamespace(ctx context.Context, c client.Client, ns *corev1.Namespace, obj client.Object) error {
+	err := c.DeleteAllOf(ctx, obj, client.InNamespace(ns.Name), client.PropagationPolicy(metav1.DeletePropagationBackground))
 	if err != nil && !apierrors.IsNotFound(err) && !errors.Is(err, &apimeta.NoKindMatchError{}) {
 		return err
 	}
 	return nil
 }
 
-func DeleteAllPodsInNamespace(ctx context.Context, c client.Client, ns *corev1.Namespace) error {
+func deleteAllPodsInNamespace(ctx context.Context, c client.Client, ns *corev1.Namespace, offset int) error {
 	if err := client.IgnoreNotFound(c.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(ns.Name))); err != nil {
 		return fmt.Errorf("deleting all Pods in namespace %q: %w", ns.Name, err)
 	}
-
-	gomega.Eventually(func(g gomega.Gomega) {
-		lst := corev1.PodList{}
-		g.Expect(client.IgnoreNotFound(c.List(ctx, &lst, client.InNamespace(ns.Name)))).
+	gomega.EventuallyWithOffset(offset, func(g gomega.Gomega) {
+		pods := corev1.PodList{}
+		g.Expect(client.IgnoreNotFound(c.List(ctx, &pods, client.InNamespace(ns.Name)))).
 			Should(gomega.Succeed(), "listing Pods with a finalizer in namespace %q", ns.Name)
-		for _, p := range lst.Items {
+		for _, p := range pods.Items {
 			if controllerutil.RemoveFinalizer(&p, pod.PodFinalizer) {
 				g.Expect(client.IgnoreNotFound(c.Update(ctx, &p))).Should(gomega.Succeed(), "removing finalizer")
 			}
 		}
 	}, LongTimeout, Interval).Should(gomega.Succeed())
-
 	return nil
 }
 
+func ExpectAllPodsInNamespaceDeleted(ctx context.Context, c client.Client, ns *corev1.Namespace) {
+	pods := corev1.PodList{}
+	gomega.Eventually(func(g gomega.Gomega) {
+		g.Expect(c.List(ctx, &pods, client.InNamespace(ns.Name))).Should(gomega.Succeed())
+		g.Expect(pods.Items).Should(gomega.BeEmpty())
+	}, LongTimeout, Interval).Should(gomega.Succeed())
+}
+
 func DeleteWorkloadsInNamespace(ctx context.Context, c client.Client, ns *corev1.Namespace) error {
 	if err := c.DeleteAllOf(ctx, &kueue.Workload{}, client.InNamespace(ns.Name)); err != nil && !apierrors.IsNotFound(err) {
 		return err


Reviews:
user: login: mimowo
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-06T07:48:21Z
user: login: mimowo
body: LGTM overall, I'm just a but cautious about cherry-picking rbac changes, it is surprising why they would be needed for setting the TopologyAssignment on the workload. It does not seem directly related.
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-06T07:49:36Z
user: login: mimowo
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-06T07:55:48Z
user: login: mszadkow
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-06T12:13:14Z

Review comments:
diff_hunk: @@ -60,7 +60,7 @@ func init() {
 
 // +kubebuilder:rbac:groups="",resources=events,verbs=create;watch;update
 // +kubebuilder:rbac:groups=ray.io,resources=rayjobs,verbs=get;list;watch;update;patch
-// +kubebuilder:rbac:groups=ray.io,resources=rayjobs/status,verbs=get;update
+// +kubebuilder:rbac:groups=ray.io,resources=rayjobs/status,verbs=get;update;patch
path: pkg/controller/jobs/rayjob/rayjob_controller.go
user: login: mimowo
body: Could you please remind me why we increase the rbac here? Also, provide the PR which does it on main. 
created_at: 2025-03-06T07:48:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -60,7 +60,7 @@ func init() {
 
 // +kubebuilder:rbac:groups="",resources=events,verbs=create;watch;update
 // +kubebuilder:rbac:groups=ray.io,resources=rayjobs,verbs=get;list;watch;update;patch
-// +kubebuilder:rbac:groups=ray.io,resources=rayjobs/status,verbs=get;update
+// +kubebuilder:rbac:groups=ray.io,resources=rayjobs/status,verbs=get;update;patch
path: pkg/controller/jobs/rayjob/rayjob_controller.go
user: login: mimowo
body: Ah, ok I synced with @mszadkow the changes were added on the main branch to support MultiKueue, as it requires the ability to patch status for the live status updates. However, Ray via MultiKueue is not supported in release-0.10, so I think we can safely remove cherry-picking the rbac changes.
created_at: 2025-03-06T07:55:48Z
author_association: CONTRIBUTOR
diff_hunk: @@ -60,7 +60,7 @@ func init() {
 
 // +kubebuilder:rbac:groups="",resources=events,verbs=create;watch;update
 // +kubebuilder:rbac:groups=ray.io,resources=rayjobs,verbs=get;list;watch;update;patch
-// +kubebuilder:rbac:groups=ray.io,resources=rayjobs/status,verbs=get;update
+// +kubebuilder:rbac:groups=ray.io,resources=rayjobs/status,verbs=get;update;patch
path: pkg/controller/jobs/rayjob/rayjob_controller.go
user: login: mszadkow
body: They have been removed, ptal
created_at: 2025-03-06T12:13:14Z
author_association: CONTRIBUTOR

PR comments:
user: login: netlify[bot]
created_at: 2025-03-05T11:29:31Z
body: ### <span aria-hidden="true">✅</span> Deploy Preview for *kubernetes-sigs-kueue* canceled.


|  Name | Link |
|:-:|------------------------|
|<span aria-hidden="true">🔨</span> Latest commit | f2237fff581659bc776b301b4c16cc6108cf39be |
|<span aria-hidden="true">🔍</span> Latest deploy log | https://app.netlify.com/sites/kubernetes-sigs-kueue/deploys/67c9562c9caee1000862c244 |
author_association: NONE
user: login: mimowo
created_at: 2025-03-06T15:58:19Z
body: /lgtm
/approve
Thanks!
author_association: CONTRIBUTOR
user: login: k8s-ci-robot
created_at: 2025-03-06T15:58:26Z
body: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/kubernetes-sigs/kueue/pull/4502#issuecomment-2704261718" title="Approved">mimowo</a>*, *<a href="https://github.com/kubernetes-sigs/kueue/pull/4502#" title="Author self-approved">mszadkow</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=kubernetes-sigs%2Fkueue).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/kubernetes-sigs/kueue/blob/release-0.10/OWNERS)~~ [mimowo]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
author_association: CONTRIBUTOR
user: login: k8s-ci-robot
created_at: 2025-03-06T15:58:26Z
body: LGTM label has been added.  <details>Git tree hash: 8c8754dcb4ebd505838a5c155d4e4b78b77543f5</details>
author_association: CONTRIBUTOR
user: login: tenzen-y
created_at: 2025-03-07T19:42:40Z
body: /release-note-edit
```release-note
Fix the bug which didn't respect the TAS annotations on the template for the Ray submitter Job.
```

author_association: MEMBER
user: login: tenzen-y
created_at: 2025-03-07T20:13:56Z
body: /release-note-edit

```release-note
TAS: Fix the bug which didn't respect the TAS annotations on the template for the Ray submitter Job.
```
author_association: MEMBER
user: login: mimowo
created_at: 2025-03-19T11:29:01Z
body: /release-note-edit
```release-note
TAS: Do not ignore the TAS annotation if set on the template for the Ray submitter Job.
```
/kind bug
author_association: CONTRIBUTOR