Repository: kubernetes-sigs/kueue

Issue:
Flaky Test: TopologyAwareScheduling for RayJob when Creating a RayJob Should place pods based on the ranks-ordering
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!

If the matter is security related, please disclose it privately via https://kubernetes.io/security/
-->


**What happened**:
Failed the `End To End TAS Suite: kindest/node:v1.31.1: [It] TopologyAwareScheduling for RayJob when Creating a RayJob Should place pods based on the ranks-ordering` in periodic CI

```shell
{Timed out after 300.000s.
The function passed to Eventually failed at /home/prow/go/src/kubernetes-sigs/kueue/test/e2e/tas/rayjob_test.go:203 with:
Expected
    <[]v1.Pod | len:4, cap:4>: [
        {
            TypeMeta: {Kind: "", APIVersion: ""},
            ObjectMeta: {
                Name: "ranks-ray-raycluster-gpzrs-head-mm657",
                GenerateName: "ranks-ray-raycluster-gpzrs-head-",
                Namespace: "e2e-tas-rayjob-lgmbj",
                SelfLink: "",
                UID: "cbf00188-3d57-4dce-a72c-3c6827775792",
                ResourceVersion: "4265",
                Generation: 0,
                CreationTimestamp: {
                    Time: 2025-03-05T22:40:41Z,
                },
                DeletionTimestamp: nil,
                DeletionGracePeriodSeconds: nil,
                Labels: {
                    "kueue.x-k8s.io/tas": "true",
                    "ray.io/group": "headgroup",
                    "ray.io/identifier": "ranks-ray-raycluster-gpzrs-head",
                    "ray.io/is-ray-node": "yes",
                    "app.kubernetes.io/created-by": "kuberay-operator",
                    "kueue.x-k8s.io/podset": "head",
                    "ray.io/cluster": "ranks-ray-raycluster-gpzrs",
                    "ray.io/node-type": "head",
                    "app.kubernetes.io/name": "kuberay",
                },
                Annotations: {
                    "kueue.x-k8s.io/podset-preferred-topology": "cloud.provider.com/topology-rack",
                    "kueue.x-k8s.io/workload": "rayjob-ranks-ray-8120c",
                    "ray.io/ft-enabled": "false",
                },
                OwnerReferences: [
                    {
                        APIVersion: "ray.io/v1",
                        Kind: "RayCluster",
                        Name: "ranks-ray-raycluster-gpzrs",
                        UID: "0616b0a6-e920-481b-9516-db1f0880f2b1",
                        Controller: true,
                        BlockOwnerDeletion: true,
                    },
                ],
                Finalizers: nil,
                ManagedFields: [
                    {
                        Manager: "kuberay-operator",
                        Operation: "Update",
                        APIVersion: "v1",
                        Time: {
                            Time: 2025-03-05T22:40:41Z,
                        },
                        FieldsType: "FieldsV1",
                        FieldsV1: {
                            Raw: "{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:kueue.x-k8s.io/podset-preferred-topology\":{},\"f:kueue.x-k8s.io/workload\":{},\"f:ray.io/ft-enabled\":{}},\"f:generateName\":{},\"f:labels\":{\".\":{},\"f:app.kubernetes.io/created-by\":{},\"f:app.kubernetes.io/name\":{},\"f:kueue.x-k8s.io/podset\":{},\"f:kueue.x-k8s.io/tas\":{},\"f:ray.io/cluster\":{},\"f:ray.io/group\":{},\"f:ray.io/identifier\":{},\"f:ray.io/is-ray-node\":{},\"f:ray.io/node-type\":{}},\"f:ownerReferences\":{\".\":{},\"k:{\\\"uid\\\":\\\"0616b0a6-e920-481b-9516-db1f0880f2b1\\\"}\":{}}},\"f:spec\":{\"f:containers\":{\"k:{\\\"name\\\":\\\"head-container\\\"}\":{\".\":{},\"f:args\":{},\"f:command\":{},\"f:env\":{\".\":{},\"k:{\\\"name\\\":\\\"KUBERAY_GEN_RAY_START_CMD\\\"}\":{\".\":{},\"f:name\":{},\"f:value\":{}},\"k:{\\\"name\\\":\\\"RAY_ADDRESS\\\"}\":{\".\":{},\"f:name\":{},\"f:value\":{}},\"k:{\\\"name\\\":\\\"RAY_CLOUD_INSTANCE_ID\\\"}\":{\".\":{},\"f:name\":{},\"f:valueFrom\":{\".\":{},\"f:fieldRef\":{}}},\"k:{\\\"name\\\":\\\"RAY_CLUSTER_NAME\\\"}\":{\".\":{},\"f:name\":{},\"f:valueFrom\":{\".\":{},\"f:fieldRef\":{}}},\"k:{\\\"name\\\":\\\"RAY_DASHBOARD_ENABLE_K8S_DISK_USAGE\\\"}\":{\".\":{},\"f:name\":{},\"f:value\":{}},\"k:{\\\"name\\\":\\\"RAY_NODE_TYPE_NAME\\\"}\":{\".\":{},\"f:name\":{},\"f:valueFrom\":{\".\":{},\"f:fieldRef\":{}}},\"k:{\\\"name\\\":\\\"RAY_PORT\\\"}\":{\".\":{},\"f:name\":{},\"f:value\":{}},\"k:{\\\"name\\\":\\\"RAY_USAGE_STATS_EXTRA_TAGS\\\"}\":{\".\":{},\"f:name\":{},\"f:value\":{}},\"k:{\\\"name\\\":\\\"RAY_USAGE_STATS_KUBERAY_IN_USE\\\"}\":{\".\":{},\"f:name\":{},\"f:value\":{}},\"k:{\\\"name\\\":\\\"REDIS_PASSWORD\\\"}\":{\".\":{},\"f...

Gomega truncated this representation as it exceeds 'format.MaxLength'.
Consider having the object provide a custom 'GomegaStringer' representation
or adjust the parameters in Gomega's 'format' package.

Learn more here: https://onsi.github.io/gomega/#adjusting-output

to have length 5 failed [FAILED] Timed out after 300.000s.
The function passed to Eventually failed at /home/prow/go/src/kubernetes-sigs/kueue/test/e2e/tas/rayjob_test.go:203 with:
Expected
    <[]v1.Pod | len:4, cap:4>: [
        {
            TypeMeta: {Kind: "", APIVersion: ""},
            ObjectMeta: {
                Name: "ranks-ray-raycluster-gpzrs-head-mm657",
                GenerateName: "ranks-ray-raycluster-gpzrs-head-",
                Namespace: "e2e-tas-rayjob-lgmbj",
                SelfLink: "",
                UID: "cbf00188-3d57-4dce-a72c-3c6827775792",
                ResourceVersion: "4265",
                Generation: 0,
                CreationTimestamp: {
                    Time: 2025-03-05T22:40:41Z,
                },
                DeletionTimestamp: nil,
                DeletionGracePeriodSeconds: nil,
                Labels: {
                    "kueue.x-k8s.io/tas": "true",
                    "ray.io/group": "headgroup",
                    "ray.io/identifier": "ranks-ray-raycluster-gpzrs-head",
                    "ray.io/is-ray-node": "yes",
                    "app.kubernetes.io/created-by": "kuberay-operator",
                    "kueue.x-k8s.io/podset": "head",
                    "ray.io/cluster": "ranks-ray-raycluster-gpzrs",
                    "ray.io/node-type": "head",
                    "app.kubernetes.io/name": "kuberay",
                },
                Annotations: {
                    "kueue.x-k8s.io/podset-preferred-topology": "cloud.provider.com/topology-rack",
                    "kueue.x-k8s.io/workload": "rayjob-ranks-ray-8120c",
                    "ray.io/ft-enabled": "false",
                },
                OwnerReferences: [
                    {
                        APIVersion: "ray.io/v1",
                        Kind: "RayCluster",
                        Name: "ranks-ray-raycluster-gpzrs",
                        UID: "0616b0a6-e920-481b-9516-db1f0880f2b1",
                        Controller: true,
                        BlockOwnerDeletion: true,
                    },
                ],
                Finalizers: nil,
                ManagedFields: [
                    {
                        Manager: "kuberay-operator",
                        Operation: "Update",
                        APIVersion: "v1",
                        Time: {
                            Time: 2025-03-05T22:40:41Z,
                        },
                        FieldsType: "FieldsV1",
                        FieldsV1: {
                            Raw: "{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:kueue.x-k8s.io/podset-preferred-topology\":{},\"f:kueue.x-k8s.io/workload\":{},\"f:ray.io/ft-enabled\":{}},\"f:generateName\":{},\"f:labels\":{\".\":{},\"f:app.kubernetes.io/created-by\":{},\"f:app.kubernetes.io/name\":{},\"f:kueue.x-k8s.io/podset\":{},\"f:kueue.x-k8s.io/tas\":{},\"f:ray.io/cluster\":{},\"f:ray.io/group\":{},\"f:ray.io/identifier\":{},\"f:ray.io/is-ray-node\":{},\"f:ray.io/node-type\":{}},\"f:ownerReferences\":{\".\":{},\"k:{\\\"uid\\\":\\\"0616b0a6-e920-481b-9516-db1f0880f2b1\\\"}\":{}}},\"f:spec\":{\"f:containers\":{\"k:{\\\"name\\\":\\\"head-container\\\"}\":{\".\":{},\"f:args\":{},\"f:command\":{},\"f:env\":{\".\":{},\"k:{\\\"name\\\":\\\"KUBERAY_GEN_RAY_START_CMD\\\"}\":{\".\":{},\"f:name\":{},\"f:value\":{}},\"k:{\\\"name\\\":\\\"RAY_ADDRESS\\\"}\":{\".\":{},\"f:name\":{},\"f:value\":{}},\"k:{\\\"name\\\":\\\"RAY_CLOUD_INSTANCE_ID\\\"}\":{\".\":{},\"f:name\":{},\"f:valueFrom\":{\".\":{},\"f:fieldRef\":{}}},\"k:{\\\"name\\\":\\\"RAY_CLUSTER_NAME\\\"}\":{\".\":{},\"f:name\":{},\"f:valueFrom\":{\".\":{},\"f:fieldRef\":{}}},\"k:{\\\"name\\\":\\\"RAY_DASHBOARD_ENABLE_K8S_DISK_USAGE\\\"}\":{\".\":{},\"f:name\":{},\"f:value\":{}},\"k:{\\\"name\\\":\\\"RAY_NODE_TYPE_NAME\\\"}\":{\".\":{},\"f:name\":{},\"f:valueFrom\":{\".\":{},\"f:fieldRef\":{}}},\"k:{\\\"name\\\":\\\"RAY_PORT\\\"}\":{\".\":{},\"f:name\":{},\"f:value\":{}},\"k:{\\\"name\\\":\\\"RAY_USAGE_STATS_EXTRA_TAGS\\\"}\":{\".\":{},\"f:name\":{},\"f:value\":{}},\"k:{\\\"name\\\":\\\"RAY_USAGE_STATS_KUBERAY_IN_USE\\\"}\":{\".\":{},\"f:name\":{},\"f:value\":{}},\"k:{\\\"name\\\":\\\"REDIS_PASSWORD\\\"}\":{\".\":{},\"f...

Gomega truncated this representation as it exceeds 'format.MaxLength'.
Consider having the object provide a custom 'GomegaStringer' representation
or adjust the parameters in Gomega's 'format' package.

Learn more here: https://onsi.github.io/gomega/#adjusting-output

to have length 5
In [It] at: /home/prow/go/src/kubernetes-sigs/kueue/test/e2e/tas/rayjob_test.go:207 @ 03/05/25 22:45:41.951
}
```

**What you expected to happen**:
No errors

**How to reproduce it (as minimally and precisely as possible)**:

https://prow.k8s.io/view/gs/kubernetes-ci-logs/logs/periodic-kueue-test-e2e-tas-main/1897412525620727808

<img width="1261" alt="Image" src="https://github.com/user-attachments/assets/514c7460-678d-4d9c-b04f-c6df0a12789a" />

**Anything else we need to know?**:

**Environment**:
- Kubernetes version (use `kubectl version`):
- Kueue version (use `git describe --tags --dirty --always`):
- Cloud provider or hardware configuration:
- OS (e.g: `cat /etc/os-release`):
- Kernel (e.g. `uname -a`):
- Install tools:
- Others:

Hints from issue comments:
@mbobrovskyi: cc: @mszadkow @mimowo: btw, the error log is truncated. I suppose one of the pods failed for some reason (and was replaced - this is why we had 5 pods). To investigate this one could follow steps similar to https://github.com/kubernetes-sigs/kueue/issues/4495. 

However, another improvement would be to increase the MaxLength for the output - currently it is too short to fit even a single Pod with status - and the status would be the most relevant case here. 

I will open a dedicated issue for that.@mimowo: Opened: https://github.com/kubernetes-sigs/kueue/issues/4509@mszadkow: /assign@mszadkow: This timeout again after the increase to 5 minutes...that's weird, we need to have better insight when it fails, agree@mimowo: I think the key thing is that for some reason there was 5 pods, so probably one failed and was replaced. Try to identify the failed pod, then inspect its container logs, and logs of kubelet related to the pod (assuming the pod failure is the reason here).@mimowo: This seems like another occurrence, but on 0.10 branch: https://prow.k8s.io/view/gs/kubernetes-ci-logs/logs/periodic-kueue-test-e2e-tas-release-0-10/1898862090471346176@mszadkow: Ack, actively investigating this@mimowo: fyi, one more: https://prow.k8s.io/view/gs/kubernetes-ci-logs/pr-logs/pull/kubernetes-sigs_kueue/4524/pull-kueue-test-e2e-tas-main/1899040138269298688. Seems quite often recently@mbobrovskyi: one more: https://prow.k8s.io/view/gs/kubernetes-ci-logs/pr-logs/pull/kubernetes-sigs_kueue/4585/pull-kueue-test-e2e-tas-main/1900113461916995584@mszadkow: For this one I think we might want to change the approach.
This submitter job created by rayJob might be short lived and fall into the gap of interval.
What if we check ray cluster nodes, but later verify rayJob status, that should change after the submitter is finished anyway.
This could make it less flaky for tests while we still prove sanity of the RayJob with tas@mimowo: I looked into the Ray failures too

First, the tests fail expecting 5 pods, but they only observe 4. Looking at the API server logs we can confirm that RayOperator only created 4:

```
> cat api.log | grep e2e-tas-rayjob-lgmbj/pods | grep POST | grep 201
2025-03-05T22:40:41.977203895Z stderr F I0305 22:40:41.976983       1 httplog.go:134] "HTTP" verb="POST" URI="/api/v1/namespaces/e2e-tas-rayjob-lgmbj/pods" latency="11.586316ms" userAgent="kuberay-operator/v1.2.2" audit-ID="33e27fd2-d22d-4c4d-9b46-d4efeca06f05" srcIP="172.18.0.5:37344" apf_pl="workload-low" apf_fs="service-accounts" apf_iseats=1 apf_fseats=2 apf_additionalLatency="5ms" apf_execution_time="11.024848ms" resp=201
2025-03-05T22:40:42.025410431Z stderr F I0305 22:40:42.025081       1 httplog.go:134] "HTTP" verb="POST" URI="/api/v1/namespaces/e2e-tas-rayjob-lgmbj/pods" latency="8.426362ms" userAgent="kuberay-operator/v1.2.2" audit-ID="4d674538-ad9c-45f4-b055-dff0bd1db0a2" srcIP="172.18.0.5:37344" apf_pl="workload-low" apf_fs="service-accounts" apf_iseats=1 apf_fseats=2 apf_additionalLatency="5ms" apf_execution_time="8.135949ms" resp=201
2025-03-05T22:40:42.035689969Z stderr F I0305 22:40:42.035397       1 httplog.go:134] "HTTP" verb="POST" URI="/api/v1/namespaces/e2e-tas-rayjob-lgmbj/pods" latency="8.717217ms" userAgent="kuberay-operator/v1.2.2" audit-ID="42111a1c-315b-47bc-a77a-aeefaa2310c2" srcIP="172.18.0.5:37344" apf_pl="workload-low" apf_fs="service-accounts" apf_iseats=1 apf_fseats=2 apf_additionalLatency="5ms" apf_execution_time="8.339393ms" resp=201
2025-03-05T22:40:42.124730152Z stderr F I0305 22:40:42.124491       1 httplog.go:134] "HTTP" verb="POST" URI="/api/v1/namespaces/e2e-tas-rayjob-lgmbj/pods" latency="8.2536ms" userAgent="kuberay-operator/v1.2.2" audit-ID="fab4d7eb-ff9a-40cc-8cb1-d75d56f50c29" srcIP="172.18.0.5:37344" apf_pl="workload-low" apf_fs="service-accounts" apf_iseats=1 apf_fseats=2 apf_additionalLatency="5ms" apf_execution_time="7.949246ms" resp=201
2025-03-05T22:40:42.999711374Z stderr F I0305 22:40:42.999502       1 httplog.go:134] "HTTP" verb="POST" URI="/api/v1/namespaces/e2e-tas-rayjob-lgmbj/pods/ranks-ray-raycluster-gpzrs-head-mm657/binding" latency="3.6966ms" userAgent="kube-scheduler/v1.31.1 (linux/amd64) kubernetes/948afe5/scheduler" audit-ID="1af828d9-206a-4a8a-babe-407f9529889f" srcIP="172.18.0.4:40688" apf_pl="workload-high" apf_fs="kube-scheduler" apf_iseats=1 apf_fseats=2 apf_additionalLatency="5ms" apf_execution_time="3.282724ms" resp=201
2025-03-05T22:40:42.999746864Z stderr F I0305 22:40:42.999677       1 httplog.go:134] "HTTP" verb="POST" URI="/api/v1/namespaces/e2e-tas-rayjob-lgmbj/pods/ranks-ray-raycluster-gpzrs-workers-group-0-worker-d97p4/binding" latency="3.335175ms" userAgent="kube-scheduler/v1.31.1 (linux/amd64) kubernetes/948afe5/scheduler" audit-ID="402af477-ddef-48ea-9fcf-19f58cf7eb3f" srcIP="172.18.0.4:40688" apf_pl="workload-high" apf_fs="kube-scheduler" apf_iseats=1 apf_fseats=2 apf_additionalLatency="5ms" apf_execution_time="2.960229ms" resp=201
2025-03-05T22:40:43.000747168Z stderr F I0305 22:40:43.000337       1 httplog.go:134] "HTTP" verb="POST" URI="/api/v1/namespaces/e2e-tas-rayjob-lgmbj/pods/ranks-ray-raycluster-gpzrs-workers-group-0-worker-4g6fc/binding" latency="2.859619ms" userAgent="kube-scheduler/v1.31.1 (linux/amd64) kubernetes/948afe5/scheduler" audit-ID="fe2da5fa-309d-4575-91d1-40d63398db2b" srcIP="172.18.0.4:40688" apf_pl="workload-high" apf_fs="kube-scheduler" apf_iseats=1 apf_fseats=2 apf_additionalLatency="5ms" apf_execution_time="2.503914ms" resp=201
2025-03-05T22:40:43.002457031Z stderr F I0305 22:40:43.002218       1 httplog.go:134] "HTTP" verb="POST" URI="/api/v1/namespaces/e2e-tas-rayjob-lgmbj/pods/ranks-ray-raycluster-gpzrs-workers-group-0-worker-zqfwn/binding" latency="4.112935ms" userAgent="kube-scheduler/v1.31.1 (linux/amd64) kubernetes/948afe5/scheduler" audit-ID="54c48759-ad7b-4b77-b91e-a565ded1429b" srcIP="172.18.0.4:40688" apf_pl="workload-high" apf_fs="kube-scheduler" apf_iseats=1 apf_fseats=2 apf_additionalLatency="5ms" apf_execution_time="3.769201ms" resp=201
```
Also `> cat api.log | grep e2e-tas-rayjob-lgmbj/jobs | grep POST | grep 201` returns empty indicating the Submitter Job was never created.

To give more context I grepped the kuberay operator logs which show some errors:
```
> cat ray.log | grep e2e-tas-rayjob-lgmbj               
2025-03-05T22:40:41.691025738Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.690Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"5a22cd41-3f27-4aa3-8a54-d8e1735be202","name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
2025-03-05T22:40:41.691074429Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.690Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"5a22cd41-3f27-4aa3-8a54-d8e1735be202","finalizer":"ray.io/rayjob-finalizer"}
2025-03-05T22:40:41.730733001Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.730Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"5a22cd41-3f27-4aa3-8a54-d8e1735be202","RayJob":"ranks-ray"}
2025-03-05T22:40:41.730743501Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.730Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"5a22cd41-3f27-4aa3-8a54-d8e1735be202","shouldUpdateStatus":true,"RayJob":"ranks-ray","jobId":"","rayClusterName":"","jobStatus":""}
2025-03-05T22:40:41.730750971Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.730Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"5a22cd41-3f27-4aa3-8a54-d8e1735be202","oldRayJobStatus":{"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"jobId":"ranks-ray-cmmqx","rayClusterName":"ranks-ray-raycluster-sr57x","jobDeploymentStatus":"Initializing","startTime":"2025-03-05T22:40:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
2025-03-05T22:40:41.730781381Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.730Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"5a22cd41-3f27-4aa3-8a54-d8e1735be202","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
2025-03-05T22:40:41.74035455Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.740Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"82ef785a-2582-44f0-87d6-d131e2ddee90","name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
2025-03-05T22:40:41.74038324Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.740Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"82ef785a-2582-44f0-87d6-d131e2ddee90","RayJob":"ranks-ray"}
2025-03-05T22:40:41.74039279Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.740Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"82ef785a-2582-44f0-87d6-d131e2ddee90","shouldUpdateStatus":true,"RayJob":"ranks-ray","jobId":"","rayClusterName":"","jobStatus":""}
2025-03-05T22:40:41.740412Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.740Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"82ef785a-2582-44f0-87d6-d131e2ddee90","oldRayJobStatus":{"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"jobId":"ranks-ray-qcdv2","rayClusterName":"ranks-ray-raycluster-669x2","jobDeploymentStatus":"Initializing","startTime":"2025-03-05T22:40:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
2025-03-05T22:40:41.74041605Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.740Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"82ef785a-2582-44f0-87d6-d131e2ddee90","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
2025-03-05T22:40:41.747129491Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.746Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"82ef785a-2582-44f0-87d6-d131e2ddee90","error":"Operation cannot be fulfilled on rayjobs.ray.io \"ranks-ray\": the object has been modified; please apply your changes to the latest version and try again"}
2025-03-05T22:40:41.747149781Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.747Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"82ef785a-2582-44f0-87d6-d131e2ddee90"}
2025-03-05T22:40:41.747178671Z stderr F {"level":"error","ts":"2025-03-05T22:40:41.747Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"82ef785a-2582-44f0-87d6-d131e2ddee90","error":"Operation cannot be fulfilled on rayjobs.ray.io \"ranks-ray\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.5/pkg/internal/controller/controller.go:329\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.5/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.5/pkg/internal/controller/controller.go:227"}
2025-03-05T22:40:41.747340263Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.747Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"9ea0d50f-11cb-4f19-b0f9-67d9fae463b5","name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
2025-03-05T22:40:41.747351023Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.747Z","logger":"controllers.RayJob","msg":"Try to transition the status from `Initializing` to `Suspending`","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"9ea0d50f-11cb-4f19-b0f9-67d9fae463b5","RayJob":"ranks-ray"}
2025-03-05T22:40:41.747357754Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.747Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"9ea0d50f-11cb-4f19-b0f9-67d9fae463b5","oldRayJobStatus":{"jobId":"ranks-ray-cmmqx","rayClusterName":"ranks-ray-raycluster-sr57x","jobDeploymentStatus":"Initializing","startTime":"2025-03-05T22:40:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"jobId":"ranks-ray-cmmqx","rayClusterName":"ranks-ray-raycluster-sr57x","jobDeploymentStatus":"Suspending","startTime":"2025-03-05T22:40:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
2025-03-05T22:40:41.747373524Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.747Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"9ea0d50f-11cb-4f19-b0f9-67d9fae463b5","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Suspending"}
2025-03-05T22:40:41.816458069Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.816Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"1b4aac45-a7ca-4c4e-937d-db5893fa9ce8","name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
2025-03-05T22:40:41.81648789Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.816Z","logger":"controllers.RayJob","msg":"Try to transition the status from `Initializing` to `Suspending`","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"1b4aac45-a7ca-4c4e-937d-db5893fa9ce8","RayJob":"ranks-ray"}
2025-03-05T22:40:41.81649374Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.816Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"1b4aac45-a7ca-4c4e-937d-db5893fa9ce8","oldRayJobStatus":{"jobId":"ranks-ray-cmmqx","rayClusterName":"ranks-ray-raycluster-sr57x","jobDeploymentStatus":"Initializing","startTime":"2025-03-05T22:40:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"jobId":"ranks-ray-cmmqx","rayClusterName":"ranks-ray-raycluster-sr57x","jobDeploymentStatus":"Suspending","startTime":"2025-03-05T22:40:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
2025-03-05T22:40:41.81649839Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.816Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"1b4aac45-a7ca-4c4e-937d-db5893fa9ce8","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Suspending"}
2025-03-05T22:40:41.82394229Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.823Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"1b4aac45-a7ca-4c4e-937d-db5893fa9ce8","error":"Operation cannot be fulfilled on rayjobs.ray.io \"ranks-ray\": the object has been modified; please apply your changes to the latest version and try again"}
2025-03-05T22:40:41.82396275Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.823Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"1b4aac45-a7ca-4c4e-937d-db5893fa9ce8"}
2025-03-05T22:40:41.823982881Z stderr F {"level":"error","ts":"2025-03-05T22:40:41.823Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"1b4aac45-a7ca-4c4e-937d-db5893fa9ce8","error":"Operation cannot be fulfilled on rayjobs.ray.io \"ranks-ray\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.5/pkg/internal/controller/controller.go:329\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.5/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.5/pkg/internal/controller/controller.go:227"}
2025-03-05T22:40:41.824197244Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.824Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"f07521e0-7c67-4257-b8d3-8dc5f4dea4f5","name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
2025-03-05T22:40:41.824208124Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.824Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"f07521e0-7c67-4257-b8d3-8dc5f4dea4f5","RayCluster":{"name":"ranks-ray-raycluster-sr57x","namespace":"e2e-tas-rayjob-lgmbj"},"RayJob":"ranks-ray"}
2025-03-05T22:40:41.824212334Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.824Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"f07521e0-7c67-4257-b8d3-8dc5f4dea4f5","isClusterDeleted":true}
2025-03-05T22:40:41.824235294Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.824Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"f07521e0-7c67-4257-b8d3-8dc5f4dea4f5","RayJob":"ranks-ray","Kubernetes Job":""}
2025-03-05T22:40:41.824245074Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.824Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"f07521e0-7c67-4257-b8d3-8dc5f4dea4f5","isJobDeleted":true}
2025-03-05T22:40:41.824268085Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.824Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"f07521e0-7c67-4257-b8d3-8dc5f4dea4f5","oldRayJobStatus":{"jobId":"ranks-ray-cmmqx","rayClusterName":"ranks-ray-raycluster-sr57x","jobDeploymentStatus":"Suspending","startTime":"2025-03-05T22:40:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"jobDeploymentStatus":"Suspended","startTime":"2025-03-05T22:40:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
2025-03-05T22:40:41.824272795Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.824Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"f07521e0-7c67-4257-b8d3-8dc5f4dea4f5","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspending","new JobDeploymentStatus":"Suspended"}
2025-03-05T22:40:41.915895512Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.915Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"e3781e9d-5a57-4d4c-b995-ee8a346246e1","name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
2025-03-05T22:40:41.915933503Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.915Z","logger":"controllers.RayJob","msg":"The status is 'Suspended', but the suspend flag is false. Transition the status to 'New'.","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"e3781e9d-5a57-4d4c-b995-ee8a346246e1"}
2025-03-05T22:40:41.915943313Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.915Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"e3781e9d-5a57-4d4c-b995-ee8a346246e1","oldRayJobStatus":{"jobDeploymentStatus":"Suspended","startTime":"2025-03-05T22:40:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"startTime":"2025-03-05T22:40:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
2025-03-05T22:40:41.915947083Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.915Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"e3781e9d-5a57-4d4c-b995-ee8a346246e1","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspended","new JobDeploymentStatus":""}
2025-03-05T22:40:41.928126647Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.927Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"d716e4ba-f93f-427b-935c-dfae9eec7d52","name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
2025-03-05T22:40:41.928151017Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.927Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"d716e4ba-f93f-427b-935c-dfae9eec7d52","RayJob":"ranks-ray"}
2025-03-05T22:40:41.928156777Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.927Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"d716e4ba-f93f-427b-935c-dfae9eec7d52","shouldUpdateStatus":true,"RayJob":"ranks-ray","jobId":"","rayClusterName":"","jobStatus":""}
2025-03-05T22:40:41.928162858Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.927Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"d716e4ba-f93f-427b-935c-dfae9eec7d52","oldRayJobStatus":{"startTime":"2025-03-05T22:40:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"jobId":"ranks-ray-vzvd2","rayClusterName":"ranks-ray-raycluster-gpzrs","jobDeploymentStatus":"Initializing","startTime":"2025-03-05T22:40:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
2025-03-05T22:40:41.928183918Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.928Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"d716e4ba-f93f-427b-935c-dfae9eec7d52","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
2025-03-05T22:40:41.939842544Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.939Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"ace14a20-d46a-48ce-b78b-e792f4a93afc","name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
2025-03-05T22:40:41.939879045Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.939Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"ace14a20-d46a-48ce-b78b-e792f4a93afc","name":"ranks-ray-raycluster-gpzrs"}
2025-03-05T22:40:41.939882965Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.939Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"ace14a20-d46a-48ce-b78b-e792f4a93afc","RayJob":"ranks-ray","RayCluster":{"name":"ranks-ray-raycluster-gpzrs","namespace":"e2e-tas-rayjob-lgmbj"}}
2025-03-05T22:40:41.939886725Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.939Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"ace14a20-d46a-48ce-b78b-e792f4a93afc","RayCluster":{"name":"ranks-ray-raycluster-gpzrs","namespace":"e2e-tas-rayjob-lgmbj"}}
2025-03-05T22:40:41.959965164Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.959Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"ace14a20-d46a-48ce-b78b-e792f4a93afc","RayJob":"ranks-ray","RayCluster":{"name":"ranks-ray-raycluster-gpzrs","namespace":"e2e-tas-rayjob-lgmbj"}}
2025-03-05T22:40:41.960283748Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.960Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"ace14a20-d46a-48ce-b78b-e792f4a93afc","RayCluster":"ranks-ray-raycluster-gpzrs","State":""}
2025-03-05T22:40:41.96039025Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.960Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ranks-ray-raycluster-gpzrs","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"40c15835-bf09-4127-bb0b-cbc807ce37c2"}
2025-03-05T22:40:41.960543902Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.960Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"efb844e5-185c-4b5a-9702-eaf3d34be173","name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
2025-03-05T22:40:41.960555872Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.960Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"efb844e5-185c-4b5a-9702-eaf3d34be173","name":"ranks-ray-raycluster-gpzrs"}
2025-03-05T22:40:41.960561152Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.960Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"efb844e5-185c-4b5a-9702-eaf3d34be173","RayJob":"ranks-ray","RayCluster":{"name":"ranks-ray-raycluster-gpzrs","namespace":"e2e-tas-rayjob-lgmbj"}}
2025-03-05T22:40:41.960836436Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.960Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ranks-ray","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"efb844e5-185c-4b5a-9702-eaf3d34be173","RayCluster":"ranks-ray-raycluster-gpzrs","State":""}
2025-03-05T22:40:41.964783758Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.964Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"ranks-ray-raycluster-gpzrs","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"40c15835-bf09-4127-bb0b-cbc807ce37c2","name":"ranks-ray-raycluster-gpzrs-head-svc"}
2025-03-05T22:40:41.964833089Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.964Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ranks-ray-raycluster-gpzrs","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"40c15835-bf09-4127-bb0b-cbc807ce37c2","Found 0 head Pods; creating a head Pod for the RayCluster.":"ranks-ray-raycluster-gpzrs"}
2025-03-05T22:40:41.964956361Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.964Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ranks-ray-raycluster-gpzrs","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"40c15835-bf09-4127-bb0b-cbc807ce37c2","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","kueue.x-k8s.io/podset":"head","kueue.x-k8s.io/tas":"true","ray.io/cluster":"ranks-ray-raycluster-gpzrs","ray.io/group":"headgroup","ray.io/identifier":"ranks-ray-raycluster-gpzrs-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
2025-03-05T22:40:41.964969311Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.964Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ranks-ray-raycluster-gpzrs","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"40c15835-bf09-4127-bb0b-cbc807ce37c2","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"100m"},"requests":{"cpu":"100m"}}}
2025-03-05T22:40:41.964974551Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.964Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ranks-ray-raycluster-gpzrs","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"40c15835-bf09-4127-bb0b-cbc807ce37c2","rayStartCmd":"ray start --head  --num-cpus=1  --dashboard-host=0.0.0.0  --metrics-export-port=8080  --block  --dashboard-agent-listen-port=52365 "}
2025-03-05T22:40:41.964990721Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.964Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ranks-ray-raycluster-gpzrs","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"40c15835-bf09-4127-bb0b-cbc807ce37c2","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --num-cpus=1  --dashboard-host=0.0.0.0  --metrics-export-port=8080  --block  --dashboard-agent-listen-port=52365 "}
2025-03-05T22:40:41.965002051Z stderr F {"level":"info","ts":"2025-03-05T22:40:41.964Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ranks-ray-raycluster-gpzrs","namespace":"e2e-tas-rayjob-lgmbj"},"reconcileID":"40c15835-bf09-4127-bb0b-cbc807ce37c2","enabled":true}
```

So some interesting logs are **The associated RayCluster for RayJob has been already deleted and it can not be found**, or **The submitter Kubernetes Job has been already deleted** at `2025-03-05T22:40:41.824` even though the kube-apiserver logs don't show any DELETE requests to the Ray APIs in this perdiod. Maybe there is some racing with the creation events not being delivered on time?

So, I expect this is actually a bug / flake in KubeRay 1.2.2. 

cc @andrewsykim have you seen something like this? If so, is this fixed in 1.3.x?@mimowo: cc @tenzen-y @tenzen-y: Thank you for investigating this one. If this actually KubeRay side bug, we can skip this case until this fixes are shipped from KubeRay.
@mimowo: > For this one I think we might want to change the approach.

I synced with @andrewsykim who says the issue looks "vaguely familiar", and there is some chance it is already fixed in 1.3. So instead of refactoring or relaxing tests we can just wait a bit. 

cc @mszadkow @tenzen-y: > > For this one I think we might want to change the approach.
> 
> I synced with [@andrewsykim](https://github.com/andrewsykim) who says the issue looks "vaguely familiar", and there is some chance it is already fixed in 1.3. So instead of refactoring or relaxing tests we can just wait a bit.
> 
> cc [@mszadkow](https://github.com/mszadkow)

Thank you for letting me know.

Solution patch:
diff --git a/apis/config/v1beta1/configuration_types.go b/apis/config/v1beta1/configuration_types.go
index d2484a6d4..6c29d467e 100644
--- a/apis/config/v1beta1/configuration_types.go
+++ b/apis/config/v1beta1/configuration_types.go
@@ -92,7 +92,7 @@ type Configuration struct {
 	// MultiKueue controls the behaviour of the MultiKueue AdmissionCheck Controller.
 	MultiKueue *MultiKueue `json:"multiKueue,omitempty"`
 
-	// FairSharing controls the fair sharing semantics across the cluster.
+	// FairSharing controls the Fair Sharing semantics across the cluster.
 	FairSharing *FairSharing `json:"fairSharing,omitempty"`
 
 	// Resources provides additional configuration options for handling the resources.
@@ -450,7 +450,7 @@ const (
 )
 
 type FairSharing struct {
-	// enable indicates whether to enable fair sharing for all cohorts.
+	// enable indicates whether to enable Fair Sharing for all cohorts.
 	// Defaults to false.
 	Enable bool `json:"enable"`
 
diff --git a/apis/kueue/v1beta1/clusterqueue_types.go b/apis/kueue/v1beta1/clusterqueue_types.go
index 3f7ba1b28..478b70737 100644
--- a/apis/kueue/v1beta1/clusterqueue_types.go
+++ b/apis/kueue/v1beta1/clusterqueue_types.go
@@ -431,11 +431,11 @@ type ClusterQueuePreemption struct {
 	//   Workloads in the cohort that have lower priority than the pending
 	//   Workload. **Fair Sharing** only preempt Workloads in the cohort that
 	//   have lower priority than the pending Workload and that satisfy the
-	//   fair sharing preemptionStategies.
+	//   Fair Sharing preemptionStategies.
 	// - `Any`: **Classic Preemption** if the pending Workload fits within
 	//    the nominal quota of its ClusterQueue, preempt any Workload in the
 	//    cohort, irrespective of priority. **Fair Sharing** preempt Workloads
-	//    in the cohort that satisfy the fair sharing preemptionStrategies.
+	//    in the cohort that satisfy the Fair Sharing preemptionStrategies.
 	//
 	// +kubebuilder:default=Never
 	// +kubebuilder:validation:Enum=Never;LowerPriority;Any
diff --git a/apis/kueue/v1beta1/fairsharing_types.go b/apis/kueue/v1beta1/fairsharing_types.go
index 9a5b7389b..bb175d975 100644
--- a/apis/kueue/v1beta1/fairsharing_types.go
+++ b/apis/kueue/v1beta1/fairsharing_types.go
@@ -39,7 +39,7 @@ type FairSharing struct {
 	Weight *resource.Quantity `json:"weight,omitempty"`
 }
 
-// fairSharing contains the information about the current status of fair sharing.
+// fairSharing contains the information about the current status of Fair Sharing.
 type FairSharingStatus struct {
 	// WeightedShare represent the maximum of the ratios of usage
 	// above nominal quota to the lendable resources in the
diff --git a/apis/kueue/v1beta1/workload_types.go b/apis/kueue/v1beta1/workload_types.go
index db8a7c2b1..6a717e827 100644
--- a/apis/kueue/v1beta1/workload_types.go
+++ b/apis/kueue/v1beta1/workload_types.go
@@ -524,7 +524,7 @@ const (
 	InCohortReclamationReason string = "InCohortReclamation"
 
 	// InCohortFairSharingReason indicates the Workload was preempted due to
-	// fair sharing within the cohort.
+	// Fair Sharing within the cohort.
 	InCohortFairSharingReason string = "InCohortFairSharing"
 
 	// InCohortReclaimWhileBorrowingReason indicates the Workload was preempted
diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml
index 369792831..50271b502 100644
--- a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml
+++ b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml
@@ -307,11 +307,11 @@ spec:
                         Workloads in the cohort that have lower priority than the pending
                         Workload. **Fair Sharing** only preempt Workloads in the cohort that
                         have lower priority than the pending Workload and that satisfy the
-                        fair sharing preemptionStategies.
+                        Fair Sharing preemptionStategies.
                       - `Any`: **Classic Preemption** if the pending Workload fits within
                          the nominal quota of its ClusterQueue, preempt any Workload in the
                          cohort, irrespective of priority. **Fair Sharing** preempt Workloads
-                         in the cohort that satisfy the fair sharing preemptionStrategies.
+                         in the cohort that satisfy the Fair Sharing preemptionStrategies.
                     enum:
                     - Never
                     - LowerPriority
@@ -583,7 +583,7 @@ spec:
                 x-kubernetes-list-type: map
               fairSharing:
                 description: fairSharing contains the information about the current
-                  status of fair sharing.
+                  status of Fair Sharing.
                 properties:
                   weightedShare:
                     description: |-
diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml
index 784b5c98c..3619a48f8 100644
--- a/charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml
+++ b/charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml
@@ -247,7 +247,7 @@ spec:
             properties:
               fairSharing:
                 description: fairSharing contains the information about the current
-                  status of fair sharing.
+                  status of Fair Sharing.
                 properties:
                   weightedShare:
                     description: |-
diff --git a/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml b/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml
index 337f4585a..962623045 100644
--- a/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml
+++ b/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml
@@ -292,11 +292,11 @@ spec:
                         Workloads in the cohort that have lower priority than the pending
                         Workload. **Fair Sharing** only preempt Workloads in the cohort that
                         have lower priority than the pending Workload and that satisfy the
-                        fair sharing preemptionStategies.
+                        Fair Sharing preemptionStategies.
                       - `Any`: **Classic Preemption** if the pending Workload fits within
                          the nominal quota of its ClusterQueue, preempt any Workload in the
                          cohort, irrespective of priority. **Fair Sharing** preempt Workloads
-                         in the cohort that satisfy the fair sharing preemptionStrategies.
+                         in the cohort that satisfy the Fair Sharing preemptionStrategies.
                     enum:
                     - Never
                     - LowerPriority
@@ -568,7 +568,7 @@ spec:
                 x-kubernetes-list-type: map
               fairSharing:
                 description: fairSharing contains the information about the current
-                  status of fair sharing.
+                  status of Fair Sharing.
                 properties:
                   weightedShare:
                     description: |-
diff --git a/config/components/crd/bases/kueue.x-k8s.io_cohorts.yaml b/config/components/crd/bases/kueue.x-k8s.io_cohorts.yaml
index 80ee76aa3..d56c73ac0 100644
--- a/config/components/crd/bases/kueue.x-k8s.io_cohorts.yaml
+++ b/config/components/crd/bases/kueue.x-k8s.io_cohorts.yaml
@@ -232,7 +232,7 @@ spec:
             properties:
               fairSharing:
                 description: fairSharing contains the information about the current
-                  status of fair sharing.
+                  status of Fair Sharing.
                 properties:
                   weightedShare:
                     description: |-
diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go
index 3249155c1..0b8edf7a1 100644
--- a/pkg/metrics/metrics.go
+++ b/pkg/metrics/metrics.go
@@ -242,7 +242,7 @@ The label 'reason' can have the following values:
 The label 'reason' can have the following values:
 - "InClusterQueue" means that the workload was preempted by a workload in the same ClusterQueue.
 - "InCohortReclamation" means that the workload was preempted by a workload in the same cohort due to reclamation of nominal quota.
-- "InCohortFairSharing" means that the workload was preempted by a workload in the same cohort due to fair sharing.
+- "InCohortFairSharing" means that the workload was preempted by a workload in the same cohort Fair Sharing.
 - "InCohortReclaimWhileBorrowing" means that the workload was preempted by a workload in the same cohort due to reclamation of nominal quota while borrowing.`,
 		}, []string{"preempting_cluster_queue", "reason"},
 	)
diff --git a/pkg/scheduler/preemption/preemption.go b/pkg/scheduler/preemption/preemption.go
index 082c41c71..44f2593a3 100644
--- a/pkg/scheduler/preemption/preemption.go
+++ b/pkg/scheduler/preemption/preemption.go
@@ -206,7 +206,7 @@ func canBorrowWithinCohort(preemptionCtx *preemptionCtx) (bool, *int32) {
 var HumanReadablePreemptionReasons = map[string]string{
 	kueue.InClusterQueueReason:                "prioritization in the ClusterQueue",
 	kueue.InCohortReclamationReason:           "reclamation within the cohort",
-	kueue.InCohortFairSharingReason:           "fair sharing within the cohort",
+	kueue.InCohortFairSharingReason:           "Fair Sharing within the cohort",
 	kueue.InCohortReclaimWhileBorrowingReason: "reclamation within the cohort while borrowing",
 	"": "UNKNOWN",
 }
diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go
index fe11884ae..378a2099a 100644
--- a/pkg/scheduler/scheduler.go
+++ b/pkg/scheduler/scheduler.go
@@ -606,7 +606,7 @@ func makeIterator(ctx context.Context, entries []entry, workloadOrdering workloa
 
 // classicalIterator returns entries ordered on:
 // 1. request under nominal quota before borrowing.
-// 2. fair sharing: lower DominantResourceShare first.
+// 2. Fair Sharing: lower DominantResourceShare first.
 // 3. higher priority first.
 // 4. FIFO on eviction or creation timestamp.
 type classicalIterator struct {
diff --git a/site/content/en/docs/reference/kueue-config.v1beta1.md b/site/content/en/docs/reference/kueue-config.v1beta1.md
index ec9c395b0..009c9242e 100644
--- a/site/content/en/docs/reference/kueue-config.v1beta1.md
+++ b/site/content/en/docs/reference/kueue-config.v1beta1.md
@@ -189,7 +189,7 @@ instead.</p>
 <a href="#FairSharing"><code>FairSharing</code></a>
 </td>
 <td>
-   <p>FairSharing controls the fair sharing semantics across the cluster.</p>
+   <p>FairSharing controls the Fair Sharing semantics across the cluster.</p>
 </td>
 </tr>
 <tr><td><code>resources</code> <B>[Required]</B><br/>
@@ -458,7 +458,7 @@ must be named tls.key and tls.crt, respectively.</p>
 <code>bool</code>
 </td>
 <td>
-   <p>enable indicates whether to enable fair sharing for all cohorts.
+   <p>enable indicates whether to enable Fair Sharing for all cohorts.
 Defaults to false.</p>
 </td>
 </tr>
diff --git a/site/content/en/docs/reference/kueue.v1beta1.md b/site/content/en/docs/reference/kueue.v1beta1.md
index 3256c1e0b..08f65f7bf 100644
--- a/site/content/en/docs/reference/kueue.v1beta1.md
+++ b/site/content/en/docs/reference/kueue.v1beta1.md
@@ -743,11 +743,11 @@ fits within the nominal quota of its ClusterQueue, only preempt
 Workloads in the cohort that have lower priority than the pending
 Workload. <strong>Fair Sharing</strong> only preempt Workloads in the cohort that
 have lower priority than the pending Workload and that satisfy the
-fair sharing preemptionStategies.</li>
+Fair Sharing preemptionStategies.</li>
 <li><code>Any</code>: <strong>Classic Preemption</strong> if the pending Workload fits within
 the nominal quota of its ClusterQueue, preempt any Workload in the
 cohort, irrespective of priority. <strong>Fair Sharing</strong> preempt Workloads
-in the cohort that satisfy the fair sharing preemptionStrategies.</li>
+in the cohort that satisfy the Fair Sharing preemptionStrategies.</li>
 </ul>
 </td>
 </tr>
@@ -1067,7 +1067,7 @@ disadvantage against other ClusterQueues and Cohorts.</p>
 - [ClusterQueueStatus](#kueue-x-k8s-io-v1beta1-ClusterQueueStatus)
 
 
-<p>fairSharing contains the information about the current status of fair sharing.</p>
+<p>fairSharing contains the information about the current status of Fair Sharing.</p>
 
 
 <table class="table">



Test patch:
diff --git a/pkg/scheduler/preemption/preemption_test.go b/pkg/scheduler/preemption/preemption_test.go
index 6e43944f1..8b49d0387 100644
--- a/pkg/scheduler/preemption/preemption_test.go
+++ b/pkg/scheduler/preemption/preemption_test.go
@@ -2165,7 +2165,7 @@ func TestFairPreemptions(t *testing.T) {
 			targetCQ: "a",
 		},
 		// preemption.borrowWithinCohort does not affect how
-		// we handle fair sharing preemptions. Lower priority
+		// we handle Fair Sharing preemptions. Lower priority
 		// workloads are not preempted unless
 		// DominantResourceShare value indicates that they
 		// should be preempted.
diff --git a/pkg/util/testingjobs/deployment/wrappers.go b/pkg/util/testingjobs/deployment/wrappers.go
index facd3e8b2..293233c4b 100644
--- a/pkg/util/testingjobs/deployment/wrappers.go
+++ b/pkg/util/testingjobs/deployment/wrappers.go
@@ -155,3 +155,8 @@ func (d *DeploymentWrapper) PodTemplateAnnotation(k, v string) *DeploymentWrappe
 func (d *DeploymentWrapper) PodTemplateSpecQueue(q string) *DeploymentWrapper {
 	return d.PodTemplateSpecLabel(constants.QueueLabel, q)
 }
+
+func (d *DeploymentWrapper) TerminationGracePeriod(seconds int64) *DeploymentWrapper {
+	d.Spec.Template.Spec.TerminationGracePeriodSeconds = &seconds
+	return d
+}
diff --git a/pkg/util/testingjobs/leaderworkerset/wrappers.go b/pkg/util/testingjobs/leaderworkerset/wrappers.go
index 45c22f14d..4d9efe332 100644
--- a/pkg/util/testingjobs/leaderworkerset/wrappers.go
+++ b/pkg/util/testingjobs/leaderworkerset/wrappers.go
@@ -202,3 +202,11 @@ func (w *LeaderWorkerSetWrapper) WorkerTemplate(worker corev1.PodTemplateSpec) *
 	w.Spec.LeaderWorkerTemplate.WorkerTemplate = worker
 	return w
 }
+
+func (w *LeaderWorkerSetWrapper) TerminationGracePeriod(seconds int64) *LeaderWorkerSetWrapper {
+	if w.Spec.LeaderWorkerTemplate.LeaderTemplate != nil {
+		w.Spec.LeaderWorkerTemplate.LeaderTemplate.Spec.TerminationGracePeriodSeconds = &seconds
+	}
+	w.Spec.LeaderWorkerTemplate.WorkerTemplate.Spec.TerminationGracePeriodSeconds = &seconds
+	return w
+}
diff --git a/pkg/util/testingjobs/mpijob/wrappers_mpijob.go b/pkg/util/testingjobs/mpijob/wrappers_mpijob.go
index afc85ab6a..c1fc6a826 100644
--- a/pkg/util/testingjobs/mpijob/wrappers_mpijob.go
+++ b/pkg/util/testingjobs/mpijob/wrappers_mpijob.go
@@ -254,3 +254,9 @@ func (j *MPIJobWrapper) ManagedBy(c string) *MPIJobWrapper {
 	j.Spec.RunPolicy.ManagedBy = &c
 	return j
 }
+
+func (j *MPIJobWrapper) TerminationGracePeriodSeconds(seconds int64) *MPIJobWrapper {
+	j.Spec.MPIReplicaSpecs[kfmpi.MPIReplicaTypeLauncher].Template.Spec.TerminationGracePeriodSeconds = ptr.To(seconds)
+	j.Spec.MPIReplicaSpecs[kfmpi.MPIReplicaTypeWorker].Template.Spec.TerminationGracePeriodSeconds = ptr.To(seconds)
+	return j
+}
diff --git a/pkg/util/testingjobs/pytorchjob/wrappers_pytorchjob.go b/pkg/util/testingjobs/pytorchjob/wrappers_pytorchjob.go
index 353edca10..2ec073784 100644
--- a/pkg/util/testingjobs/pytorchjob/wrappers_pytorchjob.go
+++ b/pkg/util/testingjobs/pytorchjob/wrappers_pytorchjob.go
@@ -243,3 +243,9 @@ func (j *PyTorchJobWrapper) ManagedBy(c string) *PyTorchJobWrapper {
 	j.Spec.RunPolicy.ManagedBy = &c
 	return j
 }
+
+func (j *PyTorchJobWrapper) TerminationGracePeriodSeconds(seconds int64) *PyTorchJobWrapper {
+	j.Spec.PyTorchReplicaSpecs[kftraining.PyTorchJobReplicaTypeMaster].Template.Spec.TerminationGracePeriodSeconds = ptr.To(seconds)
+	j.Spec.PyTorchReplicaSpecs[kftraining.PyTorchJobReplicaTypeWorker].Template.Spec.TerminationGracePeriodSeconds = ptr.To(seconds)
+	return j
+}
diff --git a/pkg/util/testingjobs/statefulset/wrappers.go b/pkg/util/testingjobs/statefulset/wrappers.go
index f28c168b9..96861b03f 100644
--- a/pkg/util/testingjobs/statefulset/wrappers.go
+++ b/pkg/util/testingjobs/statefulset/wrappers.go
@@ -215,3 +215,9 @@ func (ss *StatefulSetWrapper) Limit(r corev1.ResourceName, v string) *StatefulSe
 func (ss *StatefulSetWrapper) RequestAndLimit(r corev1.ResourceName, v string) *StatefulSetWrapper {
 	return ss.Request(r, v).Limit(r, v)
 }
+
+// TerminationGracePeriod sets terminationGracePeriodSeconds for the pod object
+func (ss *StatefulSetWrapper) TerminationGracePeriod(seconds int64) *StatefulSetWrapper {
+	ss.Spec.Template.Spec.TerminationGracePeriodSeconds = &seconds
+	return ss
+}
diff --git a/test/e2e/customconfigs/skipjobswithoutqueuename_test.go b/test/e2e/customconfigs/skipjobswithoutqueuename_test.go
index c4cc5fdad..c25072f35 100644
--- a/test/e2e/customconfigs/skipjobswithoutqueuename_test.go
+++ b/test/e2e/customconfigs/skipjobswithoutqueuename_test.go
@@ -264,6 +264,7 @@ var _ = ginkgo.Describe("ManageJobsWithoutQueueName", ginkgo.Ordered, func() {
 			ginkgo.By("creating a pod without a queue name", func() {
 				testPod = testingpod.MakePod("test-pod", "kube-system").
 					Image(util.E2eTestAgnHostImage, util.BehaviorWaitForDeletion).
+					TerminationGracePeriod(1).
 					RequestAndLimit(corev1.ResourceCPU, "1").
 					RequestAndLimit(corev1.ResourceMemory, "2Gi").
 					Obj()
@@ -321,6 +322,7 @@ var _ = ginkgo.Describe("ManageJobsWithoutQueueName", ginkgo.Ordered, func() {
 					Image(util.E2eTestAgnHostImage, util.BehaviorWaitForDeletion).
 					RequestAndLimit(corev1.ResourceCPU, "1").
 					RequestAndLimit(corev1.ResourceMemory, "2Gi").
+					TerminationGracePeriod(1).
 					Replicas(2).
 					Obj()
 				gomega.Expect(k8sClient.Create(ctx, testDeploy)).Should(gomega.Succeed())
diff --git a/test/e2e/multikueue/e2e_test.go b/test/e2e/multikueue/e2e_test.go
index ee2d83aea..ff8d842a2 100644
--- a/test/e2e/multikueue/e2e_test.go
+++ b/test/e2e/multikueue/e2e_test.go
@@ -398,6 +398,7 @@ var _ = ginkgo.Describe("MultiKueue", func() {
 				Queue(managerLq.Name).
 				RequestAndLimit("cpu", "1").
 				RequestAndLimit("memory", "2G").
+				TerminationGracePeriod(1).
 				// Give it the time to be observed Active in the live status update step.
 				Image(util.E2eTestAgnHostImage, util.BehaviorWaitForDeletion).
 				Obj()
@@ -443,7 +444,8 @@ var _ = ginkgo.Describe("MultiKueue", func() {
 			})
 
 			ginkgo.By("Finishing the job's pod", func() {
-				util.WaitForActivePodsAndTerminate(ctx, k8sWorker2Client, worker2RestClient, worker2Cfg, job.Namespace, 1, 0)
+				listOpts := util.GetListOptsFromLabel(fmt.Sprintf("batch.kubernetes.io/job-name=%s", job.Name))
+				util.WaitForActivePodsAndTerminate(ctx, k8sWorker2Client, worker2RestClient, worker2Cfg, job.Namespace, 1, 0, listOpts)
 			})
 
 			ginkgo.By("Waiting for the job to finish", func() {
@@ -537,7 +539,8 @@ var _ = ginkgo.Describe("MultiKueue", func() {
 			})
 
 			ginkgo.By("Finishing the jobset pods", func() {
-				util.WaitForActivePodsAndTerminate(ctx, k8sWorker1Client, worker1RestClient, worker1Cfg, jobSet.Namespace, 4, 0)
+				listOpts := util.GetListOptsFromLabel(fmt.Sprintf("jobset.sigs.k8s.io/jobset-name=%s", jobSet.Name))
+				util.WaitForActivePodsAndTerminate(ctx, k8sWorker1Client, worker1RestClient, worker1Cfg, jobSet.Namespace, 4, 0, listOpts)
 			})
 
 			ginkgo.By("Waiting for the jobSet to finish", func() {
@@ -579,14 +582,16 @@ var _ = ginkgo.Describe("MultiKueue", func() {
 
 		ginkgo.It("Should run an appwrapper containing a job on worker if admitted", func() {
 			// Since it requires 2 CPU in total, this appwrapper can only be admitted in worker 1.
+			jobName := "job-1"
 			aw := testingaw.MakeAppWrapper("aw", managerNs.Name).
 				Queue(managerLq.Name).
-				Component(testingjob.MakeJob("job-1", managerNs.Name).
+				Component(testingjob.MakeJob(jobName, managerNs.Name).
 					SetTypeMeta().
 					Suspend(false).
 					Image(util.E2eTestAgnHostImage, util.BehaviorWaitForDeletion). // Give it the time to be observed Active in the live status update step.
 					Parallelism(2).
 					RequestAndLimit(corev1.ResourceCPU, "1").
+					TerminationGracePeriod(1).
 					SetTypeMeta().Obj()).
 				Obj()
 
@@ -609,7 +614,8 @@ var _ = ginkgo.Describe("MultiKueue", func() {
 			})
 
 			ginkgo.By("Finishing the wrapped job's pods", func() {
-				util.WaitForActivePodsAndTerminate(ctx, k8sWorker1Client, worker1RestClient, worker1Cfg, aw.Namespace, 2, 0)
+				listOpts := util.GetListOptsFromLabel(fmt.Sprintf("batch.kubernetes.io/job-name=%s", jobName))
+				util.WaitForActivePodsAndTerminate(ctx, k8sWorker1Client, worker1RestClient, worker1Cfg, aw.Namespace, 2, 0, listOpts)
 			})
 
 			ginkgo.By("Waiting for the appwrapper to finish", func() {
diff --git a/test/e2e/singlecluster/deployment_test.go b/test/e2e/singlecluster/deployment_test.go
index c0056de7d..a69d48e3d 100644
--- a/test/e2e/singlecluster/deployment_test.go
+++ b/test/e2e/singlecluster/deployment_test.go
@@ -88,6 +88,7 @@ var _ = ginkgo.Describe("Deployment", func() {
 		deployment := deploymenttesting.MakeDeployment("deployment", ns.Name).
 			Image(util.E2eTestAgnHostImage, util.BehaviorWaitForDeletion).
 			RequestAndLimit(corev1.ResourceCPU, "200m").
+			TerminationGracePeriod(1).
 			Replicas(3).
 			Queue(lq.Name).
 			Obj()
@@ -137,6 +138,7 @@ var _ = ginkgo.Describe("Deployment", func() {
 		deployment := deploymenttesting.MakeDeployment("deployment", ns.Name).
 			Image(util.E2eTestAgnHostImage, util.BehaviorWaitForDeletion).
 			RequestAndLimit(corev1.ResourceCPU, "200m").
+			TerminationGracePeriod(1).
 			Replicas(3).
 			Queue("invalid-queue-name").
 			Obj()
diff --git a/test/e2e/singlecluster/leaderworkerset_test.go b/test/e2e/singlecluster/leaderworkerset_test.go
index c18e25aa5..2591b689f 100644
--- a/test/e2e/singlecluster/leaderworkerset_test.go
+++ b/test/e2e/singlecluster/leaderworkerset_test.go
@@ -89,6 +89,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() {
 				Size(1).
 				Replicas(1).
 				RequestAndLimit(corev1.ResourceCPU, "200m").
+				TerminationGracePeriod(1).
 				Queue(lq.Name).
 				Obj()
 
@@ -147,6 +148,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() {
 				Size(3).
 				Replicas(1).
 				RequestAndLimit(corev1.ResourceCPU, "200m").
+				TerminationGracePeriod(1).
 				Queue(lq.Name).
 				Obj()
 			ginkgo.By("Create a LeaderWorkerSet", func() {
@@ -201,6 +203,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() {
 				Size(3).
 				Replicas(2).
 				RequestAndLimit(corev1.ResourceCPU, "200m").
+				TerminationGracePeriod(1).
 				Queue(lq.Name).
 				Obj()
 
@@ -264,6 +267,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() {
 					Size(3).
 					Replicas(1).
 					RequestAndLimit(corev1.ResourceCPU, "200m").
+					TerminationGracePeriod(1).
 					Queue(lq.Name).
 					StartupPolicy(startupPolicyType).
 					Obj()
@@ -358,6 +362,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() {
 					Size(3).
 					Replicas(2).
 					RequestAndLimit(corev1.ResourceCPU, "200m").
+					TerminationGracePeriod(1).
 					Queue(lq.Name).
 					StartupPolicy(startupPolicyType).
 					Obj()
@@ -455,6 +460,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() {
 					Size(3).
 					Replicas(2).
 					RequestAndLimit(corev1.ResourceCPU, "200m").
+					TerminationGracePeriod(1).
 					Queue(lq.Name).
 					LeaderTemplate(corev1.PodTemplateSpec{
 						Spec: corev1.PodSpec{
@@ -474,6 +480,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() {
 						},
 					}).
 					StartupPolicy(startupPolicyType).
+					TerminationGracePeriod(1).
 					Obj()
 
 				ginkgo.By("Create a LeaderWorkerSet", func() {
@@ -556,6 +563,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() {
 						NodeSelector: map[string]string{},
 					},
 				}).
+				TerminationGracePeriod(1).
 				Obj()
 
 			ginkgo.By("Create a LeaderWorkerSet", func() {
diff --git a/test/e2e/singlecluster/metrics_test.go b/test/e2e/singlecluster/metrics_test.go
index 7bfa873b3..5b3244b02 100644
--- a/test/e2e/singlecluster/metrics_test.go
+++ b/test/e2e/singlecluster/metrics_test.go
@@ -84,6 +84,7 @@ var _ = ginkgo.Describe("Metrics", func() {
 		curlPod = testingjobspod.MakePod("curl-metrics", config.DefaultNamespace).
 			ServiceAccountName(serviceAccountName).
 			Image(util.E2eTestAgnHostImage, util.BehaviorWaitForDeletion).
+			TerminationGracePeriod(1).
 			Obj()
 		gomega.Expect(k8sClient.Create(ctx, curlPod)).Should(gomega.Succeed())
 
diff --git a/test/e2e/singlecluster/pod_test.go b/test/e2e/singlecluster/pod_test.go
index 547a93415..3f9b72340 100644
--- a/test/e2e/singlecluster/pod_test.go
+++ b/test/e2e/singlecluster/pod_test.go
@@ -17,13 +17,14 @@ limitations under the License.
 package e2e
 
 import (
+	"fmt"
+
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -451,6 +452,7 @@ var _ = ginkgo.Describe("Pod groups", func() {
 				Queue(lq.Name).
 				PriorityClass("high").
 				RequestAndLimit(corev1.ResourceCPU, "1").
+				TerminationGracePeriod(1).
 				MakeGroup(2)
 			highGroupKey := client.ObjectKey{Namespace: ns.Name, Name: "high-priority-group"}
 
@@ -484,7 +486,7 @@ var _ = ginkgo.Describe("Pod groups", func() {
 					for _, origPod := range defaultPriorityGroup {
 						var p corev1.Pod
 						g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(origPod), &p)).To(gomega.Succeed())
-						g.Expect(p.Status.Phase).To(gomega.Equal(corev1.PodFailed))
+						g.Expect(p.Status.Phase).To(gomega.Equal(corev1.PodFailed), fmt.Sprintf("%#v", p.Status))
 					}
 				}, util.LongTimeout, util.Interval).Should(gomega.Succeed())
 			})
@@ -522,11 +524,7 @@ var _ = ginkgo.Describe("Pod groups", func() {
 			})
 
 			ginkgo.By("Call high priority group pods to complete", func() {
-				selector, err := labels.Parse("kueue.x-k8s.io/pod-group-name=high-priority-group")
-				gomega.Expect(err).NotTo(gomega.HaveOccurred())
-				listOpts := &client.ListOptions{
-					LabelSelector: selector,
-				}
+				listOpts := util.GetListOptsFromLabel("kueue.x-k8s.io/pod-group-name=high-priority-group")
 				util.WaitForActivePodsAndTerminate(ctx, k8sClient, restClient, cfg, ns.Name, 2, 0, listOpts)
 			})
 
diff --git a/test/e2e/singlecluster/statefulset_test.go b/test/e2e/singlecluster/statefulset_test.go
index 164b7880d..eda654dae 100644
--- a/test/e2e/singlecluster/statefulset_test.go
+++ b/test/e2e/singlecluster/statefulset_test.go
@@ -90,6 +90,7 @@ var _ = ginkgo.Describe("StatefulSet integration", func() {
 			statefulSet := statefulsettesting.MakeStatefulSet("sts", ns.Name).
 				Image(util.E2eTestAgnHostImage, util.BehaviorWaitForDeletion).
 				RequestAndLimit(corev1.ResourceCPU, "200m").
+				TerminationGracePeriod(1).
 				Replicas(3).
 				Queue(lq.Name).
 				Obj()
@@ -109,6 +110,7 @@ var _ = ginkgo.Describe("StatefulSet integration", func() {
 				conflictingStatefulSet := statefulsettesting.MakeStatefulSet("sts-conflict", ns.Name).
 					Image(util.E2eTestAgnHostImage, util.BehaviorWaitForDeletion).
 					RequestAndLimit(corev1.ResourceCPU, "200m").
+					TerminationGracePeriod(1).
 					Replicas(1).
 					Queue(lq.Name).
 					Obj()
@@ -146,6 +148,7 @@ var _ = ginkgo.Describe("StatefulSet integration", func() {
 			statefulSet := statefulsettesting.MakeStatefulSet("sts", ns.Name).
 				Image(util.E2eTestAgnHostImageOld, util.BehaviorWaitForDeletion).
 				RequestAndLimit(corev1.ResourceCPU, "200m").
+				TerminationGracePeriod(1).
 				Replicas(3).
 				Queue(lq.Name).
 				Obj()
@@ -196,6 +199,7 @@ var _ = ginkgo.Describe("StatefulSet integration", func() {
 			statefulSet := statefulsettesting.MakeStatefulSet("sts", ns.Name).
 				Image(util.E2eTestAgnHostImage, util.BehaviorWaitForDeletion).
 				RequestAndLimit(corev1.ResourceCPU, "200m").
+				TerminationGracePeriod(1).
 				Replicas(3).
 				Queue(lq.Name).
 				Obj()
@@ -280,6 +284,7 @@ var _ = ginkgo.Describe("StatefulSet integration", func() {
 			statefulSet := statefulsettesting.MakeStatefulSet("sts", ns.Name).
 				Image(util.E2eTestAgnHostImage, util.BehaviorWaitForDeletion).
 				RequestAndLimit(corev1.ResourceCPU, "200m").
+				TerminationGracePeriod(1).
 				Replicas(3).
 				Queue(lq.Name).
 				Obj()
@@ -386,6 +391,7 @@ var _ = ginkgo.Describe("StatefulSet integration", func() {
 			statefulSet := statefulsettesting.MakeStatefulSet("sts", ns.Name).
 				Image(util.E2eTestAgnHostImage, util.BehaviorWaitForDeletion).
 				RequestAndLimit(corev1.ResourceCPU, "200m").
+				TerminationGracePeriod(1).
 				Replicas(3).
 				Queue(localQueueName).
 				Obj()
diff --git a/test/e2e/tas/leaderworkerset_test.go b/test/e2e/tas/leaderworkerset_test.go
index f1a7c36de..5724aa256 100644
--- a/test/e2e/tas/leaderworkerset_test.go
+++ b/test/e2e/tas/leaderworkerset_test.go
@@ -114,6 +114,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for LeaderWorkerSet", func() {
 						},
 					},
 				}).
+				TerminationGracePeriod(1).
 				Obj()
 			ginkgo.By("Creating a LeaderWorkerSet", func() {
 				gomega.Expect(k8sClient.Create(ctx, lws)).To(gomega.Succeed())
diff --git a/test/e2e/tas/rayjob_test.go b/test/e2e/tas/rayjob_test.go
index e4896ff26..ead1bcd38 100644
--- a/test/e2e/tas/rayjob_test.go
+++ b/test/e2e/tas/rayjob_test.go
@@ -89,11 +89,10 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for RayJob", ginkgo.Ordered, fu
 	ginkgo.When("Creating a RayJob", func() {
 		ginkgo.It("Should place pods based on the ranks-ordering", func() {
 			const (
-				headReplicas     = 1
-				workerReplicas   = 3
-				submitterReplica = 1
+				headReplicas   = 1
+				workerReplicas = 3
 			)
-			numPods := headReplicas + workerReplicas + submitterReplica
+			numPods := headReplicas + workerReplicas
 			kuberayTestImage := util.GetKuberayTestImage()
 			rayjob := testingrayjob.MakeJob("ranks-ray", ns.Name).
 				Queue(localQueue.Name).
@@ -167,10 +166,10 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for RayJob", ginkgo.Ordered, fu
 								Image: kuberayTestImage,
 								Resources: corev1.ResourceRequirements{
 									Requests: corev1.ResourceList{
-										corev1.ResourceCPU: resource.MustParse("50m"),
+										corev1.ResourceCPU: resource.MustParse("200m"),
 									},
 									Limits: corev1.ResourceList{
-										corev1.ResourceCPU: resource.MustParse("50m"),
+										corev1.ResourceCPU: resource.MustParse("200m"),
 									},
 								},
 							},
@@ -200,7 +199,12 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for RayJob", ginkgo.Ordered, fu
 			ginkgo.By("ensure all pods are created", func() {
 				gomega.Eventually(func(g gomega.Gomega) {
 					g.Expect(k8sClient.List(ctx, pods, client.InNamespace(rayjob.Namespace))).To(gomega.Succeed())
-					g.Expect(pods.Items).Should(gomega.HaveLen(numPods))
+					// TODO(#4665): strengthen the assert after moving to Ray >=1.3.1.
+					// We temporarily don't assert on the exact number of Pods as this flakes on Ray
+					// 1.2.2 due to the submitter Pod and Job not being created occasionally (See
+					// https://github.com/kubernetes-sigs/kueue/issues/4508#issuecomment-2724257298
+					// for more details).
+					g.Expect(len(pods.Items)).Should(gomega.BeNumerically(">=", numPods))
 					// The timeout is long to ensure all cluster pods are up and running.
 					// This is because the Ray image takes long time (around 170s on the CI)
 					// to load and then to sync with head.
diff --git a/test/e2e/tas/statefulset_test.go b/test/e2e/tas/statefulset_test.go
index c248e3435..48833dae6 100644
--- a/test/e2e/tas/statefulset_test.go
+++ b/test/e2e/tas/statefulset_test.go
@@ -84,6 +84,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for StatefulSet", func() {
 				Replicas(replicas).
 				Queue(localQueue.Name).
 				PodTemplateSpecAnnotation(kueuealpha.PodSetRequiredTopologyAnnotation, testing.DefaultBlockTopologyLevel).
+				TerminationGracePeriod(1).
 				Obj()
 			gomega.Expect(k8sClient.Create(ctx, sts)).Should(gomega.Succeed())
 
diff --git a/test/util/util.go b/test/util/util.go
index b10afa727..9ef8aa86c 100644
--- a/test/util/util.go
+++ b/test/util/util.go
@@ -1036,3 +1036,11 @@ func FindDeploymentCondition(deployment *appsv1.Deployment, deploymentType appsv
 	}
 	return nil
 }
+
+func GetListOptsFromLabel(label string) *client.ListOptions {
+	selector, err := labels.Parse(label)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	return &client.ListOptions{
+		LabelSelector: selector,
+	}
+}