Repository: ray-project/kuberay (Go)

Issue:
[Bug][Multi-host][Scheduler] Yunikorn scheduler doesn't respect to numOfHosts
### Search before asking

- [x] I searched the [issues](https://github.com/ray-project/kuberay/issues) and found no similar issues.


### KubeRay Component

ray-operator

### What happened + What you expected to happen

When Yunikorn calculate resources for gang scheduling, it doesn't take numOfHosts into account. This will result in gang scheduling not working with multi-host.
https://github.com/ray-project/kuberay/blob/df5577fe1b3f537f36dbda3870b4743edd2f11bb/ray-operator/controllers/ray/batchscheduler/yunikorn/yunikorn_task_groups.go#L55

### Reproduction script

N/A

### Anything else

_No response_

### Are you willing to submit a PR?

- [ ] Yes I am willing to submit a PR!


Discussion:
Hi @troychiu, I'd like to help with this issue.



Patch:
diff --git a/ray-operator/controllers/ray/batchscheduler/yunikorn/yunikorn_task_groups.go b/ray-operator/controllers/ray/batchscheduler/yunikorn/yunikorn_task_groups.go
index 57a5914c530..63752979fd7 100644
--- a/ray-operator/controllers/ray/batchscheduler/yunikorn/yunikorn_task_groups.go
+++ b/ray-operator/controllers/ray/batchscheduler/yunikorn/yunikorn_task_groups.go
@@ -52,11 +52,11 @@ func newTaskGroupsFromApp(app *v1.RayCluster) *TaskGroups {
 	// worker groups
 	for _, workerGroupSpec := range app.Spec.WorkerGroupSpecs {
 		workerMinResource := utils.CalculatePodResource(workerGroupSpec.Template.Spec)
-		minWorkers := workerGroupSpec.MinReplicas
+		minWorkers := (*workerGroupSpec.MinReplicas) * workerGroupSpec.NumOfHosts
 		taskGroups.addTaskGroup(
 			TaskGroup{
 				Name:         workerGroupSpec.GroupName,
-				MinMember:    *minWorkers,
+				MinMember:    minWorkers,
 				MinResource:  utils.ConvertResourceListToMapString(workerMinResource),
 				NodeSelector: workerGroupSpec.Template.Spec.NodeSelector,
 				Tolerations:  workerGroupSpec.Template.Spec.Tolerations,


Test patch:
diff --git a/ray-operator/controllers/ray/batchscheduler/yunikorn/yunikorn_scheduler_test.go b/ray-operator/controllers/ray/batchscheduler/yunikorn/yunikorn_scheduler_test.go
index 949fd1b52dd..eb4636172d1 100644
--- a/ray-operator/controllers/ray/batchscheduler/yunikorn/yunikorn_scheduler_test.go
+++ b/ray-operator/controllers/ray/batchscheduler/yunikorn/yunikorn_scheduler_test.go
@@ -142,11 +142,12 @@ func TestPopulateGangSchedulingAnnotations(t *testing.T) {
 	//   memory: 10Gi
 	//   nvidia.com/gpu: 1
 	addWorkerPodSpec(rayClusterWithGangScheduling,
-		"worker-group-1", 1, 1, 2, v1.ResourceList{
+		"worker-group-1", 2, 2, 2, v1.ResourceList{
 			v1.ResourceCPU:    resource.MustParse("2"),
 			v1.ResourceMemory: resource.MustParse("10Gi"),
 			"nvidia.com/gpu":  resource.MustParse("1"),
 		})
+	rayClusterWithGangScheduling.Spec.WorkerGroupSpecs[0].NumOfHosts = 3
 
 	// gang-scheduling enabled case, the plugin should populate the taskGroup annotation to the app
 	rayPod := createPod("ray-pod", "default")
@@ -173,7 +174,7 @@ func TestPopulateGangSchedulingAnnotations(t *testing.T) {
 	// verify the correctness of worker group
 	workerGroup := taskGroups.getTaskGroup("worker-group-1")
 	assert.NotNil(t, workerGroup)
-	assert.Equal(t, int32(1), workerGroup.MinMember)
+	assert.Equal(t, int32(6), workerGroup.MinMember)
 	assert.Equal(t, resource.MustParse("2"), workerGroup.MinResource[v1.ResourceCPU.String()])
 	assert.Equal(t, resource.MustParse("10Gi"), workerGroup.MinResource[v1.ResourceMemory.String()])
 	assert.Equal(t, resource.MustParse("1"), workerGroup.MinResource["nvidia.com/gpu"])