Repository: vllm-project/aibrix
File: pkg/controller/rayclusterreplicaset/rayclusterreplicaset_utils.go (Go)

Code before:
		} else {
			klog.V(4).Info("Ignore inactive cluster", "name", cluster.Name, "state", cluster.Status.State, "deletionTime", cluster.DeletionTimestamp)
		}

	}

	return activeClusters
}

func isClusterActive(c rayclusterv1.RayCluster) bool {
	// Case 1: This RayCluster has been marked for deletion.
	if !c.DeletionTimestamp.IsZero() {
		return false
	}

	// Case 2: Ray Pod creation or deletion failed
	isRayPodCreateOrDeleteFailed := meta.IsStatusConditionPresentAndEqual(c.Status.Conditions, string(rayclusterv1.RayClusterReplicaFailure), metav1.ConditionTrue)
	if isRayPodCreateOrDeleteFailed {
		return false
	}

	// Case 3: The RayCluster has not been provisioned yet.
	// This means the RayCluster is in the init stage, waiting for all Ray Pods to become ready for the first time.
	// We consider the RayCluster active, as the ReplicaSet also counts Pods that are in the init stage.
	isRayClusterProvisioned := meta.IsStatusConditionPresentAndEqual(c.Status.Conditions, string(rayclusterv1.RayClusterProvisioned), metav1.ConditionTrue)
	if !isRayClusterProvisioned {
		return true
	}

	// Case 4: The RayCluster has been provisioned and we need to check if it is ready.
	return rayclusterutil.IsRayClusterReady(&c)
}

func isStatusSame(rs *orchestrationv1alpha1.RayClusterReplicaSet, newStatus orchestrationv1alpha1.RayClusterReplicaSetStatus) bool {
	// Only update the status if something has actually changed
	if rs.Status.Replicas == newStatus.Replicas &&
		rs.Status.FullyLabeledReplicas == newStatus.FullyLabeledReplicas &&
		rs.Status.ReadyReplicas == newStatus.ReadyReplicas &&
		rs.Status.AvailableReplicas == newStatus.AvailableReplicas &&
		rs.Generation == rs.Status.ObservedGeneration &&
		reflect.DeepEqual(rs.Status.Conditions, newStatus.Conditions) {
		return true
	}

	return false
}

func calculateStatus(rs *orchestrationv1alpha1.RayClusterReplicaSet, filteredClusters []rayclusterv1.RayCluster, manageReplicasErr error) orchestrationv1alpha1.RayClusterReplicaSetStatus {
	newStatus := rs.Status
	// Count the number of pods that have labels matching the labels of the cluster
	// template of the replica set, the matching pods may have more

Review comment (bug):
`isClusterActive` is used to decide whether we need to count this Raycluster and later use the count to compare with the desired number to decide whether to scale up or down.

- If the Raycluster has been marked as deleted, do not count it.
- If not been marked as delete, we categorize the Ray cluster into two statuses:
   - Init status: 
     - This means the cluster is waiting for all Ray Pods to become ready for the first time. In this case, we count the cluster, as ReplicaSets also count Pods that are in the init stage. While it’s true that if there is an image pull error or resource limits and the Pod never starts, we may wait indefinitely, the ReplicaSet also doesn’t handle this issue.
  - Non-init status: 
    - We strictly check if the Ray cluster is fully ready, meaning the head Pod is ready and all worker Pods are ready. If any Pod fails, we currently treat this failure as unrecoverable, and a new Ray cluster needs to be created. We can reconsider this approach later.

Code after:
		} else {
			klog.V(4).Info("Ignore inactive cluster", "name", cluster.Name, "state", cluster.Status.State, "deletionTime", cluster.DeletionTimestamp)
		}

	}

	return activeClusters
}

func isClusterActive(c rayclusterv1.RayCluster) bool {
	// Case 1: This RayCluster has been marked for deletion.
	if !c.DeletionTimestamp.IsZero() {
		return false
	}

	// Case 2: The RayCluster has not been provisioned yet.
	// This means the RayCluster is in the init stage, waiting for all Ray Pods to become ready for the first time.
	// We consider the RayCluster active, as the ReplicaSet also counts Pods that are in the init stage.
	isRayClusterProvisioned := meta.IsStatusConditionPresentAndEqual(c.Status.Conditions, string(rayclusterv1.RayClusterProvisioned), metav1.ConditionTrue)
	if !isRayClusterProvisioned {
		return true
	}

	// Case 3: The RayCluster has been provisioned and we need to check if it is ready.
	// Currently, we consider any Ray Pod failure is unrecoverable and need to recreate a new RayCluster.
	return rayclusterutil.IsRayClusterReady(&c)
}

func isStatusSame(rs *orchestrationv1alpha1.RayClusterReplicaSet, newStatus orchestrationv1alpha1.RayClusterReplicaSetStatus) bool {
	// Only update the status if something has actually changed
	if rs.Status.Replicas == newStatus.Replicas &&
		rs.Status.FullyLabeledReplicas == newStatus.FullyLabeledReplicas &&
		rs.Status.ReadyReplicas == newStatus.ReadyReplicas &&
		rs.Status.AvailableReplicas == newStatus.AvailableReplicas &&
		rs.Generation == rs.Status.ObservedGeneration &&
		reflect.DeepEqual(rs.Status.Conditions, newStatus.Conditions) {
		return true
	}

	return false
}

func calculateStatus(rs *orchestrationv1alpha1.RayClusterReplicaSet, filteredClusters []rayclusterv1.RayCluster, manageReplicasErr error) orchestrationv1alpha1.RayClusterReplicaSetStatus {
	newStatus := rs.Status
	// Count the number of pods that have labels matching the labels of the cluster
	// template of the replica set, the matching pods may have more
	// labels than are in the template. Because the label of podTemplateSpec is
	// a superset of the selector of the replica set, so the possible
	// matching pods must be part of the filteredClusters.
	fullyLabeledReplicasCount := 0
	readyReplicasCount := 0