Repository: koordinator-sh/koordinator

Issue:
[proposal] Support dominant resource weight for scoring in load aware scheduling.
<!-- Please only use this template for submitting proposal -->

**What is your proposal**:

We can add new scoring weight arg for dominant resource in load aware scheduling.

The node's dominant resource is defined as the most used resource on node:

> dominant resource score = min(allocatable - usedOfExistingPods - estimatedOfIncomingPod) / allocatable)

This idea comes from the concept of Dominant Resource Fairness, known as DRF. Paper is here: [www.cs.berkeley.edu/~matei/papers/2011/nsdi_drf.pdf](https://people.eecs.berkeley.edu/~matei/papers/2011/nsdi_drf.pdf)


**Why is this needed**:

Currently, We can only set resource weight base on resource name. The weight is hard to set when resources usage are already unbalanced in a multi-resource environment because of multiple node type, long run pods' affinity specs or other reasons.

For example, if node A uses cpu heavily and memory are available, node B uses memory heavily and cpu are available. We should prefer to schedule cpu-heavy pods on B and memory-heavy pods on A. We can only have fixed weight on resource now and all pods will prefer to A if memory has higher weight or B if cpu has higher weight.

**Is there a suggested solution, if so, please add it**:

Provided in proposal.


Discussion:
@zheng-weihao: @saintube @ZiMengSheng Hi, any thought on this issue? PR is ready after #2582 is merged.@ZiMengSheng: Will we introduce a new plugin similar to BalancedResourceAllocation of NodeResourcesFit?@zheng-weihao: > Will we introduce a new plugin similar to BalancedResourceAllocation of NodeResourcesFit?

I prefer to add args on load aware scheduling plugin, since the main goal for load aware is to balance resource usage.

Solution patch:
diff --git a/pkg/scheduler/apis/config/types.go b/pkg/scheduler/apis/config/types.go
index 5392698b..4bb9595d 100644
--- a/pkg/scheduler/apis/config/types.go
+++ b/pkg/scheduler/apis/config/types.go
@@ -43,6 +43,9 @@ type LoadAwareSchedulingArgs struct {
 	// ResourceWeights indicates the weights of resources.
 	// The weights of CPU and Memory are both 1 by default.
 	ResourceWeights map[corev1.ResourceName]int64
+	// DominantResourceWeight indicates the weight of the dominant resource.
+	// Dominant resource is the resource with the maximum utilization, which is based on the concept of Dominant Resource Fairness.
+	DominantResourceWeight int64
 	// UsageThresholds indicates the resource utilization threshold of the whole machine.
 	// The default for CPU is 65%, and the default for memory is 95%.
 	UsageThresholds map[corev1.ResourceName]int64
diff --git a/pkg/scheduler/apis/config/v1/defaults.go b/pkg/scheduler/apis/config/v1/defaults.go
index cc22e965..d1aee193 100644
--- a/pkg/scheduler/apis/config/v1/defaults.go
+++ b/pkg/scheduler/apis/config/v1/defaults.go
@@ -98,7 +98,7 @@ func SetDefaults_LoadAwareSchedulingArgs(obj *LoadAwareSchedulingArgs) {
 	if obj.NodeMetricExpirationSeconds == nil {
 		obj.NodeMetricExpirationSeconds = pointer.Int64(defaultNodeMetricExpirationSeconds)
 	}
-	if len(obj.ResourceWeights) == 0 {
+	if len(obj.ResourceWeights) == 0 && obj.DominantResourceWeight == 0 {
 		obj.ResourceWeights = defaultResourceWeights
 	}
 	if len(obj.UsageThresholds) == 0 {
diff --git a/pkg/scheduler/apis/config/v1/types.go b/pkg/scheduler/apis/config/v1/types.go
index 1cf9561f..594fc071 100644
--- a/pkg/scheduler/apis/config/v1/types.go
+++ b/pkg/scheduler/apis/config/v1/types.go
@@ -42,6 +42,9 @@ type LoadAwareSchedulingArgs struct {
 	// ResourceWeights indicates the weights of resources.
 	// The weights of CPU and Memory are both 1 by default.
 	ResourceWeights map[corev1.ResourceName]int64 `json:"resourceWeights,omitempty"`
+	// DominantResourceWeight indicates the weight of the dominant resource.
+	// Dominant resource is the resource with the maximum utilization, which is based on the concept of Dominant Resource Fairness.
+	DominantResourceWeight int64 `json:"dominantResourceWeight,omitempty"`
 	// UsageThresholds indicates the resource utilization threshold of the whole machine.
 	// The default for CPU is 65%, and the default for memory is 95%.
 	UsageThresholds map[corev1.ResourceName]int64 `json:"usageThresholds,omitempty"`
diff --git a/pkg/scheduler/apis/config/v1/zz_generated.conversion.go b/pkg/scheduler/apis/config/v1/zz_generated.conversion.go
index 66ee75a0..97ad7283 100644
--- a/pkg/scheduler/apis/config/v1/zz_generated.conversion.go
+++ b/pkg/scheduler/apis/config/v1/zz_generated.conversion.go
@@ -393,6 +393,7 @@ func autoConvert_v1_LoadAwareSchedulingArgs_To_config_LoadAwareSchedulingArgs(in
 	out.NodeMetricExpirationSeconds = (*int64)(unsafe.Pointer(in.NodeMetricExpirationSeconds))
 	out.EnableScheduleWhenNodeMetricsExpired = (*bool)(unsafe.Pointer(in.EnableScheduleWhenNodeMetricsExpired))
 	out.ResourceWeights = *(*map[corev1.ResourceName]int64)(unsafe.Pointer(&in.ResourceWeights))
+	out.DominantResourceWeight = in.DominantResourceWeight
 	out.UsageThresholds = *(*map[corev1.ResourceName]int64)(unsafe.Pointer(&in.UsageThresholds))
 	out.ProdUsageThresholds = *(*map[corev1.ResourceName]int64)(unsafe.Pointer(&in.ProdUsageThresholds))
 	out.ProdUsageIncludeSys = in.ProdUsageIncludeSys
@@ -427,6 +428,7 @@ func autoConvert_config_LoadAwareSchedulingArgs_To_v1_LoadAwareSchedulingArgs(in
 	out.NodeMetricExpirationSeconds = (*int64)(unsafe.Pointer(in.NodeMetricExpirationSeconds))
 	out.EnableScheduleWhenNodeMetricsExpired = (*bool)(unsafe.Pointer(in.EnableScheduleWhenNodeMetricsExpired))
 	out.ResourceWeights = *(*map[corev1.ResourceName]int64)(unsafe.Pointer(&in.ResourceWeights))
+	out.DominantResourceWeight = in.DominantResourceWeight
 	out.UsageThresholds = *(*map[corev1.ResourceName]int64)(unsafe.Pointer(&in.UsageThresholds))
 	out.ProdUsageThresholds = *(*map[corev1.ResourceName]int64)(unsafe.Pointer(&in.ProdUsageThresholds))
 	out.ProdUsageIncludeSys = in.ProdUsageIncludeSys
diff --git a/pkg/scheduler/apis/config/v1beta3/defaults.go b/pkg/scheduler/apis/config/v1beta3/defaults.go
index f7913a8d..088f60aa 100644
--- a/pkg/scheduler/apis/config/v1beta3/defaults.go
+++ b/pkg/scheduler/apis/config/v1beta3/defaults.go
@@ -98,7 +98,7 @@ func SetDefaults_LoadAwareSchedulingArgs(obj *LoadAwareSchedulingArgs) {
 	if obj.NodeMetricExpirationSeconds == nil {
 		obj.NodeMetricExpirationSeconds = pointer.Int64(defaultNodeMetricExpirationSeconds)
 	}
-	if len(obj.ResourceWeights) == 0 {
+	if len(obj.ResourceWeights) == 0 && obj.DominantResourceWeight == 0 {
 		obj.ResourceWeights = defaultResourceWeights
 	}
 	if len(obj.UsageThresholds) == 0 {
diff --git a/pkg/scheduler/apis/config/v1beta3/types.go b/pkg/scheduler/apis/config/v1beta3/types.go
index ef33e46a..3af32587 100644
--- a/pkg/scheduler/apis/config/v1beta3/types.go
+++ b/pkg/scheduler/apis/config/v1beta3/types.go
@@ -43,6 +43,9 @@ type LoadAwareSchedulingArgs struct {
 	// ResourceWeights indicates the weights of resources.
 	// The weights of CPU and Memory are both 1 by default.
 	ResourceWeights map[corev1.ResourceName]int64 `json:"resourceWeights,omitempty"`
+	// DominantResourceWeight indicates the weight of the dominant resource.
+	// Dominant resource is the resource with the maximum utilization, which is based on the concept of Dominant Resource Fairness.
+	DominantResourceWeight int64 `json:"dominantResourceWeight,omitempty"`
 	// UsageThresholds indicates the resource utilization threshold of the whole machine.
 	// The default for CPU is 65%, and the default for memory is 95%.
 	UsageThresholds map[corev1.ResourceName]int64 `json:"usageThresholds,omitempty"`
diff --git a/pkg/scheduler/apis/config/v1beta3/zz_generated.conversion.go b/pkg/scheduler/apis/config/v1beta3/zz_generated.conversion.go
index fb176ba7..c928cbd4 100644
--- a/pkg/scheduler/apis/config/v1beta3/zz_generated.conversion.go
+++ b/pkg/scheduler/apis/config/v1beta3/zz_generated.conversion.go
@@ -393,6 +393,7 @@ func autoConvert_v1beta3_LoadAwareSchedulingArgs_To_config_LoadAwareSchedulingAr
 	out.NodeMetricExpirationSeconds = (*int64)(unsafe.Pointer(in.NodeMetricExpirationSeconds))
 	out.EnableScheduleWhenNodeMetricsExpired = (*bool)(unsafe.Pointer(in.EnableScheduleWhenNodeMetricsExpired))
 	out.ResourceWeights = *(*map[corev1.ResourceName]int64)(unsafe.Pointer(&in.ResourceWeights))
+	out.DominantResourceWeight = in.DominantResourceWeight
 	out.UsageThresholds = *(*map[corev1.ResourceName]int64)(unsafe.Pointer(&in.UsageThresholds))
 	out.ProdUsageThresholds = *(*map[corev1.ResourceName]int64)(unsafe.Pointer(&in.ProdUsageThresholds))
 	out.ProdUsageIncludeSys = in.ProdUsageIncludeSys
@@ -422,6 +423,7 @@ func autoConvert_config_LoadAwareSchedulingArgs_To_v1beta3_LoadAwareSchedulingAr
 	out.NodeMetricExpirationSeconds = (*int64)(unsafe.Pointer(in.NodeMetricExpirationSeconds))
 	out.EnableScheduleWhenNodeMetricsExpired = (*bool)(unsafe.Pointer(in.EnableScheduleWhenNodeMetricsExpired))
 	out.ResourceWeights = *(*map[corev1.ResourceName]int64)(unsafe.Pointer(&in.ResourceWeights))
+	out.DominantResourceWeight = in.DominantResourceWeight
 	out.UsageThresholds = *(*map[corev1.ResourceName]int64)(unsafe.Pointer(&in.UsageThresholds))
 	out.ProdUsageThresholds = *(*map[corev1.ResourceName]int64)(unsafe.Pointer(&in.ProdUsageThresholds))
 	out.ProdUsageIncludeSys = in.ProdUsageIncludeSys
diff --git a/pkg/scheduler/apis/config/validation/validation_pluginargs.go b/pkg/scheduler/apis/config/validation/validation_pluginargs.go
index 40b85bc6..9922b409 100644
--- a/pkg/scheduler/apis/config/validation/validation_pluginargs.go
+++ b/pkg/scheduler/apis/config/validation/validation_pluginargs.go
@@ -39,6 +39,12 @@ func ValidateLoadAwareSchedulingArgs(args *config.LoadAwareSchedulingArgs) error
 	if err := validateResourceWeights(args.ResourceWeights); err != nil {
 		allErrs = append(allErrs, field.Invalid(field.NewPath("resourceWeights"), args.ResourceWeights, err.Error()))
 	}
+	if args.DominantResourceWeight < 0 {
+		allErrs = append(allErrs, field.Invalid(field.NewPath("dominantResourceWeight"), args.DominantResourceWeight, "dominantResourceWeight should not be a negative value"))
+	}
+	if args.DominantResourceWeight > 100 {
+		allErrs = append(allErrs, field.Invalid(field.NewPath("dominantResourceWeight"), args.DominantResourceWeight, "dominantResourceWeight should be less than 100"))
+	}
 	if err := validateResourceThresholds(args.UsageThresholds); err != nil {
 		allErrs = append(allErrs, field.Invalid(field.NewPath("usageThresholds"), args.UsageThresholds, err.Error()))
 	}
diff --git a/pkg/scheduler/plugins/loadaware/load_aware.go b/pkg/scheduler/plugins/loadaware/load_aware.go
index 557ab935..55747b66 100644
--- a/pkg/scheduler/plugins/loadaware/load_aware.go
+++ b/pkg/scheduler/plugins/loadaware/load_aware.go
@@ -110,7 +110,7 @@ func New(args runtime.Object, handle framework.Handle) (framework.Plugin, error)
 	nodeMetricInformer := koordInformers.Slo().V1alpha1().NodeMetrics()
 	frameworkexthelper.ForceSyncFromInformer(context.TODO().Done(), koordInformers, nodeMetricInformer.Informer(), assignCache.NodeMetricHandler())
 	scoreWeights := vectorizer.ToFactorVec(pluginArgs.ResourceWeights)
-	if scoreWeights.Empty() {
+	if pluginArgs.DominantResourceWeight == 0 && scoreWeights.Empty() {
 		scoreWeights = nil
 	}
 	return &Plugin{
@@ -286,7 +286,7 @@ func (p *Plugin) Score(ctx context.Context, state *framework.CycleState, pod *co
 			"estimated", klog.Format(p.vectorizer.ToList(estimated)),
 			"estimatedExistingPods", klog.KObjSlice(estimatedPods))
 	}
-	score := loadAwareSchedulingScorer(p.scoreWeights, estimated, allocatable)
+	score := loadAwareSchedulingScorer(p.args.DominantResourceWeight, p.scoreWeights, estimated, allocatable)
 	return score, nil
 }
 
@@ -343,12 +343,20 @@ func (p *Plugin) filterNodeUsage(nodeName string, pod *corev1.Pod, usageThreshol
 	return nil
 }
 
-func loadAwareSchedulingScorer(resToWeightMap, used, allocatable ResourceVector) int64 {
-	var nodeScore, weightSum int64
+func loadAwareSchedulingScorer(dominantWeight int64, resToWeightMap, used, allocatable ResourceVector) int64 {
+	var nodeScore, dominantScore, weightSum int64
+	if dominantWeight != 0 {
+		dominantScore, weightSum = framework.MaxNodeScore, dominantWeight
+	}
 	for i, weight := range resToWeightMap {
-		nodeScore += leastUsedScore(used[i], allocatable[i]) * weight
+		score := leastUsedScore(used[i], allocatable[i])
+		nodeScore += score * weight
 		weightSum += weight
+		if dominantScore > score {
+			dominantScore = score
+		}
 	}
+	nodeScore += dominantScore * dominantWeight
 	if weightSum <= 0 {
 		return 0
 	}



Test patch:
diff --git a/pkg/scheduler/plugins/loadaware/load_aware_test.go b/pkg/scheduler/plugins/loadaware/load_aware_test.go
index 0e7ece76..8a951911 100644
--- a/pkg/scheduler/plugins/loadaware/load_aware_test.go
+++ b/pkg/scheduler/plugins/loadaware/load_aware_test.go
@@ -1462,6 +1462,7 @@ func TestScore(t *testing.T) {
 		assignedPod             []*podAssignInfo
 		nodeName                string
 		nodeMetric              *slov1alpha1.NodeMetric
+		dominantResourceWeight  int64
 		scoreAccordingProdUsage bool
 		aggregatedArgs          *v1beta3.LoadAwareSchedulingAggregatedArgs
 		wantScore               int64
@@ -2287,6 +2288,56 @@ func TestScore(t *testing.T) {
 			wantScore:  90,
 			wantStatus: nil,
 		},
+		{
+			name: "score with drf",
+			pod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "default",
+					Name:      "test-pod-1",
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name: "test-container",
+							Resources: corev1.ResourceRequirements{
+								Limits: corev1.ResourceList{
+									corev1.ResourceCPU:    resource.MustParse("16"),
+									corev1.ResourceMemory: resource.MustParse("32Gi"),
+								},
+								Requests: corev1.ResourceList{
+									corev1.ResourceCPU:    resource.MustParse("16"),
+									corev1.ResourceMemory: resource.MustParse("32Gi"),
+								},
+							},
+						},
+					},
+				},
+			},
+			nodeName: "test-node-1",
+			nodeMetric: &slov1alpha1.NodeMetric{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-node-1",
+				},
+				Spec: slov1alpha1.NodeMetricSpec{
+					CollectPolicy: &slov1alpha1.NodeMetricCollectPolicy{
+						ReportIntervalSeconds: pointer.Int64(60),
+					},
+				},
+				Status: slov1alpha1.NodeMetricStatus{
+					UpdateTime: &metav1.Time{
+						Time: time.Now(),
+					},
+					NodeMetric: &slov1alpha1.NodeMetricInfo{
+						NodeUsage: slov1alpha1.ResourceMap{
+							ResourceList: corev1.ResourceList{},
+						},
+					},
+				},
+			},
+			dominantResourceWeight: 10,
+			wantScore:              85,
+			wantStatus:             nil,
+		},
 		{
 			name: "score empty pod",
 			pod: &corev1.Pod{
@@ -2331,6 +2382,7 @@ func TestScore(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			var v1beta3args v1beta3.LoadAwareSchedulingArgs
 			v1beta3args.ScoreAccordingProdUsage = &tt.scoreAccordingProdUsage
+			v1beta3args.DominantResourceWeight = tt.dominantResourceWeight
 			if tt.aggregatedArgs != nil {
 				v1beta3args.Aggregated = tt.aggregatedArgs
 			}