Repository: clusternet/clusternet
Pull request #366 (2022-06-08 08:43:27)

add dynamic dividing API
Signed-off-by: Garrybest <garrybest@foxmail.com>

<!--  Thanks for sending a pull request!

-->

#### What type of PR is this?

#### What this PR does / why we need it:

#### Which issue(s) this PR fixes:
<!--
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
-->
Fixes #

#### Special notes for your reviewer:



Patch:
diff --git a/manifests/crds/apps.clusternet.io_subscriptions.yaml b/manifests/crds/apps.clusternet.io_subscriptions.yaml
index 18bc01c..b91cd51 100644
--- a/manifests/crds/apps.clusternet.io_subscriptions.yaml
+++ b/manifests/crds/apps.clusternet.io_subscriptions.yaml
@@ -86,15 +86,251 @@ spec:
                       type: string
                   type: object
                 type: array
-              dividingSchedulingStrategy:
-                description: Dividing scheduling config params. Present only if SchedulingStrategyType
+              dividingScheduling:
+                description: Dividing scheduling config params. Present only if SchedulingStrategy
                   = Dividing.
                 properties:
+                  dynamicDividing:
+                    description: DynamicDividing describes how to divide replicas
+                      into target clusters dynamically.
+                    properties:
+                      maxClusters:
+                        description: MaxClusters describes the upper bound number
+                          of target clusters.
+                        format: int32
+                        type: integer
+                      minClusters:
+                        description: MinClusters describes the lower bound number
+                          of target clusters.
+                        format: int32
+                        type: integer
+                      preferredClusters:
+                        description: PreferredClusters describes the assigning preference.
+                          If we have a preference for cluster group A compared to
+                          cluster group B (i.e., group A has a larger Weight), desired
+                          replicas will be assigned to cluster group A as many as
+                          possible, while the rest ones will be assigned to cluster
+                          group B.
+                        items:
+                          description: An empty preferred scheduling term matches
+                            all objects with implicit weight 0 (i.e. it's a no-op).
+                            A null preferred scheduling term matches no objects (i.e.
+                            is also a no-op).
+                          properties:
+                            preference:
+                              description: A node selector term, associated with the
+                                corresponding weight.
+                              properties:
+                                matchExpressions:
+                                  description: A list of node selector requirements
+                                    by node's labels.
+                                  items:
+                                    description: A node selector requirement is a
+                                      selector that contains values, a key, and an
+                                      operator that relates the key and values.
+                                    properties:
+                                      key:
+                                        description: The label key that the selector
+                                          applies to.
+                                        type: string
+                                      operator:
+                                        description: Represents a key's relationship
+                                          to a set of values. Valid operators are
+                                          In, NotIn, Exists, DoesNotExist. Gt, and
+                                          Lt.
+                                        type: string
+                                      values:
+                                        description: An array of string values. If
+                                          the operator is In or NotIn, the values
+                                          array must be non-empty. If the operator
+                                          is Exists or DoesNotExist, the values array
+                                          must be empty. If the operator is Gt or
+                                          Lt, the values array must have a single
+                                          element, which will be interpreted as an
+                                          integer. This array is replaced during a
+                                          strategic merge patch.
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchFields:
+                                  description: A list of node selector requirements
+                                    by node's fields.
+                                  items:
+                                    description: A node selector requirement is a
+                                      selector that contains values, a key, and an
+                                      operator that relates the key and values.
+                                    properties:
+                                      key:
+                                        description: The label key that the selector
+                                          applies to.
+                                        type: string
+                                      operator:
+                                        description: Represents a key's relationship
+                                          to a set of values. Valid operators are
+                                          In, NotIn, Exists, DoesNotExist. Gt, and
+                                          Lt.
+                                        type: string
+                                      values:
+                                        description: An array of string values. If
+                                          the operator is In or NotIn, the values
+                                          array must be non-empty. If the operator
+                                          is Exists or DoesNotExist, the values array
+                                          must be empty. If the operator is Gt or
+                                          Lt, the values array must have a single
+                                          element, which will be interpreted as an
+                                          integer. This array is replaced during a
+                                          strategic merge patch.
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                              type: object
+                            weight:
+                              description: Weight associated with matching the corresponding
+                                nodeSelectorTerm, in the range 1-100.
+                              format: int32
+                              type: integer
+                          required:
+                          - preference
+                          - weight
+                          type: object
+                        type: array
+                      strategy:
+                        default: Spread
+                        description: Type of dynamic dividing replica strategy.
+                        enum:
+                        - Spread
+                        - Binpack
+                        type: string
+                      topologySpreadConstraints:
+                        description: TopologySpreadConstraints describes how a group
+                          of replicas ought to spread across topology domains. Scheduler
+                          will schedule pods in a way which abides by the constraints.
+                          All topologySpreadConstraints are ANDed. Present only for
+                          spread divided scheduling.
+                        items:
+                          description: TopologySpreadConstraint specifies how to spread
+                            matching pods among the given topology.
+                          properties:
+                            labelSelector:
+                              description: LabelSelector is used to find matching
+                                pods. Pods that match this label selector are counted
+                                to determine the number of pods in their corresponding
+                                topology domain.
+                              properties:
+                                matchExpressions:
+                                  description: matchExpressions is a list of label
+                                    selector requirements. The requirements are ANDed.
+                                  items:
+                                    description: A label selector requirement is a
+                                      selector that contains values, a key, and an
+                                      operator that relates the key and values.
+                                    properties:
+                                      key:
+                                        description: key is the label key that the
+                                          selector applies to.
+                                        type: string
+                                      operator:
+                                        description: operator represents a key's relationship
+                                          to a set of values. Valid operators are
+                                          In, NotIn, Exists and DoesNotExist.
+                                        type: string
+                                      values:
+                                        description: values is an array of string
+                                          values. If the operator is In or NotIn,
+                                          the values array must be non-empty. If the
+                                          operator is Exists or DoesNotExist, the
+                                          values array must be empty. This array is
+                                          replaced during a strategic merge patch.
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  description: matchLabels is a map of {key,value}
+                                    pairs. A single {key,value} in the matchLabels
+                                    map is equivalent to an element of matchExpressions,
+                                    whose key field is "key", the operator is "In",
+                                    and the values array contains only "value". The
+                                    requirements are ANDed.
+                                  type: object
+                              type: object
+                            maxSkew:
+                              description: 'MaxSkew describes the degree to which
+                                pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`,
+                                it is the maximum permitted difference between the
+                                number of matching pods in the target topology and
+                                the global minimum. For example, in a 3-zone cluster,
+                                MaxSkew is set to 1, and pods with the same labelSelector
+                                spread as 1/1/0: | zone1 | zone2 | zone3 | |   P   |   P   |       |
+                                - if MaxSkew is 1, incoming pod can only be scheduled
+                                to zone3 to become 1/1/1; scheduling it onto zone1(zone2)
+                                would make the ActualSkew(2-0) on zone1(zone2) violate
+                                MaxSkew(1). - if MaxSkew is 2, incoming pod can be
+                                scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`,
+                                it is used to give higher precedence to topologies
+                                that satisfy it. It''s a required field. Default value
+                                is 1 and 0 is not allowed.'
+                              format: int32
+                              type: integer
+                            topologyKey:
+                              description: TopologyKey is the key of node labels.
+                                Nodes that have a label with this key and identical
+                                values are considered to be in the same topology.
+                                We consider each <key, value> as a "bucket", and try
+                                to put balanced number of pods into each bucket. It's
+                                a required field.
+                              type: string
+                            whenUnsatisfiable:
+                              description: 'WhenUnsatisfiable indicates how to deal
+                                with a pod if it doesn''t satisfy the spread constraint.
+                                - DoNotSchedule (default) tells the scheduler not
+                                to schedule it. - ScheduleAnyway tells the scheduler
+                                to schedule the pod in any location,   but giving
+                                higher precedence to topologies that would help reduce
+                                the   skew. A constraint is considered "Unsatisfiable"
+                                for an incoming pod if and only if every possible
+                                node assignment for that pod would violate "MaxSkew"
+                                on some topology. For example, in a 3-zone cluster,
+                                MaxSkew is set to 1, and pods with the same labelSelector
+                                spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P
+                                |   P   |   P   | If WhenUnsatisfiable is set to DoNotSchedule,
+                                incoming pod can only be scheduled to zone2(zone3)
+                                to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3)
+                                satisfies MaxSkew(1). In other words, the cluster
+                                can still be imbalanced, but scheduler won''t make
+                                it *more* imbalanced. It''s a required field.'
+                              type: string
+                          required:
+                          - maxSkew
+                          - topologyKey
+                          - whenUnsatisfiable
+                          type: object
+                        type: array
+                    required:
+                    - strategy
+                    type: object
                   type:
                     default: Static
                     description: Type of dividing replica scheduling.
                     enum:
                     - Static
+                    - Dynamic
                     type: string
                 required:
                 - type
diff --git a/pkg/apis/apps/v1alpha1/subscription.go b/pkg/apis/apps/v1alpha1/subscription.go
index 6d99a2b..37443e3 100644
--- a/pkg/apis/apps/v1alpha1/subscription.go
+++ b/pkg/apis/apps/v1alpha1/subscription.go
@@ -57,10 +57,10 @@ type SubscriptionSpec struct {
 	// +kubebuilder:default=Replication
 	SchedulingStrategy SchedulingStrategyType `json:"schedulingStrategy,omitempty"`
 
-	// Dividing scheduling config params. Present only if SchedulingStrategyType = Dividing.
+	// Dividing scheduling config params. Present only if SchedulingStrategy = Dividing.
 	//
 	// +optional
-	DividingScheduling *DividingSchedulingStrategy `json:"dividingSchedulingStrategy,omitempty"`
+	DividingScheduling *DividingScheduling `json:"dividingScheduling,omitempty"`
 
 	// Subscribers subscribes
 	//
@@ -257,16 +257,58 @@ type Feed struct {
 	Name string `json:"name"`
 }
 
-// DividingSchedulingStrategy describes how to divide replicas into target clusters.
-type DividingSchedulingStrategy struct {
+// DividingScheduling describes how to divide replicas into target clusters.
+type DividingScheduling struct {
 	// Type of dividing replica scheduling.
 	//
 	// +required
 	// +kubebuilder:validation:Required
-	// +kubebuilder:validation:Enum=Static
+	// +kubebuilder:validation:Enum=Static;Dynamic
 	// +kubebuilder:validation:Type=string
 	// +kubebuilder:default=Static
 	Type ReplicaDividingType `json:"type"`
+
+	// DynamicDividing describes how to divide replicas into target clusters dynamically.
+	//
+	// +optional
+	DynamicDividing *DynamicDividing `json:"dynamicDividing,omitempty"`
+}
+
+// DynamicDividing describes how to divide replicas into target clusters dynamically.
+type DynamicDividing struct {
+	// Type of dynamic dividing replica strategy.
+	//
+	// +required
+	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:Enum=Spread;Binpack
+	// +kubebuilder:validation:Type=string
+	// +kubebuilder:default=Spread
+	Strategy DynamicDividingStrategy `json:"strategy"`
+
+	// TopologySpreadConstraints describes how a group of replicas ought to spread across topology
+	// domains. Scheduler will schedule pods in a way which abides by the constraints.
+	// All topologySpreadConstraints are ANDed.
+	// Present only for spread divided scheduling.
+	//
+	// +optional
+	TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"`
+
+	// PreferredClusters describes the assigning preference. If we have a preference for cluster group A
+	// compared to cluster group B (i.e., group A has a larger Weight), desired replicas will be assigned
+	// to cluster group A as many as possible, while the rest ones will be assigned to cluster group B.
+	//
+	// +optional
+	PreferredClusters []corev1.PreferredSchedulingTerm `json:"preferredClusters,omitempty"`
+
+	// MinClusters describes the lower bound number of target clusters.
+	//
+	// +optional
+	MinClusters *int32 `json:"minClusters,omitempty"`
+
+	// MaxClusters describes the upper bound number of target clusters.
+	//
+	// +optional
+	MaxClusters *int32 `json:"maxClusters,omitempty"`
 }
 
 type SchedulingStrategyType string
@@ -289,6 +331,16 @@ const (
 	DynamicReplicaDividingType ReplicaDividingType = "Dynamic"
 )
 
+type DynamicDividingStrategy string
+
+const (
+	// SpreadDividingStrategy spreads out replicas as much as possible.
+	SpreadDividingStrategy DynamicDividingStrategy = "Spread"
+
+	// BinpackDividingStrategy aggregates replicas as much as possible.
+	BinpackDividingStrategy DynamicDividingStrategy = "Binpack"
+)
+
 // +kubebuilder:object:root=true
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
diff --git a/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go
index 32665b7..3138089 100644
--- a/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go
+++ b/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go
@@ -21,8 +21,8 @@ limitations under the License.
 package v1alpha1
 
 import (
-	corev1 "k8s.io/api/core/v1"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
@@ -279,17 +279,62 @@ func (in *DescriptionStatus) DeepCopy() *DescriptionStatus {
 }
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *DividingSchedulingStrategy) DeepCopyInto(out *DividingSchedulingStrategy) {
+func (in *DividingScheduling) DeepCopyInto(out *DividingScheduling) {
 	*out = *in
+	if in.DynamicDividing != nil {
+		in, out := &in.DynamicDividing, &out.DynamicDividing
+		*out = new(DynamicDividing)
+		(*in).DeepCopyInto(*out)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DividingScheduling.
+func (in *DividingScheduling) DeepCopy() *DividingScheduling {
+	if in == nil {
+		return nil
+	}
+	out := new(DividingScheduling)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DynamicDividing) DeepCopyInto(out *DynamicDividing) {
+	*out = *in
+	if in.TopologySpreadConstraints != nil {
+		in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints
+		*out = make([]v1.TopologySpreadConstraint, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.PreferredClusters != nil {
+		in, out := &in.PreferredClusters, &out.PreferredClusters
+		*out = make([]v1.PreferredSchedulingTerm, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.MinClusters != nil {
+		in, out := &in.MinClusters, &out.MinClusters
+		*out = new(int32)
+		**out = **in
+	}
+	if in.MaxClusters != nil {
+		in, out := &in.MaxClusters, &out.MaxClusters
+		*out = new(int32)
+		**out = **in
+	}
 	return
 }
 
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DividingSchedulingStrategy.
-func (in *DividingSchedulingStrategy) DeepCopy() *DividingSchedulingStrategy {
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamicDividing.
+func (in *DynamicDividing) DeepCopy() *DynamicDividing {
 	if in == nil {
 		return nil
 	}
-	out := new(DividingSchedulingStrategy)
+	out := new(DynamicDividing)
 	in.DeepCopyInto(out)
 	return out
 }
@@ -515,7 +560,7 @@ func (in *GlobalizationSpec) DeepCopyInto(out *GlobalizationSpec) {
 	*out = *in
 	if in.ClusterAffinity != nil {
 		in, out := &in.ClusterAffinity, &out.ClusterAffinity
-		*out = new(v1.LabelSelector)
+		*out = new(metav1.LabelSelector)
 		(*in).DeepCopyInto(*out)
 	}
 	if in.Overrides != nil {
@@ -940,14 +985,14 @@ func (in *ReplicaRequirements) DeepCopyInto(out *ReplicaRequirements) {
 	}
 	if in.Tolerations != nil {
 		in, out := &in.Tolerations, &out.Tolerations
-		*out = make([]corev1.Toleration, len(*in))
+		*out = make([]v1.Toleration, len(*in))
 		for i := range *in {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
 	if in.Affinity != nil {
 		in, out := &in.Affinity, &out.Affinity
-		*out = new(corev1.Affinity)
+		*out = new(v1.Affinity)
 		(*in).DeepCopyInto(*out)
 	}
 	in.Resources.DeepCopyInto(&out.Resources)
@@ -985,7 +1030,7 @@ func (in *Subscriber) DeepCopyInto(out *Subscriber) {
 	*out = *in
 	if in.ClusterAffinity != nil {
 		in, out := &in.ClusterAffinity, &out.ClusterAffinity
-		*out = new(v1.LabelSelector)
+		*out = new(metav1.LabelSelector)
 		(*in).DeepCopyInto(*out)
 	}
 	return
@@ -1067,8 +1112,8 @@ func (in *SubscriptionSpec) DeepCopyInto(out *SubscriptionSpec) {
 	*out = *in
 	if in.DividingScheduling != nil {
 		in, out := &in.DividingScheduling, &out.DividingScheduling
-		*out = new(DividingSchedulingStrategy)
-		**out = **in
+		*out = new(DividingScheduling)
+		(*in).DeepCopyInto(*out)
 	}
 	if in.Subscribers != nil {
 		in, out := &in.Subscribers, &out.Subscribers
@@ -1079,7 +1124,7 @@ func (in *SubscriptionSpec) DeepCopyInto(out *SubscriptionSpec) {
 	}
 	if in.ClusterTolerations != nil {
 		in, out := &in.ClusterTolerations, &out.ClusterTolerations
-		*out = make([]corev1.Toleration, len(*in))
+		*out = make([]v1.Toleration, len(*in))
 		for i := range *in {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}


Test patch:
diff --git a/pkg/scheduler/framework/plugins/defaultassigner/algorithm_test.go b/pkg/scheduler/framework/plugins/defaultassigner/algorithm_test.go
index c628091..5a668af 100644
--- a/pkg/scheduler/framework/plugins/defaultassigner/algorithm_test.go
+++ b/pkg/scheduler/framework/plugins/defaultassigner/algorithm_test.go
@@ -154,7 +154,7 @@ func TestDivideReplicas(t *testing.T) {
 				sub: &appsapi.Subscription{
 					Spec: appsapi.SubscriptionSpec{
 						SchedulingStrategy: appsapi.DividingSchedulingStrategyType,
-						DividingScheduling: &appsapi.DividingSchedulingStrategy{Type: appsapi.StaticReplicaDividingType},
+						DividingScheduling: &appsapi.DividingScheduling{Type: appsapi.StaticReplicaDividingType},
 						Subscribers: []appsapi.Subscriber{
 							{
 								ClusterAffinity: &metav1.LabelSelector{
@@ -191,7 +191,7 @@ func TestDivideReplicas(t *testing.T) {
 				sub: &appsapi.Subscription{
 					Spec: appsapi.SubscriptionSpec{
 						SchedulingStrategy: appsapi.DividingSchedulingStrategyType,
-						DividingScheduling: &appsapi.DividingSchedulingStrategy{Type: appsapi.StaticReplicaDividingType},
+						DividingScheduling: &appsapi.DividingScheduling{Type: appsapi.StaticReplicaDividingType},
 						Subscribers: []appsapi.Subscriber{
 							{
 								ClusterAffinity: &metav1.LabelSelector{
@@ -236,7 +236,7 @@ func TestDivideReplicas(t *testing.T) {
 				sub: &appsapi.Subscription{
 					Spec: appsapi.SubscriptionSpec{
 						SchedulingStrategy: appsapi.DividingSchedulingStrategyType,
-						DividingScheduling: &appsapi.DividingSchedulingStrategy{Type: appsapi.StaticReplicaDividingType},
+						DividingScheduling: &appsapi.DividingScheduling{Type: appsapi.StaticReplicaDividingType},
 						Subscribers: []appsapi.Subscriber{
 							{
 								ClusterAffinity: &metav1.LabelSelector{
@@ -281,7 +281,7 @@ func TestDivideReplicas(t *testing.T) {
 				sub: &appsapi.Subscription{
 					Spec: appsapi.SubscriptionSpec{
 						SchedulingStrategy: appsapi.DividingSchedulingStrategyType,
-						DividingScheduling: &appsapi.DividingSchedulingStrategy{Type: appsapi.StaticReplicaDividingType},
+						DividingScheduling: &appsapi.DividingScheduling{Type: appsapi.StaticReplicaDividingType},
 						Subscribers: []appsapi.Subscriber{
 							{
 								ClusterAffinity: &metav1.LabelSelector{
@@ -334,7 +334,7 @@ func TestDivideReplicas(t *testing.T) {
 				sub: &appsapi.Subscription{
 					Spec: appsapi.SubscriptionSpec{
 						SchedulingStrategy: appsapi.DividingSchedulingStrategyType,
-						DividingScheduling: &appsapi.DividingSchedulingStrategy{Type: appsapi.StaticReplicaDividingType},
+						DividingScheduling: &appsapi.DividingScheduling{Type: appsapi.StaticReplicaDividingType},
 						Subscribers: []appsapi.Subscriber{
 							{
 								ClusterAffinity: &metav1.LabelSelector{