Repository: kubean-io/kubean
Pull request #893 (2023-07-21 05:28:02)

enable to config max records
<!--  

Thanks for sending a pull request!  Here are some tips for you:

* make sure your commit is signed off

-->

**What type of PR is this?**
<!-- 
Uncomment only one ` /kind <>` line, hit enter to put that in a new line, and remove leading whitespaces from that line:

/kind api-change
/kind bug
/kind cleanup
/kind design
/kind documentation
/kind failing-test
/kind feature
/kind flake
-->

/kind feature

**What this PR does / why we need it**:

**Which issue(s) this PR fixes**:
<!--
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
-->
Fixes # https://github.com/kubean-io/kubean/issues/894 

**Special notes for your reviewer**:



hangscer8: test:
<img width="1440" alt="企业微信截图_ac59637c-e997-4837-a36b-8a0a3aa30139" src="https://github.com/kubean-io/kubean/assets/28976713/c7738d95-d93c-431e-988c-57857b8adfb8">

tu1h: /lgtm

Patch:
diff --git a/charts/kubean/templates/configmap.yaml b/charts/kubean/templates/configmap.yaml
index fc28e2c5e..ea3e53ede 100644
--- a/charts/kubean/templates/configmap.yaml
+++ b/charts/kubean/templates/configmap.yaml
@@ -2,9 +2,9 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ include "kubean.fullname" . }}-config
+  name: kubean-config
   namespace: {{ include "kubean.namespace" . }}
   labels:
   {{- include "kubean.labels" . | nindent 4}}
 data:
-  BACKEND_LIMIT: "10"
+  CLUSTER_OPERATIONS_BACKEND_LIMIT: "{{ .Values.kubeanOperator.operationsBackendLimit }}"
diff --git a/charts/kubean/values.yaml b/charts/kubean/values.yaml
index 4661da257..9bc49f1ec 100644
--- a/charts/kubean/values.yaml
+++ b/charts/kubean/values.yaml
@@ -7,7 +7,7 @@ kubeanOperator:
   replicaCount: 1
   nameOverride: ""
   fullnameOverride: ""
-
+  operationsBackendLimit: 5
   podAnnotations: {}
 
   podSecurityContext: {}
diff --git a/pkg/controllers/cluster/controller.go b/pkg/controllers/cluster/controller.go
index 6f57dbbc9..b301dc265 100644
--- a/pkg/controllers/cluster/controller.go
+++ b/pkg/controllers/cluster/controller.go
@@ -2,14 +2,17 @@ package cluster
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"sort"
+	"strconv"
 	"time"
 
 	clusterv1alpha1 "github.com/kubean-io/kubean-api/apis/cluster/v1alpha1"
 	clusteroperationv1alpha1 "github.com/kubean-io/kubean-api/apis/clusteroperation/v1alpha1"
 	clusterClientSet "github.com/kubean-io/kubean-api/generated/cluster/clientset/versioned"
 	clusterOperationClientSet "github.com/kubean-io/kubean-api/generated/clusteroperation/clientset/versioned"
+	"github.com/kubean-io/kubean/pkg/util"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -21,8 +24,10 @@ import (
 )
 
 const (
-	RequeueAfter = time.Second * 5
-	OpsBackupNum = 5
+	RequeueAfter                         = time.Second * 15
+	KubeanConfigMapName                  = "kubean-config"
+	DefaultClusterOperationsBackEndLimit = 30
+	MaxClusterOperationsBackEndLimit     = 200
 )
 
 type Controller struct {
@@ -106,7 +111,7 @@ func (c *Controller) SortClusterOperationsByCreation(operations []clusteroperati
 }
 
 // CleanExcessClusterOps clean up excess ClusterOperation.
-func (c *Controller) CleanExcessClusterOps(cluster *clusterv1alpha1.Cluster) (bool, error) {
+func (c *Controller) CleanExcessClusterOps(cluster *clusterv1alpha1.Cluster, OpsBackupNum int) (bool, error) {
 	listOpt := metav1.ListOptions{LabelSelector: fmt.Sprintf("clusterName=%s", cluster.Name)}
 	clusterOpsList, err := c.KubeanClusterOpsSet.KubeanV1alpha1().ClusterOperations().List(context.Background(), listOpt)
 	if err != nil {
@@ -129,6 +134,41 @@ func (c *Controller) CleanExcessClusterOps(cluster *clusterv1alpha1.Cluster) (bo
 	return true, nil
 }
 
+type ConfigProperty struct {
+	ClusterOperationsBackEndLimit string `json:"CLUSTER_OPERATIONS_BACKEND_LIMIT"`
+}
+
+func (config *ConfigProperty) GetClusterOperationsBackEndLimit() int {
+
+	value, _ := strconv.Atoi(config.ClusterOperationsBackEndLimit)
+	if value <= 0 {
+		klog.Warningf("GetClusterOperationsBackEndLimit and use default value %d", DefaultClusterOperationsBackEndLimit)
+		return DefaultClusterOperationsBackEndLimit
+	}
+	if value >= MaxClusterOperationsBackEndLimit {
+		klog.Warningf("GetClusterOperationsBackEndLimit and use max value %d", MaxClusterOperationsBackEndLimit)
+		return MaxClusterOperationsBackEndLimit
+	}
+	return value
+}
+
+func (c *Controller) FetchKubeanConfigProperty() *ConfigProperty {
+	configData, err := c.ClientSet.CoreV1().ConfigMaps(util.GetCurrentNSOrDefault()).Get(context.Background(), KubeanConfigMapName, metav1.GetOptions{})
+	if err != nil {
+		return &ConfigProperty{}
+	}
+	jsonData, err := json.Marshal(configData.Data)
+	if err != nil {
+		return &ConfigProperty{}
+	}
+	result := &ConfigProperty{}
+	err = json.Unmarshal(jsonData, result)
+	if err != nil {
+		return &ConfigProperty{}
+	}
+	return result
+}
+
 func (c *Controller) Reconcile(ctx context.Context, req controllerruntime.Request) (controllerruntime.Result, error) {
 	cluster := &clusterv1alpha1.Cluster{}
 	if err := c.Client.Get(ctx, req.NamespacedName, cluster); err != nil {
@@ -138,8 +178,8 @@ func (c *Controller) Reconcile(ctx context.Context, req controllerruntime.Reques
 		klog.ErrorS(err, "failed to get cluster", "cluster", req.String())
 		return controllerruntime.Result{RequeueAfter: RequeueAfter}, nil
 	}
-
-	needRequeue, err := c.CleanExcessClusterOps(cluster)
+	OpsBackupNum := c.FetchKubeanConfigProperty().GetClusterOperationsBackEndLimit()
+	needRequeue, err := c.CleanExcessClusterOps(cluster, OpsBackupNum)
 	if err != nil {
 		klog.ErrorS(err, "failed to clean excess cluster ops", "cluster", cluster.Name)
 		return controllerruntime.Result{RequeueAfter: RequeueAfter}, nil


Test patch:
diff --git a/pkg/controllers/cluster/controller_test.go b/pkg/controllers/cluster/controller_test.go
index dd518d1af..b7e308cc9 100644
--- a/pkg/controllers/cluster/controller_test.go
+++ b/pkg/controllers/cluster/controller_test.go
@@ -4,13 +4,16 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"os"
 	"reflect"
 	"testing"
 	"time"
 
 	localartifactsetv1alpha1 "github.com/kubean-io/kubean-api/apis/localartifactset/v1alpha1"
 	manifestv1alpha1 "github.com/kubean-io/kubean-api/apis/manifest/v1alpha1"
+	"github.com/kubean-io/kubean/pkg/util"
 
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -149,6 +152,66 @@ func TestCompareClusterConditions(t *testing.T) {
 	}
 }
 
+func TestFetchKubeanConfigProperty(t *testing.T) {
+	genController := func() *Controller {
+		return &Controller{
+			Client:              newFakeClient(),
+			ClientSet:           clientsetfake.NewSimpleClientset(),
+			KubeanClusterSet:    clusterv1alpha1fake.NewSimpleClientset(),
+			KubeanClusterOpsSet: clusteroperationv1alpha1fake.NewSimpleClientset(),
+		}
+	}
+	tests := []struct {
+		name       string
+		controller *Controller
+		args       func(controller *Controller)
+		want       int
+	}{
+		{
+			name:       "no config and default value",
+			controller: genController(),
+			args: func(controller *Controller) {
+				// do nothing
+			},
+			want: 30,
+		},
+		{
+			name:       "the config has been set to 25",
+			controller: genController(),
+			args: func(controller *Controller) {
+				os.Setenv("POD_NAMESPACE", "")
+				configMap := &corev1.ConfigMap{}
+				configMap.Name = KubeanConfigMapName
+				configMap.Namespace = util.GetCurrentNSOrDefault()
+				configMap.Data = map[string]string{"CLUSTER_OPERATIONS_BACKEND_LIMIT": "25"}
+				controller.ClientSet.CoreV1().ConfigMaps(util.GetCurrentNSOrDefault()).Create(context.Background(), configMap, metav1.CreateOptions{})
+			},
+			want: 25,
+		},
+		{
+			name:       "the config has been set to 10000",
+			controller: genController(),
+			args: func(controller *Controller) {
+				os.Setenv("POD_NAMESPACE", "")
+				configMap := &corev1.ConfigMap{}
+				configMap.Name = KubeanConfigMapName
+				configMap.Namespace = util.GetCurrentNSOrDefault()
+				configMap.Data = map[string]string{"CLUSTER_OPERATIONS_BACKEND_LIMIT": "10000"}
+				controller.ClientSet.CoreV1().ConfigMaps(util.GetCurrentNSOrDefault()).Create(context.Background(), configMap, metav1.CreateOptions{})
+			},
+			want: 200,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			test.args(test.controller)
+			if test.controller.FetchKubeanConfigProperty().GetClusterOperationsBackEndLimit() != test.want {
+				t.Fatal()
+			}
+		})
+	}
+}
+
 func TestSortClusterOperationsByCreation(t *testing.T) {
 	controller := &Controller{
 		Client:              newFakeClient(),
@@ -191,6 +254,7 @@ func TestSortClusterOperationsByCreation(t *testing.T) {
 }
 
 func Test_CleanExcessClusterOps(t *testing.T) {
+	OpsBackupNum := 5
 	controller := &Controller{
 		Client:              newFakeClient(),
 		ClientSet:           clientsetfake.NewSimpleClientset(),
@@ -218,7 +282,7 @@ func Test_CleanExcessClusterOps(t *testing.T) {
 		{
 			name: "get nothing",
 			args: func() bool {
-				result, _ := controller.CleanExcessClusterOps(exampleCluster)
+				result, _ := controller.CleanExcessClusterOps(exampleCluster, OpsBackupNum)
 				return result
 			},
 			want: false,
@@ -240,7 +304,7 @@ func Test_CleanExcessClusterOps(t *testing.T) {
 					}
 					controller.KubeanClusterOpsSet.KubeanV1alpha1().ClusterOperations().Create(context.Background(), clusterOperation, metav1.CreateOptions{})
 				}
-				result, _ := controller.CleanExcessClusterOps(exampleCluster)
+				result, _ := controller.CleanExcessClusterOps(exampleCluster, OpsBackupNum)
 				return result
 			},
 			want: false,
@@ -262,7 +326,7 @@ func Test_CleanExcessClusterOps(t *testing.T) {
 					}
 					controller.KubeanClusterOpsSet.KubeanV1alpha1().ClusterOperations().Create(context.Background(), clusterOperation, metav1.CreateOptions{})
 				}
-				result, _ := controller.CleanExcessClusterOps(exampleCluster)
+				result, _ := controller.CleanExcessClusterOps(exampleCluster, OpsBackupNum)
 				return result
 			},
 			want: true,