Repository: apecloud/kubeblocks

Issue:
[BUG]Cluster using InstanceTemplate without VolumeClaimTemplate can not operate VolumeExpansion Ops
**Describe the bug**
Cluster using InstanceTemplate without VolumeClaimTemplate can not operate VolumeExpand Ops. OpsRequest failed in validation stage, messages says `volumeClaimTemplate: [data] not support volume expansion in component: mysql, you can view infos by comm
│ and: kubectl get sc`

**To Reproduce**
Steps to reproduce the behavior:
1. create a cluster using InstanceTemplate but template didn't specify `volumeClaimTemplates`, likes below,
```
apiVersion: apps.kubeblocks.io/v1
kind: Cluster
metadata:
  annotations:
    kubeblocks.io/crd-api-version: apps.kubeblocks.io/v1
  name: mysql57-restore
  namespace: thomas-demo
spec:
  clusterDef: vip-mysql
  componentSpecs:
  - componentDef: vip-mysql-5.7-1.0.0-alpha.0
    instanceUpdateStrategy:
      type: OnDelete
    instances:
    - annotations:
        tpl: q9n0
      name: q9n0
      ordinals:
        ranges:
        - end: 299
          start: 0
      replicas: 2
    - annotations:
        tpl: iph1
      name: iph1
      ordinals:
        ranges:
        - end: 599
          start: 300
      replicas: 1
    - annotations:
        tpl: wte2
      name: wte2
      ordinals:
        ranges:
        - end: 899
          start: 600
      replicas: 2
    name: mysql
    persistentVolumeClaimRetentionPolicy:
      whenDeleted: Delete
      whenScaled: Retain
    replicas: 5
    resources:
      limits:
        cpu: "2"
        memory: 4G
      requests:
        cpu: "2"
        memory: 4G
    # serviceVersion: 5.7.44-vip.1
    volumeClaimTemplates:
    - name: data
      spec:
        accessModes:
        - ReadWriteOnce
        resources:
          requests:
            storage: 10G
        # storageClassName: open-local-lvm
  terminationPolicy: Delete
  # topology: replset
```
3. wait cluster running
4. apply a `VolumeExpansion` ops
5. See error

**Expected behavior**
OpsRequest should be success.

**Screenshots**
If applicable, add screenshots to help explain your problem.

**Desktop (please complete the following information):**
 - KB branch(release-1.0)

**Additional context**
Add any other context about the problem here.


Hints from issue comments:

Solution patch:
diff --git a/apis/operations/v1alpha1/opsrequest_validation.go b/apis/operations/v1alpha1/opsrequest_validation.go
index ce051bebe..673d185f2 100644
--- a/apis/operations/v1alpha1/opsrequest_validation.go
+++ b/apis/operations/v1alpha1/opsrequest_validation.go
@@ -604,21 +604,29 @@ func (r *OpsRequest) checkVolumesAllowExpansion(ctx context.Context, cli client.
 		allowExpansion      bool
 		requestStorage      resource.Quantity
 		isShardingComponent bool
+		hasPvc              bool
 	}
 
-	// component name/ sharding name -> vct name -> entity
+	// [component name]/ [sharding name]/ [component name.its name] -> vct name -> entity
 	vols := make(map[string]map[string]Entity)
 	setVols := func(vcts []OpsRequestVolumeClaimTemplate, componentName string) {
 		for _, vct := range vcts {
 			if _, ok := vols[componentName]; !ok {
 				vols[componentName] = make(map[string]Entity)
 			}
-			vols[componentName][vct.Name] = Entity{false, nil, false, vct.Storage, false}
+			vols[componentName][vct.Name] = Entity{false, nil, false, vct.Storage, false, false}
 		}
 	}
 
 	for _, comp := range r.Spec.VolumeExpansionList {
 		setVols(comp.VolumeClaimTemplates, comp.ComponentOps.ComponentName)
+		for _, compSpec := range cluster.Spec.ComponentSpecs {
+			if compSpec.Name == comp.ComponentOps.ComponentName {
+				for _, its := range compSpec.Instances {
+					setVols(comp.VolumeClaimTemplates, fmt.Sprintf("%s.%s", compSpec.Name, its.Name))
+				}
+			}
+		}
 	}
 	fillVol := func(vct appsv1.PersistentVolumeClaimTemplate, key string, isShardingComp bool) {
 		e, ok := vols[key][vct.Name]
@@ -638,26 +646,43 @@ func (r *OpsRequest) checkVolumesAllowExpansion(ctx context.Context, cli client.
 			fillVol(vct, componentName, isShardingComp)
 		}
 	}
+	fillItsVols := func(itsSpec appsv1.InstanceTemplate, cmpVcts []appsv1.PersistentVolumeClaimTemplate, key string) {
+		if _, ok := vols[key]; !ok {
+			return // ignore not-exist its
+		}
+		mergedVcts := mergeItsCmpTemplates(itsSpec.VolumeClaimTemplates, cmpVcts)
+		for _, vct := range mergedVcts {
+			fillVol(vct, key, false)
+		}
+	}
 	// traverse the spec to update volumes
 	for _, comp := range cluster.Spec.ComponentSpecs {
 		fillCompVols(comp, comp.Name, false)
+		for _, its := range comp.Instances {
+			// update its vct volumes
+			fillItsVols(its, comp.VolumeClaimTemplates, fmt.Sprintf("%s.%s", comp.Name, its.Name))
+		}
 	}
 	for _, sharding := range cluster.Spec.Shardings {
 		fillCompVols(sharding.Template, sharding.Name, true)
 	}
 
 	// check all used storage classes
-	var err error
 	for key, compVols := range vols {
 		for vname := range compVols {
 			e := vols[key][vname]
 			if !e.existInSpec {
 				continue
 			}
-			e.storageClassName, err = r.getSCNameByPvcAndCheckStorageSize(ctx, cli, key, vname, e.isShardingComponent, e.requestStorage)
+			found, scName, err := r.getSCNameByPvcAndCheckStorageSize(ctx, cli, key, vname, e.isShardingComponent, e.requestStorage)
 			if err != nil {
 				return err
 			}
+			if !found {
+				continue
+			}
+			e.hasPvc = found
+			e.storageClassName = scName
 			allowExpansion, err := r.checkStorageClassAllowExpansion(ctx, cli, e.storageClassName)
 			if err != nil {
 				continue // ignore the error and take it as not-supported
@@ -674,6 +699,9 @@ func (r *OpsRequest) checkVolumesAllowExpansion(ctx context.Context, cli client.
 			notSupportSc []string
 		)
 		for vct, e := range compVols {
+			if !e.hasPvc {
+				continue
+			}
 			if !e.existInSpec {
 				notFound = append(notFound, vct)
 			}
@@ -724,7 +752,7 @@ func (r *OpsRequest) getSCNameByPvcAndCheckStorageSize(ctx context.Context,
 	key,
 	vctName string,
 	isShardingComponent bool,
-	requestStorage resource.Quantity) (*string, error) {
+	requestStorage resource.Quantity) (found bool, scName *string, err error) {
 	componentName := key
 	targetInsTPLName := ""
 	if strings.Contains(key, ".") {
@@ -743,7 +771,7 @@ func (r *OpsRequest) getSCNameByPvcAndCheckStorageSize(ctx context.Context,
 	}
 	pvcList := &corev1.PersistentVolumeClaimList{}
 	if err := cli.List(ctx, pvcList, client.InNamespace(r.Namespace), matchingLabels); err != nil {
-		return nil, err
+		return false, nil, err
 	}
 	var pvc *corev1.PersistentVolumeClaim
 	for _, pvcItem := range pvcList.Items {
@@ -753,14 +781,34 @@ func (r *OpsRequest) getSCNameByPvcAndCheckStorageSize(ctx context.Context,
 		}
 	}
 	if pvc == nil {
-		return nil, nil
+		return false, nil, nil
 	}
 	previousValue := *pvc.Status.Capacity.Storage()
 	if requestStorage.Cmp(previousValue) < 0 {
-		return nil, fmt.Errorf(`requested storage size of volumeClaimTemplate "%s" can not less than status.capacity.storage "%s" `,
+		return true, nil, fmt.Errorf(`requested storage size of volumeClaimTemplate "%s" can not less than status.capacity.storage "%s" `,
 			vctName, previousValue.String())
 	}
-	return pvc.Spec.StorageClassName, nil
+	return true, pvc.Spec.StorageClassName, nil
+}
+
+func mergeItsCmpTemplates(itsVcts []appsv1.PersistentVolumeClaimTemplate, cmpVcts []appsv1.PersistentVolumeClaimTemplate) []appsv1.PersistentVolumeClaimTemplate {
+	mergedVcts := make([]appsv1.PersistentVolumeClaimTemplate, 0)
+	mergedVcts = append(mergedVcts, cmpVcts...)
+	for _, itsVct := range itsVcts {
+		found := false
+		for i, cmpVct := range mergedVcts {
+			if itsVct.Name == cmpVct.Name {
+				// cmpVct will be override by itsVct
+				mergedVcts[i] = itsVct
+				found = true
+				break
+			}
+		}
+		if !found {
+			mergedVcts = append(mergedVcts, itsVct)
+		}
+	}
+	return mergedVcts
 }
 
 // validateVerticalResourceList checks if k8s resourceList is legal
diff --git a/cmd/manager/main.go b/cmd/manager/main.go
index 38a57bbcf..910af3738 100644
--- a/cmd/manager/main.go
+++ b/cmd/manager/main.go
@@ -41,8 +41,10 @@ import (
 	discoverycli "k8s.io/client-go/discovery"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/utils/ptr"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/config"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
@@ -85,10 +87,13 @@ import (
 const (
 	appName = "kubeblocks"
 
-	probeAddrFlagKey     flagName = "health-probe-bind-address"
-	metricsAddrFlagKey   flagName = "metrics-bind-address"
-	leaderElectFlagKey   flagName = "leader-elect"
-	leaderElectIDFlagKey flagName = "leader-elect-id"
+	probeAddrFlagKey                flagName = "health-probe-bind-address"
+	metricsAddrFlagKey              flagName = "metrics-bind-address"
+	leaderElectFlagKey              flagName = "leader-elect"
+	leaderElectIDFlagKey            flagName = "leader-elect-id"
+	leaderElectLeaseDurationFlagKey flagName = "leader-elect-lease-duration"
+	leaderElectRenewDeadlineFlagKey flagName = "leader-elect-renew-deadline"
+	leaderElectRetryPeriodFlagKey   flagName = "leader-elect-retry-period"
 
 	// switch flags key for API groups
 	appsFlagKey         flagName = "apps"
@@ -151,11 +156,14 @@ func init() {
 	viper.SetDefault(instanceset.MaxPlainRevisionCount, 1024)
 	viper.SetDefault(instanceset.FeatureGateIgnorePodVerticalScaling, false)
 	viper.SetDefault(intctrlutil.FeatureGateEnableRuntimeMetrics, false)
-	viper.SetDefault(constant.CfgKBReconcileWorkers, 8)
 	viper.SetDefault(constant.FeatureGateIgnoreConfigTemplateDefaultMode, false)
 	viper.SetDefault(constant.FeatureGateInPlacePodVerticalScaling, false)
 	viper.SetDefault(constant.I18nResourcesName, "kubeblocks-i18n-resources")
 	viper.SetDefault(constant.APIVersionSupported, "")
+	viper.SetDefault(constant.CfgKBReconcileWorkers, 32)
+	viper.SetDefault(constant.CfgCacheSyncTimeout, 300)
+	viper.SetDefault(constant.CfgClientQPS, 128)
+	viper.SetDefault(constant.CfgClientBurst, 256)
 }
 
 type flagName string
@@ -171,13 +179,19 @@ func (r flagName) viperName() string {
 func setupFlags() {
 	flag.String(metricsAddrFlagKey.String(), ":8080", "The address the metric endpoint binds to.")
 	flag.String(probeAddrFlagKey.String(), ":8081", "The address the probe endpoint binds to.")
+
 	flag.Bool(leaderElectFlagKey.String(), false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
-
 	flag.String(leaderElectIDFlagKey.String(), "001c317f",
 		"The leader election ID prefix for controller manager. "+
 			"This ID must be unique to controller manager.")
+	flag.Int(leaderElectLeaseDurationFlagKey.String(), 30,
+		"The duration (in seconds) that non-leader candidates will wait to force acquire leadership.")
+	flag.Int(leaderElectRenewDeadlineFlagKey.String(), 25,
+		"The duration (in seconds) that the acting control plane will retry refreshing leadership before giving up.")
+	flag.Int(leaderElectRetryPeriodFlagKey.String(), 5,
+		"The duration (in seconds) the LeaderElector clients should wait between tries of actions.")
 
 	flag.Bool(appsFlagKey.String(), true,
 		"Enable the apps controller manager.")
@@ -356,7 +370,6 @@ func main() {
 		// pattern of '{{ hashFNV .Repo }}.{{ .Domain }}', make sure regenerate this ID
 		// if you have forked from this project template.
 		LeaderElectionID: enableLeaderElectionID + ".kubeblocks.io",
-
 		// NOTES:
 		// LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily
 		// when the Manager ends. This requires the binary to immediately end when the
@@ -369,6 +382,9 @@ func main() {
 		// if you are doing or intending to do any operation such as performing cleanups
 		// after the manager stops then its usage might be unsafe.
 		LeaderElectionReleaseOnCancel: true,
+		LeaseDuration:                 ptr.To(time.Duration(viper.GetInt(leaderElectLeaseDurationFlagKey.viperName())) * time.Second),
+		RenewDeadline:                 ptr.To(time.Duration(viper.GetInt(leaderElectRenewDeadlineFlagKey.viperName())) * time.Second),
+		RetryPeriod:                   ptr.To(time.Duration(viper.GetInt(leaderElectRetryPeriodFlagKey.viperName())) * time.Second),
 
 		WebhookServer: webhook.NewServer(webhook.Options{
 			Port:    9443,
@@ -379,6 +395,9 @@ func main() {
 				DisableFor: append(intctrlutil.GetUncachedObjects(), &parametersv1alpha1.ComponentParameter{}),
 			},
 		},
+		Controller: config.Controller{
+			CacheSyncTimeout: viper.GetDuration(constant.CfgCacheSyncTimeout) * time.Second,
+		},
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")
diff --git a/controllers/apps/cluster/cluster_controller.go b/controllers/apps/cluster/cluster_controller.go
index 86a9c7e5e..2b5749f6a 100644
--- a/controllers/apps/cluster/cluster_controller.go
+++ b/controllers/apps/cluster/cluster_controller.go
@@ -21,7 +21,6 @@ package cluster
 
 import (
 	"context"
-	"math"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
@@ -176,7 +175,7 @@ func (r *ClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return intctrlutil.NewControllerManagedBy(mgr).
 		For(&appsv1.Cluster{}).
 		WithOptions(controller.Options{
-			MaxConcurrentReconciles: int(math.Ceil(viper.GetFloat64(constant.CfgKBReconcileWorkers) / 4)),
+			MaxConcurrentReconciles: viper.GetInt(constant.CfgKBReconcileWorkers),
 		}).
 		Owns(&appsv1.Component{}).
 		Owns(&corev1.Service{}). // cluster services
diff --git a/controllers/k8score/event_controller.go b/controllers/k8score/event_controller.go
index 5b582fd39..2ae7ebc19 100644
--- a/controllers/k8score/event_controller.go
+++ b/controllers/k8score/event_controller.go
@@ -29,13 +29,16 @@ import (
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
+	"github.com/apecloud/kubeblocks/pkg/constant"
 	"github.com/apecloud/kubeblocks/pkg/controller/component"
 	"github.com/apecloud/kubeblocks/pkg/controller/instanceset"
 	"github.com/apecloud/kubeblocks/pkg/controller/multicluster"
 	intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil"
+	viper "github.com/apecloud/kubeblocks/pkg/viperx"
 )
 
 const (
@@ -99,7 +102,10 @@ func (r *EventReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl
 // SetupWithManager sets up the controller with the Manager.
 func (r *EventReconciler) SetupWithManager(mgr ctrl.Manager, multiClusterMgr multicluster.Manager) error {
 	b := intctrlutil.NewControllerManagedBy(mgr).
-		For(&corev1.Event{})
+		For(&corev1.Event{}).
+		WithOptions(controller.Options{
+			MaxConcurrentReconciles: viper.GetInt(constant.CfgKBReconcileWorkers) / 4,
+		})
 
 	if multiClusterMgr != nil {
 		multiClusterMgr.Watch(b, &corev1.Event{}, &handler.EnqueueRequestForObject{})
diff --git a/controllers/operations/opsrequest_controller.go b/controllers/operations/opsrequest_controller.go
index 0cd2e95f1..3b45d87cc 100644
--- a/controllers/operations/opsrequest_controller.go
+++ b/controllers/operations/opsrequest_controller.go
@@ -21,7 +21,6 @@ package operations
 
 import (
 	"context"
-	"math"
 	"reflect"
 	"slices"
 	"strings"
@@ -94,7 +93,7 @@ func (r *OpsRequestReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return intctrlutil.NewControllerManagedBy(mgr).
 		For(&opsv1alpha1.OpsRequest{}).
 		WithOptions(controller.Options{
-			MaxConcurrentReconciles: int(math.Ceil(viper.GetFloat64(constant.CfgKBReconcileWorkers) / 2)),
+			MaxConcurrentReconciles: viper.GetInt(constant.CfgKBReconcileWorkers) / 2,
 		}).
 		Watches(&appsv1.Cluster{}, handler.EnqueueRequestsFromMapFunc(r.parseRunningOpsRequests)).
 		Watches(&workloads.InstanceSet{}, handler.EnqueueRequestsFromMapFunc(r.parseRunningOpsRequestsForInstanceSet)).
diff --git a/controllers/parameters/componentparameter_controller.go b/controllers/parameters/componentparameter_controller.go
index 6da0f61f8..a3561b415 100644
--- a/controllers/parameters/componentparameter_controller.go
+++ b/controllers/parameters/componentparameter_controller.go
@@ -22,7 +22,6 @@ package parameters
 import (
 	"context"
 	"fmt"
-	"math"
 	"strconv"
 
 	"github.com/pkg/errors"
@@ -88,7 +87,7 @@ func (r *ComponentParameterReconciler) SetupWithManager(mgr ctrl.Manager, multiC
 	builder := intctrlutil.NewControllerManagedBy(mgr).
 		For(&parametersv1alpha1.ComponentParameter{}).
 		WithOptions(controller.Options{
-			MaxConcurrentReconciles: int(math.Ceil(viper.GetFloat64(constant.CfgKBReconcileWorkers) / 2)),
+			MaxConcurrentReconciles: viper.GetInt(constant.CfgKBReconcileWorkers) / 4,
 		}).
 		Owns(&corev1.ConfigMap{})
 	if multiClusterMgr != nil {
diff --git a/controllers/parameters/reconfigure_controller.go b/controllers/parameters/reconfigure_controller.go
index b9924fe8f..2220a4600 100644
--- a/controllers/parameters/reconfigure_controller.go
+++ b/controllers/parameters/reconfigure_controller.go
@@ -23,7 +23,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"math"
 	"time"
 
 	"github.com/pkg/errors"
@@ -143,7 +142,7 @@ func (r *ReconfigureReconciler) SetupWithManager(mgr ctrl.Manager, multiClusterM
 	b := intctrlutil.NewControllerManagedBy(mgr).
 		For(&corev1.ConfigMap{}).
 		WithOptions(controller.Options{
-			MaxConcurrentReconciles: int(math.Ceil(viper.GetFloat64(constant.CfgKBReconcileWorkers) / 4)),
+			MaxConcurrentReconciles: viper.GetInt(constant.CfgKBReconcileWorkers) / 4,
 		})
 
 	if multiClusterMgr != nil {
diff --git a/deploy/helm/templates/dataprotection.yaml b/deploy/helm/templates/dataprotection.yaml
index 21f960e3a..dcaf9c3db 100644
--- a/deploy/helm/templates/dataprotection.yaml
+++ b/deploy/helm/templates/dataprotection.yaml
@@ -52,7 +52,12 @@ spec:
           image: "{{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.tools.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           resources:
-            {{- toYaml .Values.resources | nindent 12 }}
+            limits:
+              cpu: 50m
+              memory: 64Mi
+            requests:
+              cpu: 50m
+              memory: 64Mi
           command:
             - /bin/true
       containers:
@@ -185,7 +190,12 @@ spec:
             initialDelaySeconds: 5
             periodSeconds: 10
           resources:
-            {{- toYaml .Values.resources | nindent 12 }}
+            limits:
+              cpu: {{ .Values.resources.cpu }}
+              memory: {{ .Values.resources.memory }}
+            requests:
+              cpu: {{ .Values.resources.cpu }}
+              memory: {{ .Values.resources.memory }}
           volumeMounts:
             - mountPath: /etc/kubeblocks
               name: manager-config
diff --git a/deploy/helm/templates/deployment.yaml b/deploy/helm/templates/deployment.yaml
index 8be5d811f..3575f8105 100644
--- a/deploy/helm/templates/deployment.yaml
+++ b/deploy/helm/templates/deployment.yaml
@@ -50,7 +50,12 @@ spec:
           image: "{{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.tools.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           resources:
-            {{- toYaml .Values.resources | nindent 12 }}
+            limits:
+              cpu: 50m
+              memory: 64Mi
+            requests:
+              cpu: 50m
+              memory: 64Mi
           command:
             - /bin/true
       containers:
@@ -59,8 +64,17 @@ spec:
             - "--health-probe-bind-address=:8081"
             - "--metrics-bind-address=:8080"
             - "--leader-elect"
-            {{- with .Values.manager }}
-            - "--leader-elect-id={{- default "001c317f" .leaderElectId }}"
+            {{- with .Values.manager.leaderElection }}
+            - "--leader-elect-id={{- default "001c317f" .Id }}"
+            {{- end }}
+            {{- if .Values.manager.leaderElection.leaseDuration }}
+            - "--leader-elect-lease-duration={{ .Values.manager.leaderElection.leaseDuration }}"
+            {{- end }}
+            {{- if .Values.manager.leaderElection.renewDeadline }}
+            - "--leader-elect-renew-deadline={{ .Values.manager.leaderElection.renewDeadline }}"
+            {{- end }}
+            {{- if .Values.manager.leaderElection.retryPeriod }}
+            - "--leader-elect-retry-period={{ .Values.manager.leaderElection.retryPeriod }}"
             {{- end }}
             - "--zap-devel={{- default "false" .Values.loggerSettings.developmentMode }}"
             - "--zap-time-encoding={{- default "iso8601" .Values.loggerSettings.timeEncoding }}"
@@ -102,6 +116,10 @@ spec:
             - name: KUBEBLOCKS_RECONCILE_WORKERS
               value: {{ .Values.reconcileWorkers | quote }}
             {{- end }}
+            {{- if .Values.cache.syncTimeout }}
+            - name: CACHE_SYNC_TIMEOUT
+              value: {{ .Values.cache.syncTimeout | quote }}
+            {{- end }}
             {{- if .Values.client.qps }}
             - name: CLIENT_QPS
               value: {{ .Values.client.qps | quote }}
@@ -219,7 +237,12 @@ spec:
             initialDelaySeconds: 5
             periodSeconds: 10
           resources:
-            {{- toYaml .Values.resources | nindent 12 }}
+            limits:
+              cpu: {{ .Values.resources.cpu }}
+              memory: {{ .Values.resources.memory }}
+            requests:
+              cpu: {{ .Values.resources.cpu }}
+              memory: {{ .Values.resources.memory }}
           volumeMounts:
             - mountPath: /etc/kubeblocks
               name: manager-config
diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml
index 8151ec791..5493931ca 100644
--- a/deploy/helm/values.yaml
+++ b/deploy/helm/values.yaml
@@ -25,15 +25,20 @@ image:
 ##
 replicaCount: 1
 
-## MaxConcurrentReconciles for component, instanceSet and opsRequest controllers.
-##
+## MaxConcurrentReconciles for cluster, component and instanceSet controllers.
+## default is 32
 reconcileWorkers: ""
 
+## k8s cache configuration.
+cache:
+  # default is 300 seconds
+  syncTimeout: ""
+
 ## k8s client configuration.
 client:
-  # default is 20
+  # default is 128
   qps: ""
-  # default is 30
+  # default is 256
   burst: ""
 
 ## @param nameOverride
@@ -44,7 +49,6 @@ nameOverride: ""
 ##
 fullnameOverride: ""
 
-
 ## KubeBlocks RBAC access priority setting
 ##
 ## @param rbac.enabled is used to enable or disable KubeBlocks RBAC access priority.
@@ -238,24 +242,15 @@ serviceMonitor:
 ## @param topologySpreadConstraints
 topologySpreadConstraints: []
 
-
 ## Resource settings
 ##
-## @param resources.limits
-## @param resources.requests
-resources: {}
-  # We usually recommend not to specify default resources and to leave this as a conscious
-  # choice for the user. This also increases chances charts run on environments with little
-  # resources, such as Minikube. If you do want to specify resources, uncomment the following
-  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+## @param resources.cpu
+## @param resources.memory
+resources:
   # TODO(user): Configure the resources accordingly based on the project requirements.
   # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
-  # limits:
-  #   cpu: 500m
-  #   memory: 128Mi
-  # requests:
-  #   cpu: 10m
-#   memory: 64Mi
+  cpu: 500m
+  memory: 1Gi
 
 ## @param priorityClassName
 ##
@@ -348,7 +343,16 @@ webhooks:
 
 ## manager server settings
 manager:
-  leaderElectId: ""
+  ## leader election
+  leaderElection:
+    # default is "001c317f"
+    Id: ""
+    # default is 30 seconds
+    leaseDuration:
+    # default is 25 seconds
+    renewDeadline:
+    # default is 5 seconds
+    retryPeriod:
 
 ## Data protection settings
 ##
diff --git a/pkg/constant/viper_config.go b/pkg/constant/viper_config.go
index 0e884908e..b919a5848 100644
--- a/pkg/constant/viper_config.go
+++ b/pkg/constant/viper_config.go
@@ -53,6 +53,7 @@ const (
 	CfgKeyDPBackupEncryptionAlgorithm    = "DP_BACKUP_ENCRYPTION_ALGORITHM"
 
 	CfgKBReconcileWorkers = "KUBEBLOCKS_RECONCILE_WORKERS"
+	CfgCacheSyncTimeout   = "CACHE_SYNC_TIMEOUT"
 	CfgClientQPS          = "CLIENT_QPS"
 	CfgClientBurst        = "CLIENT_BURST"
 
diff --git a/pkg/controller/instanceset/instance_util.go b/pkg/controller/instanceset/instance_util.go
index 31889d59e..0fabd9680 100644
--- a/pkg/controller/instanceset/instance_util.go
+++ b/pkg/controller/instanceset/instance_util.go
@@ -259,7 +259,7 @@ func GenerateAllInstanceNames(parentName string, replicas int32, templates []Ins
 	instanceNameList := make([]string, 0)
 	for _, template := range templates {
 		replicas := template.GetReplicas()
-		ordinalList, err := convertOrdinalsToSortedList(template.GetOrdinals())
+		ordinalList, err := ConvertOrdinalsToSortedList(template.GetOrdinals())
 		if err != nil {
 			return nil, err
 		}
@@ -271,7 +271,7 @@ func GenerateAllInstanceNames(parentName string, replicas int32, templates []Ins
 		totalReplicas += replicas
 	}
 	if totalReplicas < replicas {
-		ordinalList, err := convertOrdinalsToSortedList(defaultTemplateOrdinals)
+		ordinalList, err := ConvertOrdinalsToSortedList(defaultTemplateOrdinals)
 		if err != nil {
 			return nil, err
 		}
@@ -350,7 +350,7 @@ func generateInstanceNamesWithOrdinalList(parentName, templateName string,
 	return instanceNameList, nil
 }
 
-func convertOrdinalsToSortedList(ordinals kbappsv1.Ordinals) ([]int32, error) {
+func ConvertOrdinalsToSortedList(ordinals kbappsv1.Ordinals) ([]int32, error) {
 	ordinalList := sets.New(ordinals.Discrete...)
 	for _, item := range ordinals.Ranges {
 		start := item.Start
diff --git a/pkg/operations/volume_expansion.go b/pkg/operations/volume_expansion.go
index b2cb172af..23a62c726 100644
--- a/pkg/operations/volume_expansion.go
+++ b/pkg/operations/volume_expansion.go
@@ -49,6 +49,8 @@ type volumeExpansionHelper struct {
 	vctName              string
 	expectCount          int
 	offlineInstanceNames []string
+	templateName         string
+	ordinals             appsv1.Ordinals
 }
 
 var _ OpsHandler = volumeExpansionOpsHandler{}
@@ -137,6 +139,18 @@ func (ve volumeExpansionOpsHandler) ReconcileAction(reqCtx intctrlutil.RequestCt
 					vctName:              vct.Name,
 					offlineInstanceNames: compSpec.OfflineInstances,
 				})
+				for _, template := range compSpec.Instances {
+					// todo: consider instance template with volumeClaimTemplates
+					veHelpers = append(veHelpers, volumeExpansionHelper{
+						compOps:              compOps,
+						fullComponentName:    fullComponentName,
+						expectCount:          int(*template.Replicas),
+						vctName:              vct.Name,
+						offlineInstanceNames: compSpec.OfflineInstances,
+						templateName:         template.Name,
+						ordinals:             template.Ordinals,
+					})
+				}
 			}
 		}
 	}
@@ -285,6 +299,9 @@ func (ve volumeExpansionOpsHandler) handleVCTExpansionProgress(reqCtx intctrluti
 		completedCount int
 		err            error
 	)
+	if veHelper.expectCount <= 0 {
+		return 0, 0, nil
+	}
 	matchingLabels := client.MatchingLabels{
 		constant.AppInstanceLabelKey:             opsRes.Cluster.Name,
 		constant.VolumeClaimTemplateNameLabelKey: veHelper.vctName,
@@ -295,7 +312,11 @@ func (ve volumeExpansionOpsHandler) handleVCTExpansionProgress(reqCtx intctrluti
 		return 0, 0, err
 	}
 	workloadName := constant.GenerateWorkloadNamePattern(opsRes.Cluster.Name, veHelper.fullComponentName)
-	instanceNames, err := instanceset.GenerateInstanceNamesFromTemplate(workloadName, "", int32(veHelper.expectCount), veHelper.offlineInstanceNames, nil)
+	ordinalList, err := instanceset.ConvertOrdinalsToSortedList(veHelper.ordinals)
+	if err != nil {
+		return 0, 0, err
+	}
+	instanceNames, err := instanceset.GenerateInstanceNamesFromTemplate(workloadName, veHelper.templateName, int32(veHelper.expectCount), veHelper.offlineInstanceNames, ordinalList)
 	if err != nil {
 		return 0, 0, err
 	}



Test patch:
diff --git a/pkg/controller/instanceset/instance_util_test.go b/pkg/controller/instanceset/instance_util_test.go
index 8cc9b3caf..f43dcd94b 100644
--- a/pkg/controller/instanceset/instance_util_test.go
+++ b/pkg/controller/instanceset/instance_util_test.go
@@ -397,7 +397,7 @@ var _ = Describe("instance util test", func() {
 				},
 				Discrete: []int32{0, 6},
 			}
-			ordinalList, err := convertOrdinalsToSortedList(ordinals)
+			ordinalList, err := ConvertOrdinalsToSortedList(ordinals)
 			Expect(err).Should(BeNil())
 			sets.New(ordinalList...).Equal(sets.New[int32](0, 2, 3, 4, 6))
 		})
@@ -412,7 +412,7 @@ var _ = Describe("instance util test", func() {
 				},
 				Discrete: []int32{0},
 			}
-			ordinalList, err := convertOrdinalsToSortedList(ordinals)
+			ordinalList, err := ConvertOrdinalsToSortedList(ordinals)
 			errExpected := fmt.Errorf("range's end(%v) must >= start(%v)", 2, 4)
 			Expect(err).Should(Equal(errExpected))
 			Expect(ordinalList).Should(BeNil())
diff --git a/pkg/operations/volume_expansion_test.go b/pkg/operations/volume_expansion_test.go
index c58087ad2..2bfd21c50 100644
--- a/pkg/operations/volume_expansion_test.go
+++ b/pkg/operations/volume_expansion_test.go
@@ -21,6 +21,7 @@ package operations
 
 import (
 	"fmt"
+	"strings"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -32,6 +33,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kubectl/pkg/util/storage"
+	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	appsv1 "github.com/apecloud/kubeblocks/apis/apps/v1"
@@ -67,9 +69,14 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() {
 		// delete cluster(and all dependent sub-resources), cluster definition
 		testapps.ClearClusterResources(&testCtx)
 
+		// delete component definition resources
+		testapps.ClearComponentResourcesWithRemoveFinalizerOption(&testCtx)
+
 		// delete rest resources
 		inNS := client.InNamespace(testCtx.DefaultNamespace)
 		ml := client.HasLabels{testCtx.TestObjLabelKey}
+		// delete pvc resources
+		testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, generics.PersistentVolumeClaimSignature, true, inNS, ml)
 		// namespaced
 		testapps.ClearResources(&testCtx, generics.OpsRequestSignature, inNS, ml)
 		// non-namespaced
@@ -88,6 +95,25 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() {
 			constant.KBAppComponentLabelKey, consensusCompName).SetStorage("2Gi").SetStorageClass(storageClassName).CheckedCreate(&testCtx)
 	}
 
+	createItsPVC := func(clusterName, scName, vctName, pvcName, cmpName string) {
+		var itsName string
+		parts := strings.Split(pvcName, cmpName+"-")
+		if len(parts) == 2 && strings.Contains(parts[1], "-") {
+			p := strings.Split(parts[1], "-")
+			itsName = p[0]
+			testapps.NewPersistentVolumeClaimFactory(testCtx.DefaultNamespace, pvcName, clusterName,
+				consensusCompName, testapps.DataVolumeName).AddLabels(constant.AppInstanceLabelKey, clusterName,
+				constant.VolumeClaimTemplateNameLabelKey, testapps.DataVolumeName,
+				constant.KBAppComponentLabelKey, consensusCompName, constant.KBAppInstanceTemplateLabelKey,
+				itsName).SetStorage("2Gi").SetStorageClass(storageClassName).CheckedCreate(&testCtx)
+		} else {
+			testapps.NewPersistentVolumeClaimFactory(testCtx.DefaultNamespace, pvcName, clusterName,
+				consensusCompName, testapps.DataVolumeName).AddLabels(constant.AppInstanceLabelKey, clusterName,
+				constant.VolumeClaimTemplateNameLabelKey, testapps.DataVolumeName,
+				constant.KBAppComponentLabelKey, consensusCompName).SetStorage("2Gi").SetStorageClass(storageClassName).CheckedCreate(&testCtx)
+		}
+	}
+
 	// getVolumeClaimNames gets all PVC names of component compName.
 	//
 	// cluster.Spec.GetComponentByName(compName).VolumeClaimTemplates[*].Name will be used if no claimNames provided
@@ -140,16 +166,73 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() {
 		return pvcNames
 	}
 
-	initResourcesForVolumeExpansion := func(clusterObject *appsv1.Cluster, opsRes *OpsResource, storage string, replicas int) (*opsv1alpha1.OpsRequest, []string) {
-		pvcNames := getVolumeClaimNames(opsRes.Cluster, consensusCompName)
-		for _, pvcName := range pvcNames {
-			createPVC(clusterObject.Name, storageClassName, vctName, pvcName)
-			// mock pvc is Bound
-			Expect(testapps.GetAndChangeObjStatus(&testCtx, client.ObjectKey{Name: pvcName, Namespace: testCtx.DefaultNamespace}, func(pvc *corev1.PersistentVolumeClaim) {
-				pvc.Status.Phase = corev1.ClaimBound
-			})()).ShouldNot(HaveOccurred())
+	// getItsVolumeClaimNames gets all PVC names of component compName and instanceTemplateName.
+	//
+	// cluster.Spec.GetComponentByName(compName).VolumeClaimTemplates[*].Name will be used if no claimNames provided
+	//
+	// nil return if:
+	//   1. component compName not found or
+	//   2. len(VolumeClaimTemplates)==0 or
+	//   3. any claimNames not found
+	getItsVolumeClaimNames := func(cluster *appsv1.Cluster, compName string, claimNames ...string) []string {
+		if cluster == nil {
+			return nil
+		}
+		comp := cluster.Spec.GetComponentByName(compName)
+		if comp == nil {
+			return nil
+		}
+		if len(comp.VolumeClaimTemplates) == 0 {
+			return nil
+		}
+		if len(claimNames) == 0 {
+			for _, template := range comp.VolumeClaimTemplates {
+				claimNames = append(claimNames, template.Name)
+			}
+		}
+		allExist := true
+		for _, name := range claimNames {
+			found := false
+			for _, template := range comp.VolumeClaimTemplates {
+				if template.Name == name {
+					found = true
+					break
+				}
+			}
+			if !found {
+				allExist = false
+				break
+			}
+		}
+		if !allExist {
+			return nil
+		}
 
+		pvcNames := make([]string, 0)
+		for _, claimName := range claimNames {
+			restReplicas := comp.Replicas
+			for i := 0; i < int(comp.Replicas); i++ {
+				// here we did not handle ordinals in instance template
+				for _, its := range comp.Instances {
+					for j := 0; j < int(*its.Replicas); j++ {
+						pvcName := fmt.Sprintf("%s-%s-%s-%s-%d", claimName, cluster.Name, compName, its.Name, j)
+						pvcNames = append(pvcNames, pvcName)
+						restReplicas--
+						i++
+					}
+				}
+				for x := 0; x < int(restReplicas); x++ {
+					pvcName := fmt.Sprintf("%s-%s-%s-%d", claimName, cluster.Name, compName, x)
+					pvcNames = append(pvcNames, pvcName)
+					i++
+				}
+			}
 		}
+		return pvcNames
+	}
+
+	initResourcesForVolumeExpansionBase := func(clusterObject *appsv1.Cluster, opsRes *OpsResource, storage string, replicas int, preparePvc func() []string) (*opsv1alpha1.OpsRequest, []string) {
+		pvcNames := preparePvc()
 		currRandomStr := testCtx.GetRandomStr()
 		ops := testops.NewOpsRequestObj("volumeexpansion-ops-"+currRandomStr, testCtx.DefaultNamespace,
 			clusterObject.Name, opsv1alpha1.VolumeExpansionType)
@@ -171,10 +254,44 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() {
 		return ops, pvcNames
 	}
 
+	initResourceForItsVolumeExpansion := func(clusterObject *appsv1.Cluster, opsRes *OpsResource, storage string, replicas int) (*opsv1alpha1.OpsRequest, []string) {
+		return initResourcesForVolumeExpansionBase(clusterObject, opsRes, storage, replicas, func() []string {
+			pvcNames := getItsVolumeClaimNames(opsRes.Cluster, consensusCompName)
+			for _, pvcName := range pvcNames {
+				createItsPVC(clusterObject.Name, storageClassName, vctName, pvcName, consensusCompName)
+				// mock pvc is Bound
+				Expect(testapps.GetAndChangeObjStatus(&testCtx, client.ObjectKey{Name: pvcName, Namespace: testCtx.DefaultNamespace}, func(pvc *corev1.PersistentVolumeClaim) {
+					pvc.Status.Phase = corev1.ClaimBound
+				})()).ShouldNot(HaveOccurred())
+
+			}
+			return pvcNames
+		})
+	}
+
+	initResourcesForVolumeExpansion := func(clusterObject *appsv1.Cluster, opsRes *OpsResource, storage string, replicas int) (*opsv1alpha1.OpsRequest, []string) {
+		return initResourcesForVolumeExpansionBase(clusterObject, opsRes, storage, replicas, func() []string {
+			pvcNames := getVolumeClaimNames(opsRes.Cluster, consensusCompName)
+			for _, pvcName := range pvcNames {
+				createPVC(clusterObject.Name, storageClassName, vctName, pvcName)
+				// mock pvc is Bound
+				Expect(testapps.GetAndChangeObjStatus(&testCtx, client.ObjectKey{Name: pvcName, Namespace: testCtx.DefaultNamespace}, func(pvc *corev1.PersistentVolumeClaim) {
+					pvc.Status.Phase = corev1.ClaimBound
+				})()).ShouldNot(HaveOccurred())
+
+			}
+			return pvcNames
+		})
+	}
+
 	mockVolumeExpansionActionAndReconcile := func(reqCtx intctrlutil.RequestCtx, opsRes *OpsResource, newOps *opsv1alpha1.OpsRequest, pvcNames []string) {
 		// first step, validate ops and update phase to Creating
 		_, err := GetOpsManager().Do(reqCtx, k8sClient, opsRes)
 		Expect(err).Should(BeNil())
+		// ops phase should not failed
+		Eventually(testapps.CheckObj(&testCtx, client.ObjectKey{Name: newOps.Name, Namespace: testCtx.DefaultNamespace}, func(g Gomega, tmpOps *opsv1alpha1.OpsRequest) {
+			g.Expect(tmpOps.Status.Phase).To(Not(Equal(opsv1alpha1.OpsFailedPhase)))
+		})).Should(Succeed())
 
 		// next step, do volume-expand action
 		_, err = GetOpsManager().Do(reqCtx, k8sClient, opsRes)
@@ -203,7 +320,8 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() {
 		clusterObject *appsv1.Cluster,
 		opsRes *OpsResource,
 		requestStorage,
-		actualStorage string) {
+		actualStorage string,
+		initResourceFunc func(clusterObject *appsv1.Cluster, opsRes *OpsResource, storage string, replicas int) (*opsv1alpha1.OpsRequest, []string)) {
 		// mock cluster is Running to support volume expansion ops
 		Expect(testapps.ChangeObjStatus(&testCtx, clusterObject, func() {
 			clusterObject.Status.Phase = appsv1.RunningClusterPhase
@@ -211,7 +329,7 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() {
 
 		// init resources for volume expansion
 		comp := clusterObject.Spec.GetComponentByName(consensusCompName)
-		newOps, pvcNames := initResourcesForVolumeExpansion(clusterObject, opsRes, requestStorage, int(comp.Replicas))
+		newOps, pvcNames := initResourceFunc(clusterObject, opsRes, requestStorage, int(comp.Replicas))
 
 		By("mock run volumeExpansion action and reconcileAction")
 		mockVolumeExpansionActionAndReconcile(reqCtx, opsRes, newOps, pvcNames)
@@ -303,13 +421,52 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() {
 			})).ShouldNot(HaveOccurred())
 
 			By("Test VolumeExpansion with consistent storageSize")
-			testVolumeExpansion(reqCtx, clusterObject, opsRes, "3Gi", "3Gi")
+			testVolumeExpansion(reqCtx, clusterObject, opsRes, "3Gi", "3Gi", initResourcesForVolumeExpansion)
 
 			By("Test VolumeExpansion with inconsistent storageSize but it is valid")
-			testVolumeExpansion(reqCtx, clusterObject, opsRes, "5G", "5Gi")
+			testVolumeExpansion(reqCtx, clusterObject, opsRes, "5G", "5Gi", initResourcesForVolumeExpansion)
 
 			By("Test delete the Running VolumeExpansion OpsRequest")
 			testDeleteRunningVolumeExpansion(clusterObject, opsRes)
 		})
+
+		It("VolumeExpandsion with InstanceTemplate", func() {
+			reqCtx := intctrlutil.RequestCtx{Ctx: ctx}
+			_, clusterObject := testapps.InitConsensusMysql(&testCtx, clusterName, compDefName, consensusCompName)
+			// all pods use instance template
+			Expect(testapps.ChangeObj(&testCtx, clusterObject, func(lc *appsv1.Cluster) {
+				lc.Spec.ComponentSpecs[0].Instances = []appsv1.InstanceTemplate{
+					{
+						Name:     "foo",
+						Replicas: ptr.To(int32(1)),
+					},
+					{
+						Name:     "bar",
+						Replicas: ptr.To(int32(2)),
+					},
+				}
+			})).ShouldNot(HaveOccurred())
+			// init storageClass
+			sc := testapps.CreateStorageClass(&testCtx, storageClassName, true)
+			Expect(testapps.ChangeObj(&testCtx, sc, func(lsc *storagev1.StorageClass) {
+				lsc.Annotations = map[string]string{storage.IsDefaultStorageClassAnnotation: "true"}
+			})).ShouldNot(HaveOccurred())
+			opsRes := &OpsResource{
+				Cluster:  clusterObject,
+				Recorder: k8sManager.GetEventRecorderFor("opsrequest-controller"),
+			}
+			By("Test OpsManager.MainEnter function with ClusterOps")
+			Expect(testapps.ChangeObjStatus(&testCtx, clusterObject, func() {
+				clusterObject.Status.Phase = appsv1.RunningClusterPhase
+				clusterObject.Status.Components = map[string]appsv1.ClusterComponentStatus{
+					consensusCompName: {
+						Phase: appsv1.RunningComponentPhase,
+					},
+				}
+			})).ShouldNot(HaveOccurred())
+
+			By("Test volume expansion with instance template")
+			testVolumeExpansion(reqCtx, clusterObject, opsRes, "3Gi", "3Gi", initResourceForItsVolumeExpansion)
+		})
 	})
 })