Repository: apecloud/kubeblocks

Issue:
[BUG] cluster with multiple shards, components have no status.
![Image](https://github.com/user-attachments/assets/d53885c8-ff55-4f28-848f-8715fe8f73ed)

If we set shards to 1 or 2, the cluster goes running very fast.
If we set shards to 3, cmp has not status at all , and it will take quite a long time to have a RUNNIING status.

```yaml
apiVersion: apps.kubeblocks.io/v1
kind: Cluster
metadata:
  name: cluster-sharding
  namespace: default
spec:
  terminationPolicy: Delete
  componentSpecs:
    - name: ch-keeper # create clickhouse keeper
      componentDef: clickhouse-keeper-24
      replicas: 1
      resources:
        limits:
          cpu: "1"
          memory: "2Gi"
        requests:
          cpu: "1"
          memory: "2Gi"
      volumeClaimTemplates:
        - name: data
          spec:
            accessModes:
              - ReadWriteOnce
            resources:
              requests:
                storage: 10Gi
  shardings:
    - name: shard
      shards: 3 # need 3 shard
      template:
        name: clickhouse  # each shard is a clickhouse component, with 2 replicas
        componentDef: clickhouse-24
        replicas: 2
        systemAccounts:
          - name: admin # name of the system account
            secretRef:
              name: udf-shard-account-info
              namespace: default
        resources:
          limits:
            cpu: "1"
            memory: "2Gi"
          requests:
            cpu: "1"
            memory: "2Gi"
        volumeClaimTemplates:
          - name: data
            spec:
              accessModes:
                - ReadWriteOnce
              resources:
                requests:
                  storage: 20Gi
---
apiVersion: v1
kind: Secret
metadata:
  name: udf-shard-account-info
  namespace: default
type: Opaque
data:
  password: cGFzc3dvcmQxMjM=  # 'password123' in base64
```

Hints from issue comments:

Solution patch:
diff --git a/controllers/apps/cluster/transformer_cluster_sharding_account.go b/controllers/apps/cluster/transformer_cluster_sharding_account.go
index 07b9571e5..5b7b3002d 100644
--- a/controllers/apps/cluster/transformer_cluster_sharding_account.go
+++ b/controllers/apps/cluster/transformer_cluster_sharding_account.go
@@ -29,10 +29,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	appsv1 "github.com/apecloud/kubeblocks/apis/apps/v1"
+	appsutil "github.com/apecloud/kubeblocks/controllers/apps/util"
 	"github.com/apecloud/kubeblocks/pkg/common"
 	"github.com/apecloud/kubeblocks/pkg/constant"
 	"github.com/apecloud/kubeblocks/pkg/controller/builder"
-	"github.com/apecloud/kubeblocks/pkg/controller/factory"
 	"github.com/apecloud/kubeblocks/pkg/controller/graph"
 	"github.com/apecloud/kubeblocks/pkg/controller/model"
 )
@@ -118,7 +118,10 @@ func (t *clusterShardingAccountTransformer) newSystemAccountSecret(transCtx *clu
 	if err != nil {
 		return nil, err
 	}
-	password := t.buildPassword(transCtx, account, sharding.Name)
+	password, err := t.buildPassword(transCtx, account, sharding.Name)
+	if err != nil {
+		return nil, err
+	}
 	return t.newAccountSecretWithPassword(transCtx, sharding, accountName, password)
 }
 
@@ -154,12 +157,15 @@ func (t *clusterShardingAccountTransformer) definedSystemAccount(transCtx *clust
 	return appsv1.SystemAccount{}, fmt.Errorf("system account %s not found in component definition %s", accountName, compDef.Name)
 }
 
-func (t *clusterShardingAccountTransformer) buildPassword(transCtx *clusterTransformContext, account appsv1.SystemAccount, shardingName string) []byte {
-	password := []byte(factory.GetRestoreSystemAccountPassword(transCtx.Cluster.Annotations, shardingName, account.Name))
+func (t *clusterShardingAccountTransformer) buildPassword(transCtx *clusterTransformContext, account appsv1.SystemAccount, shardingName string) ([]byte, error) {
+	password, err := appsutil.GetRestoreSystemAccountPassword(transCtx.Context, transCtx.Client, transCtx.Cluster.Annotations, shardingName, account.Name)
+	if err != nil {
+		return nil, fmt.Errorf("failed to restore password for system account %s of shard %s from annotation", account.Name, shardingName)
+	}
 	if len(password) == 0 {
 		password = t.generatePassword(account)
 	}
-	return password
+	return password, nil
 }
 
 func (t *clusterShardingAccountTransformer) generatePassword(account appsv1.SystemAccount) []byte {
diff --git a/controllers/apps/component/transformer_component_account.go b/controllers/apps/component/transformer_component_account.go
index 3e72ee762..3445db8f5 100644
--- a/controllers/apps/component/transformer_component_account.go
+++ b/controllers/apps/component/transformer_component_account.go
@@ -168,14 +168,17 @@ func (t *componentAccountTransformer) buildAccountHash(account synthesizedSystem
 func (t *componentAccountTransformer) buildAccountSecret(ctx *componentTransformContext,
 	synthesizeComp *component.SynthesizedComponent, account synthesizedSystemAccount) (*corev1.Secret, error) {
 	var password []byte
+	var err error
 	switch {
 	case account.SecretRef != nil:
-		var err error
 		if password, err = t.getPasswordFromSecret(ctx, account); err != nil {
 			return nil, err
 		}
 	default:
-		password = t.buildPassword(ctx, account)
+		password, err = t.buildPassword(ctx, account)
+		if err != nil {
+			return nil, err
+		}
 	}
 	if len(password) > maximumPasswordLength {
 		return nil, errPasswordTooLong
@@ -203,18 +206,22 @@ func (t *componentAccountTransformer) getPasswordFromSecret(ctx graph.TransformC
 	return secret.Data[passwordKey], nil
 }
 
-func (t *componentAccountTransformer) buildPassword(ctx *componentTransformContext, account synthesizedSystemAccount) []byte {
+func (t *componentAccountTransformer) buildPassword(ctx *componentTransformContext, account synthesizedSystemAccount) ([]byte, error) {
 	// get restore password if exists during recovery.
-	password := factory.GetRestoreSystemAccountPassword(ctx.SynthesizeComponent.Annotations, ctx.SynthesizeComponent.Name, account.Name)
-	if account.InitAccount && password == "" {
+	password, err := appsutil.GetRestoreSystemAccountPassword(ctx.Context, ctx.Client,
+		ctx.SynthesizeComponent.Annotations, ctx.SynthesizeComponent.Name, account.Name)
+	if err != nil {
+		return nil, fmt.Errorf("failed to restore password for system account %s of component %s from annotation", account.Name, ctx.SynthesizeComponent.Name)
+	}
+	if account.InitAccount && len(password) == 0 {
 		// initAccount can also restore from factory.GetRestoreSystemAccountPassword(ctx.SynthesizeComponent, account).
 		// This is compatibility processing.
-		password = factory.GetRestorePassword(ctx.SynthesizeComponent)
+		password = []byte(factory.GetRestorePassword(ctx.SynthesizeComponent))
 	}
-	if password == "" {
-		return t.generatePassword(account)
+	if len(password) == 0 {
+		return t.generatePassword(account), nil
 	}
-	return []byte(password)
+	return password, nil
 }
 
 func (t *componentAccountTransformer) generatePassword(account synthesizedSystemAccount) []byte {
diff --git a/controllers/apps/component/transformer_component_notifier.go b/controllers/apps/component/transformer_component_notifier.go
index 42d06ed10..e38a5bb16 100644
--- a/controllers/apps/component/transformer_component_notifier.go
+++ b/controllers/apps/component/transformer_component_notifier.go
@@ -129,6 +129,10 @@ func (t *componentNotifierTransformer) notify(transCtx *componentTransformContex
 		return client.IgnoreNotFound(err)
 	}
 
+	if model.IsObjectUpdating(comp) {
+		return nil // skip if the target component is being updated
+	}
+
 	compCopy := comp.DeepCopy()
 	if compCopy.Annotations == nil {
 		compCopy.Annotations = make(map[string]string)
diff --git a/controllers/apps/util/utils.go b/controllers/apps/util/utils.go
index 388043b89..71fc23614 100644
--- a/controllers/apps/util/utils.go
+++ b/controllers/apps/util/utils.go
@@ -20,6 +20,9 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package util
 
 import (
+	"context"
+	"encoding/json"
+	"fmt"
 	"reflect"
 	"time"
 
@@ -27,8 +30,11 @@ import (
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1"
 	workloads "github.com/apecloud/kubeblocks/apis/workloads/v1"
+	"github.com/apecloud/kubeblocks/pkg/constant"
 	intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil"
+	viper "github.com/apecloud/kubeblocks/pkg/viperx"
 )
 
 var RequeueDuration = time.Millisecond * 1000
@@ -121,3 +127,59 @@ func IsOwnedByInstanceSet(obj client.Object) bool {
 	}
 	return false
 }
+
+func GetRestoreSystemAccountPassword(
+	ctx context.Context,
+	cli client.Reader,
+	annotations map[string]string,
+	componentName,
+	accountName string,
+) ([]byte, error) {
+	valueString := annotations[constant.RestoreFromBackupAnnotationKey]
+	if len(valueString) == 0 {
+		return nil, nil
+	}
+	backupMap := map[string]map[string]string{}
+	err := json.Unmarshal([]byte(valueString), &backupMap)
+	if err != nil {
+		return nil, err
+	}
+	backupSource, ok := backupMap[componentName]
+	if !ok {
+		return nil, nil
+	}
+	name, ok := backupSource[constant.BackupNameKeyForRestore]
+	if !ok || len(name) == 0 {
+		return nil, fmt.Errorf("backup name not found in restore annotation")
+	}
+	namespace, ok := backupSource[constant.BackupNamespaceKeyForRestore]
+	if !ok || len(namespace) == 0 {
+		return nil, fmt.Errorf("backup namespace not found in restore annotation")
+	}
+	backup := &dpv1alpha1.Backup{}
+	if err := cli.Get(ctx, client.ObjectKey{
+		Name:      name,
+		Namespace: namespace,
+	}, backup); err != nil {
+		return nil, err
+	}
+	systemAccountsMap := map[string]string{}
+	encryptedSystemAccountsString := backup.Annotations[constant.EncryptedSystemAccountsAnnotationKey]
+	if encryptedSystemAccountsString != "" {
+		encryptedSystemAccountsMap := map[string]map[string]string{}
+		if err = json.Unmarshal([]byte(encryptedSystemAccountsString), &encryptedSystemAccountsMap); err != nil {
+			return nil, err
+		}
+		if val, ok := encryptedSystemAccountsMap[componentName]; ok {
+			systemAccountsMap = val
+		}
+	}
+
+	e := intctrlutil.NewEncryptor(viper.GetString(constant.CfgKeyDPEncryptionKey))
+	encryptedPwd, ok := systemAccountsMap[accountName]
+	if !ok {
+		return nil, nil
+	}
+	password, err := e.Decrypt([]byte(encryptedPwd))
+	return []byte(password), err
+}
diff --git a/pkg/constant/backup.go b/pkg/constant/backup.go
index 4ab6d5032..b93ef28dd 100755
--- a/pkg/constant/backup.go
+++ b/pkg/constant/backup.go
@@ -28,5 +28,4 @@ const (
 	EnvForRestore                     = "restoreEnv"
 	ParametersForRestore              = "restoreParameters"
 	ConnectionPassword                = "connectionPassword"
-	EncryptedSystemAccounts           = "encryptedSystemAccounts"
 )
diff --git a/pkg/controller/factory/builder.go b/pkg/controller/factory/builder.go
index 5fedd8203..99e794c64 100644
--- a/pkg/controller/factory/builder.go
+++ b/pkg/controller/factory/builder.go
@@ -249,39 +249,6 @@ func GetRestorePassword(synthesizedComp *component.SynthesizedComponent) string
 	return password
 }
 
-// GetRestoreSystemAccountPassword gets restore password if exists during recovery.
-func GetRestoreSystemAccountPassword(annotations map[string]string, componentName, accountName string) string {
-	valueString := annotations[constant.RestoreFromBackupAnnotationKey]
-	if len(valueString) == 0 {
-		return ""
-	}
-	backupMap := map[string]map[string]string{}
-	err := json.Unmarshal([]byte(valueString), &backupMap)
-	if err != nil {
-		return ""
-	}
-	backupSource, ok := backupMap[componentName]
-	if !ok {
-		return ""
-	}
-	systemAccountsString, ok := backupSource[constant.EncryptedSystemAccounts]
-	if !ok {
-		return ""
-	}
-	systemAccountsMap := map[string]string{}
-	err = json.Unmarshal([]byte(systemAccountsString), &systemAccountsMap)
-	if err != nil {
-		return ""
-	}
-	e := intctrlutil.NewEncryptor(viper.GetString(constant.CfgKeyDPEncryptionKey))
-	encryptedPwd, ok := systemAccountsMap[accountName]
-	if !ok {
-		return ""
-	}
-	password, _ := e.Decrypt([]byte(encryptedPwd))
-	return password
-}
-
 // TODO: add dynamicLabels and dynamicAnnotations by @zhangtao
 
 func BuildConfigMapWithTemplate(cluster *kbappsv1.Cluster,
diff --git a/pkg/dataprotection/restore/utils.go b/pkg/dataprotection/restore/utils.go
index 2d1773b34..3987414dc 100644
--- a/pkg/dataprotection/restore/utils.go
+++ b/pkg/dataprotection/restore/utils.go
@@ -353,16 +353,6 @@ func GetRestoreFromBackupAnnotation(
 	if connectionPassword != "" {
 		restoreInfoMap[constant.ConnectionPassword] = connectionPassword
 	}
-	encryptedSystemAccountsString := backup.Annotations[constant.EncryptedSystemAccountsAnnotationKey]
-	if encryptedSystemAccountsString != "" {
-		encryptedSystemAccountsMap := map[string]map[string]string{}
-		_ = json.Unmarshal([]byte(encryptedSystemAccountsString), &encryptedSystemAccountsMap)
-		// only set systemAccounts owned by this component
-		if encryptedSystemAccountsMap[componentName] != nil {
-			encryptedComponentSystemAccountsBytes, _ := json.Marshal(encryptedSystemAccountsMap[componentName])
-			restoreInfoMap[constant.EncryptedSystemAccounts] = string(encryptedComponentSystemAccountsBytes)
-		}
-	}
 	restoreForClusterMap := map[string]map[string]string{}
 	restoreForClusterMap[componentName] = restoreInfoMap
 	bytes, err := json.Marshal(restoreForClusterMap)



Test patch:
diff --git a/controllers/apps/util/utils_test.go b/controllers/apps/util/utils_test.go
index f34d5fc6f..75a349251 100644
--- a/controllers/apps/util/utils_test.go
+++ b/controllers/apps/util/utils_test.go
@@ -20,6 +20,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package util
 
 import (
+	"context"
 	"fmt"
 	"reflect"
 	"testing"
@@ -27,11 +28,17 @@ import (
 	"github.com/stretchr/testify/assert"
 	appsv1 "k8s.io/api/apps/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1"
+	dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1"
 	workloads "github.com/apecloud/kubeblocks/apis/workloads/v1"
+	"github.com/apecloud/kubeblocks/pkg/constant"
+	intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil"
+	viper "github.com/apecloud/kubeblocks/pkg/viperx"
 )
 
 func TestReflect(t *testing.T) {
@@ -79,3 +86,160 @@ func TestIsOwnedByInstanceSet(t *testing.T) {
 	}
 	assert.False(t, IsOwnedByInstanceSet(its))
 }
+
+func TestGetRestoreSystemAccountPassword(t *testing.T) {
+	encryptor := intctrlutil.NewEncryptor(viper.GetString(constant.CfgKeyDPEncryptionKey))
+	encryptedPwd, _ := encryptor.Encrypt([]byte("test-password"))
+
+	tests := []struct {
+		name        string
+		annotations map[string]string
+		backup      *dpv1alpha1.Backup
+		component   string
+		account     string
+		wantPwd     []byte
+		wantErr     bool
+	}{
+		{
+			name: "normal case",
+			annotations: map[string]string{
+				constant.RestoreFromBackupAnnotationKey: `{"comp1":{"name":"backup1","namespace":"default"}}`,
+			},
+			backup: &dpv1alpha1.Backup{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "backup1",
+					Namespace: "default",
+					Annotations: map[string]string{
+						constant.EncryptedSystemAccountsAnnotationKey: fmt.Sprintf(`{"comp1":{"account1":"%s"}}`, encryptedPwd),
+					},
+				},
+			},
+			component: "comp1",
+			account:   "account1",
+			wantPwd:   []byte("test-password"),
+			wantErr:   false,
+		},
+		{
+			name: "empty restore annotation",
+			annotations: map[string]string{
+				constant.RestoreFromBackupAnnotationKey: "",
+			},
+			backup:    &dpv1alpha1.Backup{},
+			component: "comp1",
+			account:   "account1",
+			wantPwd:   nil,
+			wantErr:   false,
+		},
+		{
+			name: "invalid restore annotation json",
+			annotations: map[string]string{
+				constant.RestoreFromBackupAnnotationKey: "invalid-json",
+			},
+			backup:    &dpv1alpha1.Backup{},
+			component: "comp1",
+			account:   "account1",
+			wantPwd:   nil,
+			wantErr:   true,
+		},
+		{
+			name: "missing backup name in annotation",
+			annotations: map[string]string{
+				constant.RestoreFromBackupAnnotationKey: `{"comp1":{}}`,
+			},
+			backup:    &dpv1alpha1.Backup{},
+			component: "comp1",
+			account:   "account1",
+			wantPwd:   nil,
+			wantErr:   true,
+		},
+		{
+			name: "missing backup namespace in annotation",
+			annotations: map[string]string{
+				constant.RestoreFromBackupAnnotationKey: `{"comp1":{"name":"backup1"}}`,
+			},
+			backup:    &dpv1alpha1.Backup{},
+			component: "comp1",
+			account:   "account1",
+			wantPwd:   nil,
+			wantErr:   true,
+		},
+		{
+			name: "invalid encrypted accounts json in backup",
+			annotations: map[string]string{
+				constant.RestoreFromBackupAnnotationKey: `{"comp1":{"name":"backup1","namespace":"default"}}`,
+			},
+			backup: &dpv1alpha1.Backup{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "backup1",
+					Namespace: "default",
+					Annotations: map[string]string{
+						constant.EncryptedSystemAccountsAnnotationKey: "invalid-json",
+					},
+				},
+			},
+			component: "comp1",
+			account:   "account1",
+			wantPwd:   nil,
+			wantErr:   true,
+		},
+		{
+			name: "account not found in backup",
+			annotations: map[string]string{
+				constant.RestoreFromBackupAnnotationKey: `{"comp1":{"name":"backup1","namespace":"default"}}`,
+			},
+			backup: &dpv1alpha1.Backup{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "backup1",
+					Namespace: "default",
+					Annotations: map[string]string{
+						constant.EncryptedSystemAccountsAnnotationKey: `{"comp1":{"other-account":"pwd"}}`,
+					},
+				},
+			},
+			component: "comp1",
+			account:   "account1",
+			wantPwd:   nil,
+			wantErr:   false,
+		},
+		{
+			name: "component not found in encrypted accounts",
+			annotations: map[string]string{
+				constant.RestoreFromBackupAnnotationKey: `{"comp1":{"name":"backup1","namespace":"default"}}`,
+			},
+			backup: &dpv1alpha1.Backup{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "backup1",
+					Namespace: "default",
+					Annotations: map[string]string{
+						constant.EncryptedSystemAccountsAnnotationKey: `{"other-comp":{"account1":"pwd"}}`,
+					},
+				},
+			},
+			component: "comp1",
+			account:   "account1",
+			wantPwd:   nil,
+			wantErr:   false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			scheme := runtime.NewScheme()
+			_ = dpv1alpha1.AddToScheme(scheme)
+			cli := fake.NewClientBuilder().WithScheme(scheme)
+			if tt.backup != nil {
+				cli = cli.WithObjects(tt.backup)
+			}
+
+			pwd, err := GetRestoreSystemAccountPassword(context.Background(), cli.Build(),
+				tt.annotations, tt.component, tt.account)
+
+			if tt.wantErr {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, tt.wantPwd, pwd)
+			}
+		})
+	}
+}