Repository: stolostron/multicluster-global-hub
Pull request #1443: fix agent can not connect to kafka in dr case
Author: login: ldpliu
Labels: name: lgtm
name: approved
name: dco-signoff: yes
Created: 2025-03-04T12:27:47Z | Merged: 2025-03-05T07:21:52Z

<!--
Thanks for creating a pull request!

If this is your first time, please make sure to review CONTRIBUTING.MD.

Please copy the appropriate `:text:` or icon to the beginning of your PR title:

:sparkles: ✨ feature
:bug: 🐛 bug fix
:book: 📖 docs
:memo: 📝 proposal
:warning: ⚠️ breaking change
:seedling: 🌱 other/misc
:question: ❓ requires manual review/categorization

-->
## Summary

## Related issue(s)
https://issues.redhat.com/browse/ACM-18120
Fixes #

## Tests
* [ ] Unit/function tests have been added and incorporated into `make unit-tests`.
* [ ] Integration tests have been added and incorporated into `make integration-test`.
* [ ] E2E tests have been added and incorporated into `make e2e-test-all`.
* [ ] List other manual tests you have done.



Diff:
diff --git a/doc/reserved_names/README.md b/doc/reserved_names/README.md
index f848b8414b..7f5ebe2de3 100644
--- a/doc/reserved_names/README.md
+++ b/doc/reserved_names/README.md
@@ -22,7 +22,7 @@ List all annotations are used by multicluster global hub.
 |global-hub.open-cluster-management.io/import-cluster-in-hosted=true\|false | This annotation is used to identify if managedhub cluster should be imported in hosted mode |
 | global-hub.open-cluster-management.io/with-inventory                | This annotation is used to identify the common inventory is deployed.                                                                  |
 | global-hub.open-cluster-management.io/with-stackrox-integration | This annotation enables the experimental integration with [Stackrox](https://github.com/stackrox).|
-| global-hub.open-cluster-management.io/resign-kafka-client-secret | This annotation is used to identify if the kafka client secret is resynced in agent.|
+| global-hub.open-cluster-management.io/kafka-cluster-id | This annotation save the kafka cluster id in transport-config secret in agent part, it is used to identify if the kafka cluster changed.|
 
 # Finalizer
 
diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go
index 46d0a5850b..3622e4a1d2 100644
--- a/pkg/constants/constants.go
+++ b/pkg/constants/constants.go
@@ -117,8 +117,8 @@ const (
 	OriginOwnerReferenceAnnotation = "global-hub.open-cluster-management.io/origin-ownerreference-uid"
 	// identy the kafka is upgrade from zookeeper mode
 	UpgradeKafkaFromZookeeperAnnotation = "global-hub.open-cluster-management.io/upgrade-from-zookeeper"
-	// resync the kafka client secret in agent
-	ResyncKafkaClientSecretAnnotation = "global-hub.open-cluster-management.io/resign-kafka-client-secret" // #nosec G101
+	// kafka-cluster-id save the current kafka cluster id
+	KafkaClusterIdAnnotation = "global-hub.open-cluster-management.io/kafka-cluster-id" // #nosec G101
 )
 
 // store all the finalizers
diff --git a/pkg/transport/controller/controller.go b/pkg/transport/controller/controller.go
index 5cbddadb83..4cbea4d8a3 100644
--- a/pkg/transport/controller/controller.go
+++ b/pkg/transport/controller/controller.go
@@ -244,41 +244,51 @@ func (c *TransportCtrl) ReconcileKafkaCredential(ctx context.Context, secret *co
 	return true, nil
 }
 
-// Resync the kafka client secret because we recreate the kafka cluster.
-// TODO: Remove the code in globalhub 1.5
+// Resync the kafka client secret because we recreate the kafka cluster in globalhub 1.4 and restore case.
 func (c *TransportCtrl) ResyncKafkaClientSecret(ctx context.Context, kafkaConn *transport.KafkaConfig, secret *corev1.Secret) error {
-	log.Debugf("resync kafka client secret: %v", kafkaConn.IsNewKafkaCluster)
-	if !kafkaConn.IsNewKafkaCluster {
+	if kafkaConn.ClusterID == "" {
 		return nil
 	}
 
+	isKafkaClusterIdEqual := true
+	// if stored kafka cluster id is same as current cluster id, return
 	if secret.Annotations != nil {
-		if _, ok := secret.Annotations[constants.ResyncKafkaClientSecretAnnotation]; ok {
+		if secret.Annotations[constants.KafkaClusterIdAnnotation] == kafkaConn.ClusterID {
+			log.Debugf("cluster id is equal")
 			return nil
 		}
+		isKafkaClusterIdEqual = false
 	}
-	if kafkaConn.ClientSecretName == "" {
-		return nil
-	}
-	signedSecret := &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      kafkaConn.ClientSecretName,
-			Namespace: secret.Namespace,
-		},
-	}
-	log.Infof("remove kafka client secret: %v", kafkaConn.ClientSecretName)
 
-	err := c.runtimeClient.Delete(ctx, signedSecret)
-	if err != nil {
-		return err
+	log.Debugf("isNewKafkaCluster: %v", kafkaConn.IsNewKafkaCluster)
+	log.Debugf("isKafkaClusterIdEqual : %v", isKafkaClusterIdEqual)
+
+	// if it's new cluster(upgrade from 1.3 to 1.4) or cluster is not equal (restore globalhub)
+	if kafkaConn.IsNewKafkaCluster || !isKafkaClusterIdEqual {
+		if kafkaConn.ClientSecretName == "" {
+			return nil
+		}
+		signedSecret := &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      kafkaConn.ClientSecretName,
+				Namespace: secret.Namespace,
+			},
+		}
+		log.Infof("remove kafka client secret: %v", kafkaConn.ClientSecretName)
+
+		err := c.runtimeClient.Delete(ctx, signedSecret)
+		if err != nil {
+			return err
+		}
 	}
+
 	transportSecret := secret.DeepCopy()
 	if transportSecret.Annotations == nil {
 		transportSecret.Annotations = make(map[string]string)
 	}
 	log.Infof("update transport config secret")
 
-	transportSecret.Annotations[constants.ResyncKafkaClientSecretAnnotation] = "true"
+	transportSecret.Annotations[constants.KafkaClusterIdAnnotation] = kafkaConn.ClusterID
 	return c.runtimeClient.Update(ctx, transportSecret)
 }
 
diff --git a/pkg/transport/controller/controller_test.go b/pkg/transport/controller/controller_test.go
index a930bcd041..ba89094839 100644
--- a/pkg/transport/controller/controller_test.go
+++ b/pkg/transport/controller/controller_test.go
@@ -11,6 +11,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
@@ -210,23 +211,30 @@ func TestTransportCtrl_ResyncKafkaClientSecret(t *testing.T) {
 		initObjects []runtime.Object
 	}{
 		{
-			name: "no new kafka",
+			name: "default install",
 			kafkaConn: &transport.KafkaConfig{
 				IsNewKafkaCluster: false,
+				ClusterID:         "0001",
+			},
+			secret: &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "default",
+					Name:      "transport-config",
+				},
 			},
-			secret: nil,
 		},
 		{
 			name: "new kafka, has synced",
 			kafkaConn: &transport.KafkaConfig{
 				IsNewKafkaCluster: true,
+				ClusterID:         "0001",
 			},
 			secret: &corev1.Secret{
 				ObjectMeta: metav1.ObjectMeta{
 					Namespace: "default",
 					Name:      "transport-config",
 					Annotations: map[string]string{
-						constants.ResyncKafkaClientSecretAnnotation: "true",
+						constants.KafkaClusterIdAnnotation: "0001",
 					},
 				},
 			},
@@ -236,6 +244,7 @@ func TestTransportCtrl_ResyncKafkaClientSecret(t *testing.T) {
 			kafkaConn: &transport.KafkaConfig{
 				IsNewKafkaCluster: true,
 				ClientSecretName:  "client-secret",
+				ClusterID:         "0001",
 			},
 			secret: &corev1.Secret{
 				ObjectMeta: metav1.ObjectMeta{
@@ -266,6 +275,13 @@ func TestTransportCtrl_ResyncKafkaClientSecret(t *testing.T) {
 			if err := c.ResyncKafkaClientSecret(context.Background(), tt.kafkaConn, tt.secret); err != nil {
 				t.Errorf("TransportCtrl.ResyncKafkaClientSecret() error = %v", err)
 			}
+			secret := &corev1.Secret{}
+			if err := c.runtimeClient.Get(context.Background(), client.ObjectKeyFromObject(tt.secret), secret); err != nil {
+				if secret.Annotations[constants.KafkaClusterIdAnnotation] != tt.kafkaConn.ClusterID {
+					t.Errorf("secret.Annotations[constants.KafkaClusterIdAnnotation]:%v,tt.kafkaConn.ClusterID:%v",
+						secret.Annotations[constants.KafkaClusterIdAnnotation], tt.kafkaConn.ClusterID)
+				}
+			}
 		})
 	}
 }


Reviews:
user: login: yanmxa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-05T02:09:02Z
user: login: yanmxa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-05T02:13:32Z
user: login: yanmxa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-05T02:21:26Z
user: login: ldpliu
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-05T02:23:10Z
user: login: ldpliu
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-05T02:39:22Z
user: login: ldpliu
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-05T02:39:32Z
user: login: clyang82
body: LGTM
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-05T06:05:31Z

Review comments:
diff_hunk: @@ -22,7 +22,7 @@ List all annotations are used by multicluster global hub.
 |global-hub.open-cluster-management.io/import-cluster-in-hosted=true\|false | This annotation is used to identify if managedhub cluster should be imported in hosted mode |
 | global-hub.open-cluster-management.io/with-inventory                | This annotation is used to identify the common inventory is deployed.                                                                  |
 | global-hub.open-cluster-management.io/with-stackrox-integration | This annotation enables the experimental integration with [Stackrox](https://github.com/stackrox).|
-| global-hub.open-cluster-management.io/resign-kafka-client-secret | This annotation is used to identify if the kafka client secret is resynced in agent.|
+| global-hub.open-cluster-management.io/kafka-cluster-id | This annotation save the kafka cluster id, it is used to identify if the kafka changed in agent.|
path: doc/reserved_names/README.md
user: login: yanmxa
body: We need to clarify which object the annotation will be added to. Over time, we might lose track of the information
created_at: 2025-03-05T02:09:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -244,44 +245,65 @@ func (c *TransportCtrl) ReconcileKafkaCredential(ctx context.Context, secret *co
 	return true, nil
 }
 
-// Resync the kafka client secret because we recreate the kafka cluster.
-// TODO: Remove the code in globalhub 1.5
+// Resync the kafka client secret because we recreate the kafka cluster in globalhub 1.4 and restore case.
 func (c *TransportCtrl) ResyncKafkaClientSecret(ctx context.Context, kafkaConn *transport.KafkaConfig, secret *corev1.Secret) error {
-	log.Debugf("resync kafka client secret: %v", kafkaConn.IsNewKafkaCluster)
-	if !kafkaConn.IsNewKafkaCluster {
+	if kafkaConn.ClusterID == "" {
 		return nil
 	}
 
+	isKafkaClusterIdEqual := true
+	// if stored kafka cluster id is same as current cluster id, return
 	if secret.Annotations != nil {
-		if _, ok := secret.Annotations[constants.ResyncKafkaClientSecretAnnotation]; ok {
+		if secret.Annotations[constants.KafkaClusterIdAnnotation] == sanitizeClusterID(kafkaConn.ClusterID) {
+			log.Debugf("cluster id is equal")
 			return nil
 		}
+		isKafkaClusterIdEqual = false
 	}
-	if kafkaConn.ClientSecretName == "" {
-		return nil
-	}
-	signedSecret := &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      kafkaConn.ClientSecretName,
-			Namespace: secret.Namespace,
-		},
-	}
-	log.Infof("remove kafka client secret: %v", kafkaConn.ClientSecretName)
 
-	err := c.runtimeClient.Delete(ctx, signedSecret)
-	if err != nil {
-		return err
+	log.Debugf("isNewKafkaCluster: %v", kafkaConn.IsNewKafkaCluster)
+	log.Debugf("isKafkaClusterIdEqual : %v", isKafkaClusterIdEqual)
+
+	// if it's new cluster(upgrade from 1.3 to 1.4) or cluster is not equal (restore globalhub)
+	if kafkaConn.IsNewKafkaCluster || !isKafkaClusterIdEqual {
+		if kafkaConn.ClientSecretName == "" {
+			return nil
+		}
+		signedSecret := &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      kafkaConn.ClientSecretName,
+				Namespace: secret.Namespace,
+			},
+		}
+		log.Infof("remove kafka client secret: %v", kafkaConn.ClientSecretName)
+
+		err := c.runtimeClient.Delete(ctx, signedSecret)
+		if err != nil {
+			return err
+		}
 	}
+
 	transportSecret := secret.DeepCopy()
 	if transportSecret.Annotations == nil {
 		transportSecret.Annotations = make(map[string]string)
 	}
 	log.Infof("update transport config secret")
 
-	transportSecret.Annotations[constants.ResyncKafkaClientSecretAnnotation] = "true"
+	transportSecret.Annotations[constants.KafkaClusterIdAnnotation] = sanitizeClusterID(kafkaConn.ClusterID)
 	return c.runtimeClient.Update(ctx, transportSecret)
 }
 
+// sanitizeClusterID replace the illegal character for label
+func sanitizeClusterID(clusterID string) string {
path: pkg/transport/controller/controller.go
user: login: yanmxa
body: Why do we need to sanitize the clusterID, and that will lose the original information
created_at: 2025-03-05T02:13:31Z
author_association: CONTRIBUTOR
diff_hunk: @@ -244,44 +245,65 @@ func (c *TransportCtrl) ReconcileKafkaCredential(ctx context.Context, secret *co
 	return true, nil
 }
 
-// Resync the kafka client secret because we recreate the kafka cluster.
-// TODO: Remove the code in globalhub 1.5
+// Resync the kafka client secret because we recreate the kafka cluster in globalhub 1.4 and restore case.
 func (c *TransportCtrl) ResyncKafkaClientSecret(ctx context.Context, kafkaConn *transport.KafkaConfig, secret *corev1.Secret) error {
-	log.Debugf("resync kafka client secret: %v", kafkaConn.IsNewKafkaCluster)
-	if !kafkaConn.IsNewKafkaCluster {
+	if kafkaConn.ClusterID == "" {
 		return nil
 	}
 
+	isKafkaClusterIdEqual := true
+	// if stored kafka cluster id is same as current cluster id, return
 	if secret.Annotations != nil {
-		if _, ok := secret.Annotations[constants.ResyncKafkaClientSecretAnnotation]; ok {
+		if secret.Annotations[constants.KafkaClusterIdAnnotation] == sanitizeClusterID(kafkaConn.ClusterID) {
+			log.Debugf("cluster id is equal")
 			return nil
 		}
+		isKafkaClusterIdEqual = false
 	}
-	if kafkaConn.ClientSecretName == "" {
-		return nil
-	}
-	signedSecret := &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      kafkaConn.ClientSecretName,
-			Namespace: secret.Namespace,
-		},
-	}
-	log.Infof("remove kafka client secret: %v", kafkaConn.ClientSecretName)
 
-	err := c.runtimeClient.Delete(ctx, signedSecret)
-	if err != nil {
-		return err
+	log.Debugf("isNewKafkaCluster: %v", kafkaConn.IsNewKafkaCluster)
+	log.Debugf("isKafkaClusterIdEqual : %v", isKafkaClusterIdEqual)
+
+	// if it's new cluster(upgrade from 1.3 to 1.4) or cluster is not equal (restore globalhub)
+	if kafkaConn.IsNewKafkaCluster || !isKafkaClusterIdEqual {
+		if kafkaConn.ClientSecretName == "" {
+			return nil
+		}
+		signedSecret := &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      kafkaConn.ClientSecretName,
+				Namespace: secret.Namespace,
+			},
+		}
+		log.Infof("remove kafka client secret: %v", kafkaConn.ClientSecretName)
+
+		err := c.runtimeClient.Delete(ctx, signedSecret)
+		if err != nil {
+			return err
+		}
 	}
+
 	transportSecret := secret.DeepCopy()
 	if transportSecret.Annotations == nil {
 		transportSecret.Annotations = make(map[string]string)
 	}
 	log.Infof("update transport config secret")
 
-	transportSecret.Annotations[constants.ResyncKafkaClientSecretAnnotation] = "true"
+	transportSecret.Annotations[constants.KafkaClusterIdAnnotation] = sanitizeClusterID(kafkaConn.ClusterID)
path: pkg/transport/controller/controller.go
user: login: yanmxa
body: The `kafkaClusterID` originally comes from the payload of the transportSecret, which is generated by the gh manifestwork. However, the annotation is added by the global hub agent.
Wouldn't adding this information risk being overridden? I noticed we've done this before - just curious :)
created_at: 2025-03-05T02:21:26Z
author_association: CONTRIBUTOR
diff_hunk: @@ -244,44 +245,65 @@ func (c *TransportCtrl) ReconcileKafkaCredential(ctx context.Context, secret *co
 	return true, nil
 }
 
-// Resync the kafka client secret because we recreate the kafka cluster.
-// TODO: Remove the code in globalhub 1.5
+// Resync the kafka client secret because we recreate the kafka cluster in globalhub 1.4 and restore case.
 func (c *TransportCtrl) ResyncKafkaClientSecret(ctx context.Context, kafkaConn *transport.KafkaConfig, secret *corev1.Secret) error {
-	log.Debugf("resync kafka client secret: %v", kafkaConn.IsNewKafkaCluster)
-	if !kafkaConn.IsNewKafkaCluster {
+	if kafkaConn.ClusterID == "" {
 		return nil
 	}
 
+	isKafkaClusterIdEqual := true
+	// if stored kafka cluster id is same as current cluster id, return
 	if secret.Annotations != nil {
-		if _, ok := secret.Annotations[constants.ResyncKafkaClientSecretAnnotation]; ok {
+		if secret.Annotations[constants.KafkaClusterIdAnnotation] == sanitizeClusterID(kafkaConn.ClusterID) {
+			log.Debugf("cluster id is equal")
 			return nil
 		}
+		isKafkaClusterIdEqual = false
 	}
-	if kafkaConn.ClientSecretName == "" {
-		return nil
-	}
-	signedSecret := &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      kafkaConn.ClientSecretName,
-			Namespace: secret.Namespace,
-		},
-	}
-	log.Infof("remove kafka client secret: %v", kafkaConn.ClientSecretName)
 
-	err := c.runtimeClient.Delete(ctx, signedSecret)
-	if err != nil {
-		return err
+	log.Debugf("isNewKafkaCluster: %v", kafkaConn.IsNewKafkaCluster)
+	log.Debugf("isKafkaClusterIdEqual : %v", isKafkaClusterIdEqual)
+
+	// if it's new cluster(upgrade from 1.3 to 1.4) or cluster is not equal (restore globalhub)
+	if kafkaConn.IsNewKafkaCluster || !isKafkaClusterIdEqual {
+		if kafkaConn.ClientSecretName == "" {
+			return nil
+		}
+		signedSecret := &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      kafkaConn.ClientSecretName,
+				Namespace: secret.Namespace,
+			},
+		}
+		log.Infof("remove kafka client secret: %v", kafkaConn.ClientSecretName)
+
+		err := c.runtimeClient.Delete(ctx, signedSecret)
+		if err != nil {
+			return err
+		}
 	}
+
 	transportSecret := secret.DeepCopy()
 	if transportSecret.Annotations == nil {
 		transportSecret.Annotations = make(map[string]string)
 	}
 	log.Infof("update transport config secret")
 
-	transportSecret.Annotations[constants.ResyncKafkaClientSecretAnnotation] = "true"
+	transportSecret.Annotations[constants.KafkaClusterIdAnnotation] = sanitizeClusterID(kafkaConn.ClusterID)
path: pkg/transport/controller/controller.go
user: login: ldpliu
body: addon framework will merge the existing annotation
created_at: 2025-03-05T02:23:10Z
author_association: CONTRIBUTOR
diff_hunk: @@ -22,7 +22,7 @@ List all annotations are used by multicluster global hub.
 |global-hub.open-cluster-management.io/import-cluster-in-hosted=true\|false | This annotation is used to identify if managedhub cluster should be imported in hosted mode |
 | global-hub.open-cluster-management.io/with-inventory                | This annotation is used to identify the common inventory is deployed.                                                                  |
 | global-hub.open-cluster-management.io/with-stackrox-integration | This annotation enables the experimental integration with [Stackrox](https://github.com/stackrox).|
-| global-hub.open-cluster-management.io/resign-kafka-client-secret | This annotation is used to identify if the kafka client secret is resynced in agent.|
+| global-hub.open-cluster-management.io/kafka-cluster-id | This annotation save the kafka cluster id, it is used to identify if the kafka changed in agent.|
path: doc/reserved_names/README.md
user: login: ldpliu
body: Done
created_at: 2025-03-05T02:39:22Z
author_association: CONTRIBUTOR
diff_hunk: @@ -244,44 +245,65 @@ func (c *TransportCtrl) ReconcileKafkaCredential(ctx context.Context, secret *co
 	return true, nil
 }
 
-// Resync the kafka client secret because we recreate the kafka cluster.
-// TODO: Remove the code in globalhub 1.5
+// Resync the kafka client secret because we recreate the kafka cluster in globalhub 1.4 and restore case.
 func (c *TransportCtrl) ResyncKafkaClientSecret(ctx context.Context, kafkaConn *transport.KafkaConfig, secret *corev1.Secret) error {
-	log.Debugf("resync kafka client secret: %v", kafkaConn.IsNewKafkaCluster)
-	if !kafkaConn.IsNewKafkaCluster {
+	if kafkaConn.ClusterID == "" {
 		return nil
 	}
 
+	isKafkaClusterIdEqual := true
+	// if stored kafka cluster id is same as current cluster id, return
 	if secret.Annotations != nil {
-		if _, ok := secret.Annotations[constants.ResyncKafkaClientSecretAnnotation]; ok {
+		if secret.Annotations[constants.KafkaClusterIdAnnotation] == sanitizeClusterID(kafkaConn.ClusterID) {
+			log.Debugf("cluster id is equal")
 			return nil
 		}
+		isKafkaClusterIdEqual = false
 	}
-	if kafkaConn.ClientSecretName == "" {
-		return nil
-	}
-	signedSecret := &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      kafkaConn.ClientSecretName,
-			Namespace: secret.Namespace,
-		},
-	}
-	log.Infof("remove kafka client secret: %v", kafkaConn.ClientSecretName)
 
-	err := c.runtimeClient.Delete(ctx, signedSecret)
-	if err != nil {
-		return err
+	log.Debugf("isNewKafkaCluster: %v", kafkaConn.IsNewKafkaCluster)
+	log.Debugf("isKafkaClusterIdEqual : %v", isKafkaClusterIdEqual)
+
+	// if it's new cluster(upgrade from 1.3 to 1.4) or cluster is not equal (restore globalhub)
+	if kafkaConn.IsNewKafkaCluster || !isKafkaClusterIdEqual {
+		if kafkaConn.ClientSecretName == "" {
+			return nil
+		}
+		signedSecret := &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      kafkaConn.ClientSecretName,
+				Namespace: secret.Namespace,
+			},
+		}
+		log.Infof("remove kafka client secret: %v", kafkaConn.ClientSecretName)
+
+		err := c.runtimeClient.Delete(ctx, signedSecret)
+		if err != nil {
+			return err
+		}
 	}
+
 	transportSecret := secret.DeepCopy()
 	if transportSecret.Annotations == nil {
 		transportSecret.Annotations = make(map[string]string)
 	}
 	log.Infof("update transport config secret")
 
-	transportSecret.Annotations[constants.ResyncKafkaClientSecretAnnotation] = "true"
+	transportSecret.Annotations[constants.KafkaClusterIdAnnotation] = sanitizeClusterID(kafkaConn.ClusterID)
 	return c.runtimeClient.Update(ctx, transportSecret)
 }
 
+// sanitizeClusterID replace the illegal character for label
+func sanitizeClusterID(clusterID string) string {
path: pkg/transport/controller/controller.go
user: login: ldpliu
body: Removed the logic
created_at: 2025-03-05T02:39:32Z
author_association: CONTRIBUTOR

PR comments:
user: login: ldpliu
created_at: 2025-03-04T12:32:18Z
body: /retest
author_association: CONTRIBUTOR
user: login: ldpliu
created_at: 2025-03-05T01:12:11Z
body: /retest
author_association: CONTRIBUTOR
user: login: ldpliu
created_at: 2025-03-05T02:06:18Z
body: /retest
author_association: CONTRIBUTOR
user: login: yanmxa
created_at: 2025-03-05T03:08:52Z
body: The transport controller is intuitively designed to detect changes in the transport credentials(include the signed certificate), and then trigger a reconnection to the Kafka cluster when necessary.


For the current case/issue:

The first priority in resolving this issue is to ensure that the addon renews its signed certificate when it expires or becomes invalid, allowing the Kafka cluster to reconnect automatically. I have opened an issue in the addon-framework to track this: [#304](https://github.com/open-cluster-management-io/addon-framework/issues/304)

However, since the addon registration may not currently support certificate renewal, this solution looks good to me for the current GH case.

/cc @ldpliu @clyang82 


author_association: CONTRIBUTOR
user: login: openshift-ci[bot]
created_at: 2025-03-05T03:08:56Z
body: @yanmxa: GitHub didn't allow me to request PR reviews from the following users: ldpliu.

Note that only [stolostron members](https://github.com/orgs/stolostron/people) and repo collaborators can review this PR, and authors cannot review their own PRs.

<details>

In response to [this](https://github.com/stolostron/multicluster-global-hub/pull/1443#issuecomment-2699668093):

>The transport controller is intuitively designed to detect changes in the transport credentials(include the signed certificate), and then trigger a reconnection to the Kafka cluster when necessary.
>
>
>For the current case/issue:
>
>The first priority in resolving this issue is to ensure that the addon renews its signed certificate when it expires or becomes invalid, allowing the Kafka cluster to reconnect automatically. I have opened an issue in the addon-framework to track this: [#304](https://github.com/open-cluster-management-io/addon-framework/issues/304)
>
>However, since the addon registration may not currently support certificate renewal, this solution looks good to me for the current GH case.
>
>/cc @ldpliu @clyang82 
>
>


Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>
author_association: NONE
user: login: ldpliu
created_at: 2025-03-05T03:30:05Z
body: I tested the whole flow(backup and restore), and it works fine.
author_association: CONTRIBUTOR
user: login: openshift-ci[bot]
created_at: 2025-03-05T06:05:45Z
body: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/stolostron/multicluster-global-hub/pull/1443#pullrequestreview-2660049849" title="Approved">clyang82</a>*, *<a href="https://github.com/stolostron/multicluster-global-hub/pull/1443#" title="Author self-approved">ldpliu</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=stolostron%2Fmulticluster-global-hub).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/stolostron/multicluster-global-hub/blob/main/OWNERS)~~ [clyang82]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
author_association: NONE
user: login: ldpliu
created_at: 2025-03-05T06:12:28Z
body: /cherry-pick release-2.13
author_association: CONTRIBUTOR
user: login: openshift-cherrypick-robot
created_at: 2025-03-05T06:12:31Z
body: @ldpliu: once the present PR merges, I will cherry-pick it on top of `release-2.13` in a new PR and assign it to you.

<details>

In response to [this](https://github.com/stolostron/multicluster-global-hub/pull/1443#issuecomment-2699974147):

>/cherry-pick release-2.13


Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>
author_association: CONTRIBUTOR
user: login: sonarqubecloud[bot]
created_at: 2025-03-05T06:26:32Z
body: ## [![Quality Gate Passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-passed-20px.png 'Quality Gate Passed')](https://sonarcloud.io/dashboard?id=open-cluster-management_hub-of-hubs&pullRequest=1443) **Quality Gate passed**  
Issues  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 New issues](https://sonarcloud.io/project/issues?id=open-cluster-management_hub-of-hubs&pullRequest=1443&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/accepted-16px.png '') [0 Accepted issues](https://sonarcloud.io/project/issues?id=open-cluster-management_hub-of-hubs&pullRequest=1443&issueStatuses=ACCEPTED)

Measures  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=open-cluster-management_hub-of-hubs&pullRequest=1443&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [90.9% Coverage on New Code](https://sonarcloud.io/component_measures?id=open-cluster-management_hub-of-hubs&pullRequest=1443&metric=new_coverage&view=list)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Duplication on New Code](https://sonarcloud.io/component_measures?id=open-cluster-management_hub-of-hubs&pullRequest=1443&metric=new_duplicated_lines_density&view=list)  
  
[See analysis details on SonarQube Cloud](https://sonarcloud.io/dashboard?id=open-cluster-management_hub-of-hubs&pullRequest=1443)


author_association: NONE
user: login: openshift-cherrypick-robot
created_at: 2025-03-05T07:22:36Z
body: @ldpliu: new pull request created: #1447

<details>

In response to [this](https://github.com/stolostron/multicluster-global-hub/pull/1443#issuecomment-2699974147):

>/cherry-pick release-2.13


Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>
author_association: CONTRIBUTOR