Repository: vmware-tanzu/tanzu-framework (language: go)

Issue:
Support cloning of embedded secrets in Config resources
Cloning of providers in ClusterBootstrapTemplate currently does not support cloning of references to secrets. Clone should inspect for secrets by potentially using a naming convention to clone any referenced secrets.


Discussion / hints:

Fix patch:
diff --git a/addons/controllers/clusterbootstrap_controller.go b/addons/controllers/clusterbootstrap_controller.go
index 338b1ced6a..c6e482de67 100644
--- a/addons/controllers/clusterbootstrap_controller.go
+++ b/addons/controllers/clusterbootstrap_controller.go
@@ -17,6 +17,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/discovery"
 	cacheddiscovery "k8s.io/client-go/discovery/cached/memory"
 	"k8s.io/client-go/dynamic"
@@ -43,7 +44,7 @@ import (
 	versions "github.com/vmware-tanzu/carvel-vendir/pkg/vendir/versions/v1alpha1"
 	addonconfig "github.com/vmware-tanzu/tanzu-framework/addons/pkg/config"
 	"github.com/vmware-tanzu/tanzu-framework/addons/pkg/constants"
-	"github.com/vmware-tanzu/tanzu-framework/addons/pkg/types"
+	addontypes "github.com/vmware-tanzu/tanzu-framework/addons/pkg/types"
 	"github.com/vmware-tanzu/tanzu-framework/addons/pkg/util"
 	"github.com/vmware-tanzu/tanzu-framework/addons/predicates"
 	runtanzuv1alpha3 "github.com/vmware-tanzu/tanzu-framework/apis/run/v1alpha3"
@@ -530,7 +531,7 @@ func (r *ClusterBootstrapReconciler) createOrPatchKappPackageInstall(clusterBoot
 					UID:        cluster.UID,
 				},
 			},
-			Annotations: map[string]string{types.ClusterNameAnnotation: cluster.Name, types.ClusterNamespaceAnnotation: cluster.Namespace},
+			Annotations: map[string]string{addontypes.ClusterNameAnnotation: cluster.Name, addontypes.ClusterNamespaceAnnotation: cluster.Namespace},
 		},
 	}
 
@@ -640,7 +641,7 @@ func (r *ClusterBootstrapReconciler) createOrPatchPackageInstallOnRemote(cluster
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        util.GeneratePackageInstallName(cluster.Name, remotePackageRefName),
 			Namespace:   r.Config.SystemNamespace,
-			Annotations: map[string]string{types.ClusterNameAnnotation: cluster.Name, types.ClusterNamespaceAnnotation: cluster.Namespace},
+			Annotations: map[string]string{addontypes.ClusterNameAnnotation: cluster.Name, addontypes.ClusterNamespaceAnnotation: cluster.Namespace},
 		},
 	}
 
@@ -675,6 +676,8 @@ func (r *ClusterBootstrapReconciler) createOrPatchPackageInstallOnRemote(cluster
 	return remotePkgi, nil
 }
 
+// reconcileSystemNamespace creates system namespace on remote workload cluster. This is because the system namespace
+// might not have been created yet when this controller reconciles remote cluster.
 func (r *ClusterBootstrapReconciler) reconcileSystemNamespace(clusterClient client.Client) error {
 	namespace := &corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
@@ -684,7 +687,7 @@ func (r *ClusterBootstrapReconciler) reconcileSystemNamespace(clusterClient clie
 
 	result, err := controllerutil.CreateOrPatch(r.context, clusterClient, namespace, nil)
 	if err != nil {
-		r.Log.Error(err, "Error creating or patching system namespace")
+		r.Log.Error(err, "unable to create or patch system namespace")
 		return err
 	}
 	if result != controllerutil.OperationResultNone {
@@ -916,13 +919,13 @@ func patchSecretWithLabels(secret *corev1.Secret, pkgName, clusterName string) b
 	if secret.Labels == nil {
 		secret.Labels = map[string]string{}
 		updateLabels = true
-	} else if secret.Labels[types.PackageNameLabel] != pkgName ||
-		secret.Labels[types.ClusterNameLabel] != clusterName {
+	} else if secret.Labels[addontypes.PackageNameLabel] != pkgName ||
+		secret.Labels[addontypes.ClusterNameLabel] != clusterName {
 		updateLabels = true
 	}
 	if updateLabels {
-		secret.Labels[types.PackageNameLabel] = pkgName
-		secret.Labels[types.ClusterNameLabel] = clusterName
+		secret.Labels[addontypes.PackageNameLabel] = pkgName
+		secret.Labels[addontypes.ClusterNameLabel] = clusterName
 	}
 	return updateLabels
 }
@@ -1116,8 +1119,8 @@ func (r *ClusterBootstrapReconciler) updateValuesFromSecret(cluster *clusterapiv
 			if newSecret.Labels == nil {
 				newSecret.Labels = map[string]string{}
 			}
-			newSecret.Labels[types.PackageNameLabel] = util.ParseStringForLabel(pkg.RefName)
-			newSecret.Labels[types.ClusterNameLabel] = cluster.Name
+			newSecret.Labels[addontypes.PackageNameLabel] = util.ParseStringForLabel(pkg.RefName)
+			newSecret.Labels[addontypes.ClusterNameLabel] = cluster.Name
 			// Set secret.Type to ClusterBootstrapManagedSecret to enable us to Watch these secrets
 			newSecret.Type = constants.ClusterBootstrapManagedSecret
 			return nil
@@ -1131,6 +1134,80 @@ func (r *ClusterBootstrapReconciler) updateValuesFromSecret(cluster *clusterapiv
 	return newSecret, nil
 }
 
+// cloneEmbeddedLocalObjectRef attempts to clone the embedded local object references from provider's namespace to cluster's
+// namespace. An example of embedded local object reference is the secret reference under CPIConfig.
+func (r *ClusterBootstrapReconciler) cloneEmbeddedLocalObjectRef(cluster *clusterapiv1beta1.Cluster, provider *unstructured.Unstructured) error {
+	groupKindNamesMap := util.ExtractTypedLocalObjectRef(provider.UnstructuredContent(), constants.LocalObjectRefSuffix)
+	if len(groupKindNamesMap) == 0 {
+		return nil
+	}
+
+	providerGVK := provider.GroupVersionKind()
+	r.Log.Info(fmt.Sprintf("cloning the embedded local object references within provider: %s with name: %s from"+
+		" %s namespace to %s namespace", provider.GroupVersionKind().String(), provider.GetName(), provider.GetNamespace(), cluster.Namespace))
+	for groupKind, resourceNames := range groupKindNamesMap {
+		gvr, err := r.getGVR(groupKind)
+		if err != nil {
+			// error has been logged within getGVR()
+			return err
+		}
+		for _, resourceName := range resourceNames {
+			r.Log.Info(fmt.Sprintf("cloning the GVR %s with name %s from %s namespace to %s namespace",
+				gvr.String(), resourceName, provider.GetNamespace(), cluster.Namespace))
+			fetchedObj, err := r.dynamicClient.Resource(*gvr).Namespace(provider.GetNamespace()).Get(r.context, resourceName, metav1.GetOptions{})
+			if err != nil {
+				r.Log.Error(err, fmt.Sprintf("unable to get provider: %s with name: %s under namespace: %s",
+					providerGVK.String(), provider.GetName(), provider.GetNamespace()))
+				return err
+			}
+
+			copiedObj := fetchedObj.DeepCopy()
+			// Remove resoruceVersion to make sure dynamicClient create could work
+			unstructured.RemoveNestedField(copiedObj.Object, "metadata", "resourceVersion")
+			copiedObj.SetNamespace(cluster.Namespace)
+			ownerReferences := copiedObj.GetOwnerReferences()
+			ownerReferences = clusterapiutil.EnsureOwnerRef(ownerReferences, metav1.OwnerReference{
+				APIVersion: provider.GetAPIVersion(),
+				Kind:       provider.GetKind(),
+				Name:       provider.GetName(),
+				UID:        provider.GetUID(),
+			})
+			ownerReferences = clusterapiutil.EnsureOwnerRef(ownerReferences, metav1.OwnerReference{
+				APIVersion: clusterapiv1beta1.GroupVersion.String(),
+				Kind:       "Cluster",
+				Name:       cluster.GetName(),
+				UID:        cluster.GetUID(),
+			})
+			copiedObj.SetOwnerReferences(ownerReferences)
+			_, err = r.dynamicClient.Resource(*gvr).Namespace(cluster.Namespace).Create(r.context, copiedObj, metav1.CreateOptions{})
+			if err != nil {
+				if apierrors.IsAlreadyExists(err) {
+					// Only patch the ownerReferences
+					patchObj := unstructured.Unstructured{}
+					patchObj.SetOwnerReferences(ownerReferences)
+					var jsonData []byte
+					if jsonData, err = patchObj.MarshalJSON(); err != nil {
+						return err
+					}
+					_, err = r.dynamicClient.Resource(*gvr).Namespace(cluster.Namespace).Patch(r.context, copiedObj.GetName(), types.MergePatchType, jsonData, metav1.PatchOptions{})
+					if err != nil {
+						r.Log.Error(err, fmt.Sprintf("unable to clone the GVR %s with name %s from namespace %s to"+
+							" target namespace %s", gvr.String(), resourceName, provider.GetNamespace(), cluster.Namespace))
+						return err
+					}
+				} else {
+					r.Log.Error(err, fmt.Sprintf("unable to clone the GVR %s with name %s from namespace %s to"+
+						" target namespace %s", gvr.String(), resourceName, provider.GetNamespace(), cluster.Namespace))
+					return err
+				}
+			}
+		}
+	}
+	r.Log.Info(fmt.Sprintf("cloned the embedded local object references within provider: %s with name: %s under"+
+		" namespace: %s to target namespace %s", providerGVK.String(), provider.GetName(), provider.GetNamespace(), cluster.Namespace))
+	return nil
+}
+
 // updateValuesFromProvider updates providerRef in valuesFrom
 func (r *ClusterBootstrapReconciler) updateValuesFromProvider(cluster *clusterapiv1beta1.Cluster,
 	pkg *runtanzuv1alpha3.ClusterBootstrapPackage, cbTemplateNamespace, pkgRefName string, log logr.Logger) (*unstructured.Unstructured, error) {
@@ -1167,12 +1244,13 @@ func (r *ClusterBootstrapReconciler) updateValuesFromProvider(cluster *clusterap
 				// For example, kapp-controller.tanzu.vmware.com.0.30.0+vmware.1-tkg.1 is a
 				// valid package refName, but it contains "+" that K8S complains. We parse the refName by replacing
 				// + to ---.
-				types.PackageNameLabel: util.ParseStringForLabel(pkg.RefName),
-				types.ClusterNameLabel: cluster.Name,
+				addontypes.PackageNameLabel: util.ParseStringForLabel(pkg.RefName),
+				addontypes.ClusterNameLabel: cluster.Name,
 			})
 		} else {
-			providerLabels[types.PackageNameLabel] = util.ParseStringForLabel(pkg.RefName)
-			providerLabels[types.ClusterNameLabel] = cluster.Name
+			providerLabels[addontypes.PackageNameLabel] = util.ParseStringForLabel(pkg.RefName)
+			providerLabels[addontypes.ClusterNameLabel] = cluster.Name
+			newProvider.SetLabels(providerLabels)
 		}
 
 		newProvider.SetName(fmt.Sprintf("%s-%s-package", cluster.Name, pkgRefName))
@@ -1188,10 +1266,17 @@ func (r *ClusterBootstrapReconciler) updateValuesFromProvider(cluster *clusterap
 			// There are possibilities that current reconciliation loop fails due to various reasons, and during next reconciliation
 			// loop, it is possible that the provider resource has been created. In this case, we want to run update/patch.
 			if apierrors.IsAlreadyExists(err) {
-				// Setting the resource version is because it's been removed at L984 with 'unstructured.RemoveNestedField(newProvider.Object, "metadata")'
-				// apiserver requires that field to be present for concurrency control.
-				newProvider.SetResourceVersion(provider.GetResourceVersion())
-				createdOrUpdatedProvider, err = r.dynamicClient.Resource(*gvr).Namespace(cluster.Namespace).Update(r.context, newProvider, metav1.UpdateOptions{})
+				r.Log.Info(fmt.Sprintf("provider %s/%s already exist, patching its Labels and OwnerReferences fields", newProvider.GetNamespace(), newProvider.GetName()))
+				// Instantiate an empty unstructured and only set ownerReferences and Labels for patching
+				patchObj := unstructured.Unstructured{}
+				patchObj.SetLabels(newProvider.GetLabels())
+				patchObj.SetOwnerReferences(newProvider.GetOwnerReferences())
+				patchData, err := patchObj.MarshalJSON()
+				if err != nil {
+					log.Error(err, fmt.Sprintf("unable to patch provider %s/%s", newProvider.GetNamespace(), newProvider.GetName()), "gvr", gvr)
+					return nil, err
+				}
+				createdOrUpdatedProvider, err = r.dynamicClient.Resource(*gvr).Namespace(cluster.Namespace).Patch(r.context, newProvider.GetName(), types.MergePatchType, patchData, metav1.PatchOptions{})
 				if err != nil {
 					log.Info(fmt.Sprintf("unable to update provider %s/%s", newProvider.GetNamespace(), newProvider.GetName()), "gvr", gvr)
 					return nil, err
@@ -1204,6 +1289,10 @@ func (r *ClusterBootstrapReconciler) updateValuesFromProvider(cluster *clusterap
 
 		valuesFrom.ProviderRef.Name = createdOrUpdatedProvider.GetName()
 		log.Info(fmt.Sprintf("cloned provider %s/%s to namespace %s", createdOrUpdatedProvider.GetNamespace(), createdOrUpdatedProvider.GetName(), cluster.Namespace), "gvr", gvr)
+
+		if err := r.cloneEmbeddedLocalObjectRef(cluster, provider); err != nil {
+			return nil, err
+		}
 	}
 
 	return createdOrUpdatedProvider, nil
@@ -1355,13 +1444,13 @@ func (r *ClusterBootstrapReconciler) reconcileClusterProxyAndNetworkSettings(clu
 	if cluster.Annotations == nil {
 		cluster.Annotations = map[string]string{}
 	}
-	cluster.Annotations[types.HTTPProxyConfigAnnotation] = HTTPProxy
-	cluster.Annotations[types.HTTPSProxyConfigAnnotation] = HTTPSProxy
-	cluster.Annotations[types.NoProxyConfigAnnotation] = NoProxy
-	cluster.Annotations[types.ProxyCACertConfigAnnotation] = ProxyCACert
-	cluster.Annotations[types.IPFamilyConfigAnnotation] = IPFamily
+	cluster.Annotations[addontypes.HTTPProxyConfigAnnotation] = HTTPProxy
+	cluster.Annotations[addontypes.HTTPSProxyConfigAnnotation] = HTTPSProxy
+	cluster.Annotations[addontypes.NoProxyConfigAnnotation] = NoProxy
+	cluster.Annotations[addontypes.ProxyCACertConfigAnnotation] = ProxyCACert
+	cluster.Annotations[addontypes.IPFamilyConfigAnnotation] = IPFamily
 
-	log.Info("setting proxy and network configurations in Cluster annotation", types.HTTPProxyConfigAnnotation, HTTPProxy, types.HTTPSProxyConfigAnnotation, HTTPSProxy, types.NoProxyConfigAnnotation, NoProxy, types.ProxyCACertConfigAnnotation, ProxyCACert, types.IPFamilyConfigAnnotation, IPFamily)
+	log.Info("setting proxy and network configurations in Cluster annotation", addontypes.HTTPProxyConfigAnnotation, HTTPProxy, addontypes.HTTPSProxyConfigAnnotation, HTTPSProxy, addontypes.NoProxyConfigAnnotation, NoProxy, addontypes.ProxyCACertConfigAnnotation, ProxyCACert, addontypes.IPFamilyConfigAnnotation, IPFamily)
 
 	if err := patchHelper.Patch(r.context, cluster); err != nil {
 		log.Error(err, "unable to patch Cluster Annotation")
diff --git a/addons/controllers/cpi/vspherecpiconfig_utils.go b/addons/controllers/cpi/vspherecpiconfig_utils.go
index a4484d8d44..461bb34969 100644
--- a/addons/controllers/cpi/vspherecpiconfig_utils.go
+++ b/addons/controllers/cpi/vspherecpiconfig_utils.go
@@ -107,8 +107,8 @@ func (r *VSphereCPIConfigReconciler) mapCPIConfigToDataValuesNonParavirtual( //
 	if c.Datacenter != "" {
 		d.VSphereCPI.Datacenter = c.Datacenter
 	}
-	if c.VSphereCredentialRef != nil {
-		vsphereSecret, err := r.getSecret(ctx, c.VSphereCredentialRef.Namespace, c.VSphereCredentialRef.Name)
+	if c.VSphereCredentialLocalObjRef != nil {
+		vsphereSecret, err := r.getSecret(ctx, cpiConfig.Namespace, c.VSphereCredentialLocalObjRef.Name)
 		if err != nil {
 			return nil, err
 		}
@@ -138,8 +138,8 @@ func (r *VSphereCPIConfigReconciler) mapCPIConfigToDataValuesNonParavirtual( //
 			d.VSphereCPI.Nsxt.Routes.RouterPath = c.NSXT.Routes.RouterPath
 			d.VSphereCPI.Nsxt.Routes.ClusterCidr = c.NSXT.Routes.ClusterCidr
 		}
-		if c.NSXT.NSXTCredentialsRef != nil {
-			nsxtSecret, err := r.getSecret(ctx, c.NSXT.NSXTCredentialsRef.Namespace, c.NSXT.NSXTCredentialsRef.Name)
+		if c.NSXT.CredentialLocalObjRef != nil {
+			nsxtSecret, err := r.getSecret(ctx, cpiConfig.Namespace, c.NSXT.CredentialLocalObjRef.Name)
 			if err != nil {
 				return nil, err
 			}
diff --git a/addons/pkg/constants/constants.go b/addons/pkg/constants/constants.go
index 179da11a21..d2a95fa024 100644
--- a/addons/pkg/constants/constants.go
+++ b/addons/pkg/constants/constants.go
@@ -166,7 +166,7 @@ const (
 	// PackageInstallSyncPeriod is the sync period for kapp-controller to periodically reconcile a PackageInstall
 	PackageInstallSyncPeriod = time.Minute * 10
 
-	// RequeueAfterInterval determines the duration after which the Controller should requeue the reconcile key
+	// RequeueAfterDuration determines the duration after which the Controller should requeue the reconcile key
 	RequeueAfterDuration = time.Second * 10
 
 	// WebhookCertDir is the directory where the certificate and key are stored for webhook server TLS handshake
@@ -183,6 +183,10 @@ const (
 
 	// AddonWebhookLabelValue is the value for the label for addon admission webhooks
 	AddonWebhookLabelValue = ""
+
+	// LocalObjectRefSuffix is the suffix of a field within the provider's CR. This suffix indicates that the field is a
+	// K8S typed local object reference
+	LocalObjectRefSuffix = "LocalObjRef"
 )
 
 // ClusterKind is the Kind for cluster-api Cluster object
diff --git a/addons/pkg/util/provider_util.go b/addons/pkg/util/provider_util.go
new file mode 100644
index 0000000000..c32f5fedac
--- /dev/null
+++ b/addons/pkg/util/provider_util.go
@@ -0,0 +1,64 @@
+// Copyright 2022 VMware, Inc. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package util
+
+import (
+	"reflect"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+// ExtractTypedLocalObjectRef extracts the TypedLocalObjectReference from the unstructuredContent by looking at the fields
+// which has the specified fieldSuffix.
+// Returns a map with groupKind as the key and object names as the value
+func ExtractTypedLocalObjectRef(unstructuredContent map[string]interface{}, fieldSuffix string) map[schema.GroupKind][]string {
+	if unstructuredContent == nil || fieldSuffix == "" {
+		return nil
+	}
+	var extractedGKs = make(map[schema.GroupKind][]string)
+	for k, v := range unstructuredContent {
+		if v != nil && reflect.TypeOf(v).Kind() == reflect.Map {
+			if strings.HasSuffix(k, fieldSuffix) {
+				localObjRef := v.(map[string]interface{})
+
+				if !isValidLocalObjectRef(localObjRef) {
+					return nil
+				}
+
+				apiGroup := ""
+				if localObjRef["apiGroup"] != nil {
+					apiGroup = localObjRef["apiGroup"].(string)
+				}
+				kind := localObjRef["kind"].(string)
+				name := localObjRef["name"].(string)
+				groupKind := schema.GroupKind{Group: apiGroup, Kind: kind}
+				extractedGKs[groupKind] = append(extractedGKs[groupKind], name)
+			} else {
+				extractedGVRsFromNested := ExtractTypedLocalObjectRef(v.(map[string]interface{}), fieldSuffix)
+				// Combine the result from nested fields into the extractedGKs
+				for k, v := range extractedGVRsFromNested {
+					extractedGKs[k] = append(extractedGKs[k], v...)
+				}
+			}
+		}
+	}
+	return extractedGKs
+}
+
+func isValidLocalObjectRef(localObjRef map[string]interface{}) bool {
+	if localObjRef == nil {
+		return false
+	}
+	if _, exist := localObjRef["kind"]; !exist || reflect.TypeOf(localObjRef["kind"]).Kind() != reflect.String {
+		return false
+	}
+	if _, exist := localObjRef["name"]; !exist || reflect.TypeOf(localObjRef["name"]).Kind() != reflect.String {
+		return false
+	}
+	if localObjRef["apiGroup"] != nil && reflect.TypeOf(localObjRef["apiGroup"]).Kind() != reflect.String {
+		return false
+	}
+	return true
+}
diff --git a/apis/cpi/v1alpha1/vspherecpiconfig_types.go b/apis/cpi/v1alpha1/vspherecpiconfig_types.go
index 589a765304..24bf911248 100644
--- a/apis/cpi/v1alpha1/vspherecpiconfig_types.go
+++ b/apis/cpi/v1alpha1/vspherecpiconfig_types.go
@@ -33,10 +33,10 @@ type NSXT struct {
 	// +kubebuilder:validation:Optional
 	Routes *NSXTRoute `json:"routes"`
 
-	// A secret reference that contains Nsx-T login credentials to access NSX-T
+	// A secret reference that contains Nsx-T login credential to access NSX-T
 	// consists of the field username and password
 	// +kubebuilder:validation:Optional
-	NSXTCredentialsRef *v1.SecretReference `json:"nsxtCredentialsRef"`
+	CredentialLocalObjRef *v1.TypedLocalObjectReference `json:"credentialLocalObjRef"`
 
 	// The NSX-T server
 	// +kubebuilder:validation:Optional
@@ -92,7 +92,7 @@ type NonParavirtualConfig struct {
 	// A secret reference that contains vSphere login credentials to access a vSphere endpoint
 	// consists of the fields username and password
 	// +kubebuilder:validation:Optional
-	VSphereCredentialRef *v1.SecretReference `json:"vSphereCredentialRef"`
+	VSphereCredentialLocalObjRef *v1.TypedLocalObjectReference `json:"vSphereCredentialLocalObjRef"`
 
 	// The cryptographic thumbprint of the vSphere endpoint's certificate. Default value is "".
 	// +kubebuilder:validation:Optional
diff --git a/apis/cpi/v1alpha1/zz_generated.deepcopy.go b/apis/cpi/v1alpha1/zz_generated.deepcopy.go
index 767a7f2c81..1cd78cff2c 100644
--- a/apis/cpi/v1alpha1/zz_generated.deepcopy.go
+++ b/apis/cpi/v1alpha1/zz_generated.deepcopy.go
@@ -21,10 +21,10 @@ func (in *NSXT) DeepCopyInto(out *NSXT) {
 		*out = new(NSXTRoute)
 		**out = **in
 	}
-	if in.NSXTCredentialsRef != nil {
-		in, out := &in.NSXTCredentialsRef, &out.NSXTCredentialsRef
-		*out = new(v1.SecretReference)
-		**out = **in
+	if in.CredentialLocalObjRef != nil {
+		in, out := &in.CredentialLocalObjRef, &out.CredentialLocalObjRef
+		*out = new(v1.TypedLocalObjectReference)
+		(*in).DeepCopyInto(*out)
 	}
 }
 
@@ -56,10 +56,10 @@ func (in *NSXTRoute) DeepCopy() *NSXTRoute {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NonParavirtualConfig) DeepCopyInto(out *NonParavirtualConfig) {
 	*out = *in
-	if in.VSphereCredentialRef != nil {
-		in, out := &in.VSphereCredentialRef, &out.VSphereCredentialRef
-		*out = new(v1.SecretReference)
-		**out = **in
+	if in.VSphereCredentialLocalObjRef != nil {
+		in, out := &in.VSphereCredentialLocalObjRef, &out.VSphereCredentialLocalObjRef
+		*out = new(v1.TypedLocalObjectReference)
+		(*in).DeepCopyInto(*out)
 	}
 	if in.NSXT != nil {
 		in, out := &in.NSXT, &out.NSXT
diff --git a/config/crd/bases/cpi.tanzu.vmware.com_vspherecpiconfigs.yaml b/config/crd/bases/cpi.tanzu.vmware.com_vspherecpiconfigs.yaml
index 6296773d82..40b78bd8d7 100644
--- a/config/crd/bases/cpi.tanzu.vmware.com_vspherecpiconfigs.yaml
+++ b/config/crd/bases/cpi.tanzu.vmware.com_vspherecpiconfigs.yaml
@@ -86,6 +86,27 @@ spec:
                       clientCertKeyData:
                         description: Client certificate key for NSX-T
                         type: string
+                      credentialLocalObjRef:
+                        description: A secret reference that contains Nsx-T login
+                          credential to access NSX-T consists of the field username
+                          and password
+                        properties:
+                          apiGroup:
+                            description: APIGroup is the group for the resource being
+                              referenced. If APIGroup is not specified, the specified
+                              Kind must be in the core API group. For any other third-party
+                              types, APIGroup is required.
+                            type: string
+                          kind:
+                            description: Kind is the type of resource being referenced
+                            type: string
+                          name:
+                            description: Name is the name of resource being referenced
+                            type: string
+                        required:
+                        - kind
+                        - name
+                        type: object
                       host:
                         description: The NSX-T server
                         type: string
@@ -93,20 +114,6 @@ spec:
                         description: InsecureFlag is to be set to true if NSX-T uses
                           self-signed cert
                         type: boolean
-                      nsxtCredentialsRef:
-                        description: A secret reference that contains Nsx-T login
-                          credentials to access NSX-T consists of the field username
-                          and password
-                        properties:
-                          name:
-                            description: Name is unique within a namespace to reference
-                              a secret resource.
-                            type: string
-                          namespace:
-                            description: Namespace defines the space within which
-                              the secret name must be unique.
-                            type: string
-                        type: object
                       podRoutingEnabled:
                         default: false
                         description: A flag that enables pod routing
@@ -158,19 +165,26 @@ spec:
                     description: The cryptographic thumbprint of the vSphere endpoint's
                       certificate. Default value is "".
                     type: string
-                  vSphereCredentialRef:
+                  vSphereCredentialLocalObjRef:
                     description: A secret reference that contains vSphere login credentials
                       to access a vSphere endpoint consists of the fields username
                       and password
                     properties:
-                      name:
-                        description: Name is unique within a namespace to reference
-                          a secret resource.
+                      apiGroup:
+                        description: APIGroup is the group for the resource being
+                          referenced. If APIGroup is not specified, the specified
+                          Kind must be in the core API group. For any other third-party
+                          types, APIGroup is required.
                         type: string
-                      namespace:
-                        description: Namespace defines the space within which the
-                          secret name must be unique.
+                      kind:
+                        description: Kind is the type of resource being referenced
+                        type: string
+                      name:
+                        description: Name is the name of resource being referenced
                         type: string
+                    required:
+                    - kind
+                    - name
                     type: object
                   vmExcludeExternalNetworkSubnetCidr:
                     description: External VM network CIDR to be excluded
diff --git a/config/samples/cpi_v1alpha1_vspherecpiconfig.yaml b/config/samples/cpi_v1alpha1_vspherecpiconfig.yaml
index 2963e1b82a..c8aed5c6b6 100644
--- a/config/samples/cpi_v1alpha1_vspherecpiconfig.yaml
+++ b/config/samples/cpi_v1alpha1_vspherecpiconfig.yaml
@@ -8,9 +8,10 @@ spec:
     tlsThumbprint: "test"
     server: 10.1.1.1
     datacenter: ds-test
-    vSphereCredentialRef:
+    vSphereCredentialLocalObjRef:
+      apiGroup: ""
+      kind: Secret
       name: vsphere-credentials
-      namespace: tkg-system
     insecureFlag: False
     vmInternalNetwork: null
     vmExternalNetwork: null
@@ -20,9 +21,10 @@ spec:
       podRoutingEnabled: false
       routes:
         routerPath: ""
-      nsxtCredentialsRef:
+      credentialLocalObjRef:
+        apiGroup: ""
+        kind: Secret
         name: nsxt-credentials
-        namespace: tkg-system
       host: "10.1.1.2"
       insecureFlag: false
       remoteAuth: false


Test patch:
diff --git a/addons/controllers/clusterbootstrap_controller_test.go b/addons/controllers/clusterbootstrap_controller_test.go
index be7ab8f5b7..ecef32a5c9 100644
--- a/addons/controllers/clusterbootstrap_controller_test.go
+++ b/addons/controllers/clusterbootstrap_controller_test.go
@@ -4,6 +4,7 @@
 package controllers
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"strings"
@@ -18,6 +19,8 @@ import (
 	addontypes "github.com/vmware-tanzu/tanzu-framework/addons/pkg/types"
 	"github.com/vmware-tanzu/tanzu-framework/addons/pkg/util"
 	"github.com/vmware-tanzu/tanzu-framework/addons/testutil"
+	antreaconfigv1alpha1 "github.com/vmware-tanzu/tanzu-framework/apis/cni/v1alpha1"
+	vspherecpiv1alpha1 "github.com/vmware-tanzu/tanzu-framework/apis/cpi/v1alpha1"
 	runtanzuv1alpha3 "github.com/vmware-tanzu/tanzu-framework/apis/run/v1alpha3"
 
 	corev1 "k8s.io/api/core/v1"
@@ -410,6 +413,19 @@ var _ = Describe("ClusterBootstrap Reconciler", func() {
 					}, waitTimeout, pollingInterval).Should(BeTrue())
 				})
 
+				By("verifying the embedded local object reference is cloned into cluster namespace", func() {
+					name := "cpi-vsphere-credential"
+					assertEventuallyExistInNamespace(ctx, k8sClient, clusterNamespace, name, &corev1.Secret{})
+					assertSecretContains(ctx, k8sClient, clusterNamespace, name, map[string][]byte{
+						"username": []byte("Zm9v"), // foo
+						"password": []byte("YmFy"), // bar
+					})
+					assertOwnerReferencesExist(ctx, k8sClient, clusterNamespace, name, &corev1.Secret{}, []metav1.OwnerReference{
+						{APIVersion: clusterapiv1beta1.GroupVersion.String(), Kind: "Cluster", Name: clusterName},
+						{APIVersion: vspherecpiv1alpha1.GroupVersion.String(), Kind: "VSphereCPIConfig", Name: "test-cluster-cpi"},
+					})
+				})
+
 				By("Updating cluster TKR version", func() {
 					newTKRVersion := "v1.23.3"
 					cluster := &clusterapiv1beta1.Cluster{}
@@ -548,4 +564,103 @@ var _ = Describe("ClusterBootstrap Reconciler", func() {
 		})
 	})
 
+	When("Cluster is created", func() {
+
+		var routableAntreaConfig *antreaconfigv1alpha1.AntreaConfig
+
+		BeforeEach(func() {
+			clusterName = "test-cluster-tcbt-3"
+			clusterNamespace = "cluster-namespace-3"
+			clusterResourceFilePath = "testdata/test-cluster-bootstrap-3.yaml"
+
+			ns := &corev1.Namespace{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: clusterNamespace,
+				},
+			}
+			_ = k8sClient.Create(ctx, ns)
+
+			routableAntreaConfig = &antreaconfigv1alpha1.AntreaConfig{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      fmt.Sprintf("%s-%s-package", clusterName, "antrea.tanzu.vmware.com"),
+					Namespace: clusterNamespace,
+				},
+				Spec: antreaconfigv1alpha1.AntreaConfigSpec{
+					Antrea: antreaconfigv1alpha1.Antrea{
+						AntreaConfigDataValue: antreaconfigv1alpha1.AntreaConfigDataValue{
+							TrafficEncapMode: "noEncap",
+							NoSNAT:           true,
+							FeatureGates: antreaconfigv1alpha1.AntreaFeatureGates{
+								AntreaProxy:   true,
+								EndpointSlice: true,
+							},
+						},
+					},
+				},
+			}
+			Expect(k8sClient.Create(ctx, routableAntreaConfig)).NotTo(HaveOccurred())
+			assertEventuallyExistInNamespace(ctx, k8sClient, clusterNamespace, routableAntreaConfig.Name, &antreaconfigv1alpha1.AntreaConfig{})
+		})
+
+		Context("with a routable AntreaConfig resource already exist", func() {
+			It("clusterbootstra_controller should not overwrite the existing AntreaConfig Specs", func() {
+				createdAntreaConfig := &antreaconfigv1alpha1.AntreaConfig{}
+				assertOwnerReferencesExist(ctx, k8sClient, clusterNamespace, fmt.Sprintf("%s-%s-package", clusterName, "antrea.tanzu.vmware.com"), createdAntreaConfig, []metav1.OwnerReference{
+					{
+						APIVersion: clusterapiv1beta1.GroupVersion.String(),
+						Kind:       "Cluster",
+						Name:       clusterName,
+					},
+				})
+				Expect(createdAntreaConfig.Spec.Antrea.AntreaConfigDataValue.TrafficEncapMode).To(
+					Equal(routableAntreaConfig.Spec.Antrea.AntreaConfigDataValue.TrafficEncapMode))
+				Expect(createdAntreaConfig.Spec.Antrea.AntreaConfigDataValue.NoSNAT).To(
+					Equal(routableAntreaConfig.Spec.Antrea.AntreaConfigDataValue.NoSNAT))
+				Expect(createdAntreaConfig.Spec.Antrea.AntreaConfigDataValue.FeatureGates.AntreaProxy).To(
+					Equal(routableAntreaConfig.Spec.Antrea.AntreaConfigDataValue.FeatureGates.AntreaProxy))
+				Expect(createdAntreaConfig.Spec.Antrea.AntreaConfigDataValue.FeatureGates.EndpointSlice).To(
+					Equal(routableAntreaConfig.Spec.Antrea.AntreaConfigDataValue.FeatureGates.EndpointSlice))
+			})
+		})
+	})
+
 })
+
+func assertSecretContains(ctx context.Context, k8sClient client.Client, namespace, name string, secretContent map[string][]byte) {
+	s := &corev1.Secret{}
+	err := k8sClient.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, s)
+	Expect(err).ToNot(HaveOccurred())
+	for k, v := range secretContent {
+		Expect(string(s.Data[k]) == string(v)).ToNot(BeTrue())
+	}
+}
+
+func assertEventuallyExistInNamespace(ctx context.Context, k8sClient client.Client, namespace, name string, obj client.Object) {
+	Eventually(func() error {
+		key := client.ObjectKey{Name: name, Namespace: namespace}
+		return k8sClient.Get(ctx, key, obj)
+	}, waitTimeout, pollingInterval).Should(Succeed())
+}
+
+func assertOwnerReferencesExist(ctx context.Context, k8sClient client.Client, namespace, name string, obj client.Object, ownerReferencesToCheck []metav1.OwnerReference) {
+	Eventually(func() bool {
+		key := client.ObjectKey{Name: name, Namespace: namespace}
+		Expect(k8sClient.Get(ctx, key, obj)).NotTo(HaveOccurred())
+
+		for _, ownerReferenceToCheck := range ownerReferencesToCheck {
+			found := false
+			for _, ownerReferenceFromObj := range obj.GetOwnerReferences() {
+				// skip the comparison of UID on purpose, caller does not know the UIDs of ownerReferencesToCheck beforehand
+				if ownerReferenceToCheck.APIVersion == ownerReferenceFromObj.APIVersion &&
+					ownerReferenceToCheck.Kind == ownerReferenceFromObj.Kind &&
+					ownerReferenceToCheck.Name == ownerReferenceFromObj.Name {
+					found = true
+				}
+			}
+			if !found {
+				return false
+			}
+		}
+		return true
+	}, waitTimeout, pollingInterval).Should(BeTrue())
+}
diff --git a/addons/controllers/suite_test.go b/addons/controllers/suite_test.go
index 25edf1b48d..bae34b686f 100644
--- a/addons/controllers/suite_test.go
+++ b/addons/controllers/suite_test.go
@@ -56,7 +56,7 @@ import (
 // http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
 
 const (
-	waitTimeout             = time.Second * 60
+	waitTimeout             = time.Second * 90
 	pollingInterval         = time.Second * 2
 	appSyncPeriod           = 5 * time.Minute
 	appWaitTimeout          = 30 * time.Second
diff --git a/addons/controllers/testdata/test-cluster-bootstrap-1.yaml b/addons/controllers/testdata/test-cluster-bootstrap-1.yaml
index a47342a673..f0c12f0ba2 100644
--- a/addons/controllers/testdata/test-cluster-bootstrap-1.yaml
+++ b/addons/controllers/testdata/test-cluster-bootstrap-1.yaml
@@ -59,6 +59,31 @@ spec:
       - name: tkg.tanzu.vmware.com/tkg-ip-family
         value: "ipv4"
 ---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: VSphereCluster
+metadata:
+  name: test-cluster-tcbt
+  namespace: cluster-namespace
+spec:
+  thumbprint: test-thumbprint
+  server: vsphere-server.local
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: VSphereMachineTemplate
+metadata:
+  name: test-cluster-tcbt-control-plane
+  namespace: cluster-namespace
+spec:
+  template:
+    spec:
+      datacenter: dc0
+      template: /dc0/vm/photon-3-kube-v1.22.3+vmware.1-tkg.2
+      network:
+        devices:
+          - networkName: test-network
+            dhcp6: true
+            dhcp4: false
+---
 apiVersion: run.tanzu.vmware.com/v1alpha3
 kind: ClusterBootstrapTemplate
 metadata:
@@ -89,6 +114,13 @@ spec:
         apiGroup: cni.tanzu.vmware.com
         kind: AntreaConfig
         name: test-cluster-tcbt
+  cpi:
+    refName: vsphere-cpi.tanzu.vmware.com.1.22.3--vmware.1-tkg.1
+    valuesFrom:
+      providerRef:
+        apiGroup: cpi.tanzu.vmware.com
+        kind: VSphereCPIConfig
+        name: test-cluster-cpi
 ---
 apiVersion: run.tanzu.vmware.com/v1alpha3
 kind: ClusterBootstrapTemplate
@@ -120,6 +152,13 @@ spec:
         apiGroup: cni.tanzu.vmware.com
         kind: AntreaConfig
         name: test-cluster-tcbt
+  cpi:
+    refName: vsphere-cpi.tanzu.vmware.com.1.22.3--vmware.1-tkg.1
+    valuesFrom:
+      providerRef:
+        apiGroup: cpi.tanzu.vmware.com
+        kind: VSphereCPIConfig
+        name: test-cluster-cpi
 ---
 apiVersion: run.tanzu.vmware.com/v1alpha1
 kind: FooBar
@@ -184,6 +223,43 @@ spec:
           key: node.cloudprovider.kubernetes.io/uninitialized
           value: "true"
 ---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: test-cluster-tcbt
+  namespace: cluster-namespace
+data:
+  password: QWRtaW4hMjM= # Admin!23
+  username: YWRtaW5pc3RyYXRvckB2c3BoZXJlLmxvY2Fs # administrator@vsphere.local
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cpi-vsphere-credential
+  namespace: tkg-system
+data:
+  username: Zm9v # foo
+  password: YmFy # bar
+---
+apiVersion: cpi.tanzu.vmware.com/v1alpha1
+kind: VSphereCPIConfig
+metadata:
+  name: test-cluster-cpi
+  namespace: tkg-system
+spec:
+  vsphereCPI:
+    mode: vsphereCPI
+    region: test-region
+    zone: test-zone
+    insecureFlag: true
+    vmInternalNetwork: internal-net
+    vmExternalNetwork: external-net
+    vmExcludeInternalNetworkSubnetCidr: 192.168.3.0/24
+    vmExcludeExternalNetworkSubnetCidr: 22.22.3.0/24
+    vSphereCredentialLocalObjRef:
+      kind: Secret
+      name: cpi-vsphere-credential
+---
 apiVersion: data.packaging.carvel.dev/v1alpha1
 kind: Package
 metadata:
@@ -931,3 +1007,75 @@ spec:
   valuesSchema:
     openAPIv3:
       title: antrea.tanzu.vmware.com.1.2.3+vmware.4-tkg.2-advanced-zshippable values schema
+---
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
+metadata:
+  name: vsphere-cpi.tanzu.vmware.com.1.22.3--vmware.1-tkg.1
+  namespace: cluster-namespace
+spec:
+  refName: vsphere-cpi.tanzu.vmware.com
+  version: 1.22.3+vmware.1-tkg.1
+  releaseNotes: vsphere-cpi 1.22.3 https://github.com/kubernetes/cloud-provider-vsphere
+  licenses:
+    - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/vsphere-cpi:v1.22.3_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=30s
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+  releasedAt: "2021-12-20T10:59:32Z"
+  valuesSchema:
+    openAPIv3:
+      title: vsphere-cpi.tanzu.vmware.com.1.22.3--vmware.1-tkg.1 values schema
+---
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
+metadata:
+  name: vsphere-cpi.tanzu.vmware.com.1.22.3--vmware.1-tkg.1
+  namespace: tkg-system
+spec:
+  refName: vsphere-cpi.tanzu.vmware.com
+  version: 1.22.3+vmware.1-tkg.1
+  releaseNotes: vsphere-cpi 1.22.3 https://github.com/kubernetes/cloud-provider-vsphere
+  licenses:
+    - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/vsphere-cpi:v1.22.3_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=30s
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+  releasedAt: "2021-12-20T10:59:32Z"
+  valuesSchema:
+    openAPIv3:
+      title: vsphere-cpi.tanzu.vmware.com.1.22.3--vmware.1-tkg.1 values schema
diff --git a/addons/controllers/testdata/test-cluster-bootstrap-2.yaml b/addons/controllers/testdata/test-cluster-bootstrap-2.yaml
index 11a3e15b1e..50a0a36210 100644
--- a/addons/controllers/testdata/test-cluster-bootstrap-2.yaml
+++ b/addons/controllers/testdata/test-cluster-bootstrap-2.yaml
@@ -21,6 +21,31 @@ spec:
     services:
       cidrBlocks: [ "192.168.0.0/16","fd00:100:96::/48" ]
 ---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: VSphereCluster
+metadata:
+  name: test-cluster-tcbt-2
+  namespace: cluster-namespace-2
+spec:
+  thumbprint: test-thumbprint
+  server: vsphere-server.local
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: VSphereMachineTemplate
+metadata:
+  name: test-cluster-tcbt-2-control-plane
+  namespace: cluster-namespace-2
+spec:
+  template:
+    spec:
+      datacenter: dc0
+      template: /dc0/vm/photon-3-kube-v1.22.3+vmware.1-tkg.2
+      network:
+        devices:
+          - networkName: test-network
+            dhcp6: true
+            dhcp4: false
+---
 apiVersion: data.packaging.carvel.dev/v1alpha1
 kind: Package
 metadata:
@@ -212,3 +237,39 @@ spec:
   valuesSchema:
     openAPIv3:
       title: antrea.tanzu.vmware.com.1.2.3+vmware.1-tkg.1 values schema
+---
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
+metadata:
+  name: vsphere-cpi.tanzu.vmware.com.1.22.3--vmware.1-tkg.1
+  namespace: cluster-namespace-2
+spec:
+  refName: vsphere-cpi.tanzu.vmware.com
+  version: 1.22.3+vmware.1-tkg.1
+  releaseNotes: vsphere-cpi 1.22.3 https://github.com/kubernetes/cloud-provider-vsphere
+  licenses:
+    - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/vsphere-cpi:v1.22.3_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=30s
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+  releasedAt: "2021-12-20T10:59:32Z"
+  valuesSchema:
+    openAPIv3:
+      title: vsphere-cpi.tanzu.vmware.com.1.22.3--vmware.1-tkg.1 values schema
diff --git a/addons/controllers/testdata/test-cluster-bootstrap-3.yaml b/addons/controllers/testdata/test-cluster-bootstrap-3.yaml
new file mode 100644
index 0000000000..2622981784
--- /dev/null
+++ b/addons/controllers/testdata/test-cluster-bootstrap-3.yaml
@@ -0,0 +1,270 @@
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: test-cluster-tcbt-3
+  namespace: cluster-namespace-3
+  labels:
+    tkg.tanzu.vmware.com/cluster-name: test-cluster-tcbt-3
+    run.tanzu.vmware.com/tkr: v1.22.3
+spec:
+  infrastructureRef:
+    kind: VSphereCluster
+  clusterNetwork:
+    pods:
+      cidrBlocks: [ "192.168.0.0/16","fd00:100:96::/48" ]
+    services:
+      cidrBlocks: [ "192.168.0.0/16","fd00:100:96::/48" ]
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: VSphereCluster
+metadata:
+  name: test-cluster-tcbt-3
+  namespace: cluster-namespace-3
+spec:
+  thumbprint: test-thumbprint
+  server: vsphere-server.local
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: VSphereMachineTemplate
+metadata:
+  name: test-cluster-tcbt-3-control-plane
+  namespace: cluster-namespace-3
+spec:
+  template:
+    spec:
+      datacenter: dc0
+      template: /dc0/vm/photon-3-kube-v1.22.3+vmware.1-tkg.2
+      network:
+        devices:
+          - networkName: test-network
+            dhcp6: true
+            dhcp4: false
+---
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
+metadata:
+  name: foobar.example.com.1.17.2
+  namespace: cluster-namespace-3
+spec:
+  refName: foobar.example.com
+  version: 1.17.2
+  releasedAt: "2021-05-13T18:00:00Z"
+  releaseNotes: foobar 1.17.2
+  capacityRequirementsDescription: Varies significantly based on cluster size. This should be tuned based on observed usage.
+  valuesSchema:
+    openAPIv3:
+      title: foobar.example.com.1.17.2+vmware.1-tkg.1 values schema
+      properties:
+        namespace:
+          type: string
+          description: The namespace in which to deploy fluent-bit.
+          default: tanzu-system-logging
+  licenses:
+    - 'VMware''s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/fluent-bit:v1.7.5_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=5m
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+---
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
+metadata:
+  name: foobar1.example.com.1.17.2
+  namespace: cluster-namespace-3
+spec:
+  refName: foobar1.example.com
+  version: 1.17.2
+  releasedAt: "2021-05-13T18:00:00Z"
+  releaseNotes: foobar1 1.17.2
+  capacityRequirementsDescription: Varies significantly based on cluster size. This should be tuned based on observed usage.
+  valuesSchema:
+    openAPIv3:
+      title: foobar1.example.com.1.17.2+vmware.1-tkg.1 values schema
+      properties:
+        namespace:
+          type: string
+          description: The namespace in which to deploy fluent-bit.
+          default: tanzu-system-logging
+  licenses:
+    - 'VMware''s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/fluent-bit:v1.7.5_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=5m
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+---
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
+metadata:
+  name: kapp-controller.tanzu.vmware.com.0.30.0
+  namespace: cluster-namespace-3
+spec:
+  refName: kapp-controller.tanzu.vmware.com
+  version: 0.30.0
+  releaseNotes: kapp-controller 0.30.0 https://github.com/vmware-tanzu/carvel-kapp-controller
+  licenses:
+    - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/kapp-controller:v0.30.0_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=30s
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+  releasedAt: "2021-12-30T10:59:32Z"
+  valuesSchema:
+    openAPIv3:
+      title: kapp-controller.tanzu.vmware.com.0.30.0+vmware.1-tkg.1 values schema
+---
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
+metadata:
+  name: calico.tanzu.vmware.com.3.19.1--vmware.1-tkg.1
+  namespace: cluster-namespace-3
+spec:
+  refName: calico.tanzu.vmware.com
+  version: 3.19.1+vmware.1-tkg.1
+  releaseNotes: calico 3.19.1 https://docs.projectcalico.org/archive/v3.19/release-notes/
+  licenses:
+    - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/calico:v3.19.1_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=30s
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+  releasedAt: "2021-12-20T10:59:32Z"
+  valuesSchema:
+    openAPIv3:
+      title: calico.tanzu.vmware.com.3.19.1+vmware.1-tkg.1 values schema
+---
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
+metadata:
+  name: antrea.tanzu.vmware.com.1.2.3--vmware.1-tkg.1
+  namespace: cluster-namespace-3
+spec:
+  refName: antrea.tanzu.vmware.com
+  version: 1.2.3+vmware.1-tkg.1
+  releaseNotes: antrea 1.2.3 https://github.com/antrea-io/antrea/releases/tag/v1.2.3
+  licenses:
+    - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/antrea:v1.2.3_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=30s
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+  releasedAt: "2021-12-20T10:59:32Z"
+  valuesSchema:
+    openAPIv3:
+      title: antrea.tanzu.vmware.com.1.2.3+vmware.1-tkg.1 values schema
+---
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
+metadata:
+  name: vsphere-cpi.tanzu.vmware.com.1.22.3--vmware.1-tkg.1
+  namespace: cluster-namespace-3
+spec:
+  refName: vsphere-cpi.tanzu.vmware.com
+  version: 1.22.3+vmware.1-tkg.1
+  releaseNotes: vsphere-cpi 1.22.3 https://github.com/kubernetes/cloud-provider-vsphere
+  licenses:
+    - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/vsphere-cpi:v1.22.3_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=30s
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+  releasedAt: "2021-12-20T10:59:32Z"
+  valuesSchema:
+    openAPIv3:
+      title: vsphere-cpi.tanzu.vmware.com.1.22.3--vmware.1-tkg.1 values schema
diff --git a/addons/controllers/testdata/test-vsphere-cpi-non-paravirtual.yaml b/addons/controllers/testdata/test-vsphere-cpi-non-paravirtual.yaml
index 884b1783d5..5ebca2b5c9 100644
--- a/addons/controllers/testdata/test-vsphere-cpi-non-paravirtual.yaml
+++ b/addons/controllers/testdata/test-vsphere-cpi-non-paravirtual.yaml
@@ -85,6 +85,15 @@ spec:
 ---
 apiVersion: v1
 kind: Secret
+metadata:
+  name: cpi-vsphere-credential
+  namespace: default
+data:
+  username: Zm9v # foo
+  password: YmFy # bar
+---
+apiVersion: v1
+kind: Secret
 metadata:
   name: nsxt-credential
   namespace: default
@@ -107,3 +116,6 @@ spec:
     vmExternalNetwork: external-net
     vmExcludeInternalNetworkSubnetCidr: 192.168.3.0/24
     vmExcludeExternalNetworkSubnetCidr: 22.22.3.0/24
+    vSphereCredentialLocalObjRef:
+      kind: Secret
+      name: cpi-vsphere-credential
diff --git a/addons/controllers/vspherecpiconfig_controller_test.go b/addons/controllers/vspherecpiconfig_controller_test.go
index 545fe745e6..05e0b20af6 100644
--- a/addons/controllers/vspherecpiconfig_controller_test.go
+++ b/addons/controllers/vspherecpiconfig_controller_test.go
@@ -132,8 +132,8 @@ var _ = Describe("VSphereCPIConfig Reconciler", func() {
 				Expect(strings.Contains(secretData, "vmExcludeExternalNetworkSubnetCidr: 22.22.3.0/24")).Should(BeTrue())
 				Expect(strings.Contains(secretData, "tlsThumbprint: test-thumbprint")).Should(BeTrue())
 				Expect(strings.Contains(secretData, "server: vsphere-server.local")).Should(BeTrue())
-				Expect(strings.Contains(secretData, "username: administrator@vsphere.local")).Should(BeTrue())
-				Expect(strings.Contains(secretData, "password: Admin!23")).Should(BeTrue())
+				Expect(strings.Contains(secretData, "username: foo")).Should(BeTrue())
+				Expect(strings.Contains(secretData, "password: bar")).Should(BeTrue())
 
 				Expect(strings.Contains(secretData, "nsxt:")).Should(BeTrue())
 				Expect(strings.Contains(secretData, "podRoutingEnabled: true")).Should(BeTrue())
diff --git a/addons/pkg/util/provider_util_test.go b/addons/pkg/util/provider_util_test.go
new file mode 100644
index 0000000000..d3f0f4db47
--- /dev/null
+++ b/addons/pkg/util/provider_util_test.go
@@ -0,0 +1,108 @@
+// Copyright 2022 VMware, Inc. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package util_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vmware-tanzu/tanzu-framework/addons/pkg/util"
+
+	"gopkg.in/yaml.v3"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+var validInput1 = `
+apiVersion: cpi.tanzu.vmware.com/v1alpha1
+kind: VSphereCPIConfig
+metadata:
+  name: vspherecpiconfig-sample
+spec:
+  vsphereCPI:
+    mode: vsphereCPI
+    tlsThumbprint: "test"
+    server: 10.1.1.1
+    datacenter: ds-test
+    vSphereCredentialLocalObjRef:
+      kind: Secret
+      name: vsphere-credentials
+    insecureFlag: False
+    vmInternalNetwork: null
+    vmExternalNetwork: null
+    cloudProviderExtraArgs:
+      tlsCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+    nsxt:
+      podRoutingEnabled: false
+      routes:
+        routerPath: ""
+      credentialLocalObjRef:
+        apiGroup: ""
+        kind: Secret
+        name: nsxt-credentials
+      nestedForTesting:
+        dummyLocalObjRef:
+          apiGroup: ""
+          kind: "ConfigMap"
+          name: "dummy-configmap"
+      host: "10.1.1.2"
+      insecureFlag: false
+      remoteAuth: false
+      vmcAccessToken: ""
+      vmcAuthHost: ""
+      clientCertKeyData: ""
+      clientCertData: ""
+      rootCAData: ""
+      secretName: "cloud-provider-vsphere-nsxt-credentials"
+      secretNamespace: "kube-system"
+`
+
+var invalidInput1 = `
+vsphereCPI:
+  mode: vsphereCPI
+  tlsThumbprint: "test"
+  server: 10.1.1.1
+  datacenter: ds-test
+  vSphereCredentialLocalObjRef:
+    foo:  bar
+`
+
+var invalidInput2 = `
+vsphereCPI:
+  mode: vsphereCPI
+  tlsThumbprint: "test"
+  server: 10.1.1.1
+  datacenter: ds-test
+`
+
+var _ = Describe("Verifying the functionality of provider_util", func() {
+
+	var testData map[string]interface{}
+	JustBeforeEach(func() {
+		testData = make(map[string]interface{})
+	})
+
+	Context("when the unstructuredContent passed into ExtractTypedLocalObjectRef() has valid embedded local object reference", func() {
+
+		It("should return non-empty result", func() {
+			Expect(yaml.Unmarshal([]byte(validInput1), testData)).Should(Succeed())
+			result := util.ExtractTypedLocalObjectRef(testData, "LocalObjRef")
+			Expect(len(result)).To(Equal(2))
+			Expect(len(result[schema.GroupKind{Group: "", Kind: "Secret"}])).To(Equal(2))
+			Expect(len(result[schema.GroupKind{Group: "", Kind: "ConfigMap"}])).To(Equal(1))
+		})
+	})
+
+	Context("when the unstructuredContent passed into ExtractTypedLocalObjectRef() has invalid embedded local object reference", func() {
+		It("should return empty result if the content of local object reference is invalid", func() {
+			Expect(yaml.Unmarshal([]byte(invalidInput1), testData)).Should(Succeed())
+			result := util.ExtractTypedLocalObjectRef(testData, "LocalObjRef")
+			Expect(len(result)).To(Equal(0))
+		})
+		It("should return empty result if no embedded local object reference", func() {
+			Expect(yaml.Unmarshal([]byte(invalidInput2), testData)).Should(Succeed())
+			result := util.ExtractTypedLocalObjectRef(testData, "LocalObjRef")
+			Expect(len(result)).To(Equal(0))
+		})
+	})
+})