Repository: vmware-tanzu/tanzu-framework (language: go)

Issue:
Package reconcile for Kapp,CNI, CPI, CSI
(This is used to request new product features)

**Describe the feature request**
Package reconcile for Kapp, CNI, CPI and CSI packages in TanzuClusterBootstrap controller. See placeholder TODOs in vmware-tanzu/tanzu-framework#1541

**Describe alternatives you've considered**

**Affected product area (please put an X in all that apply)**

- [ ] APIs
- [x] Addons
- [ ] CLI
- [ ] Docs
- [ ] IAM
- [ ] Installation
- [ ] Plugin
- [ ] Security
- [ ] Test and Release
- [ ] User Experience

**Additional context**



Discussion / hints:

Fix patch:
diff --git a/addons/controllers/antrea/antreaconfig_controller.go b/addons/controllers/antrea/antreaconfig_controller.go
index e908995003..1ba5ecf7fa 100644
--- a/addons/controllers/antrea/antreaconfig_controller.go
+++ b/addons/controllers/antrea/antreaconfig_controller.go
@@ -67,7 +67,7 @@ func (r *AntreaConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	clusterNamespacedName := req.NamespacedName
 	cluster := &clusterapiv1beta1.Cluster{}
 	for _, owner := range antreaConfig.OwnerReferences {
-		if owner.Kind == cluster.Kind {
+		if owner.Kind == constants.ClusterKind {
 			clusterNamespacedName.Name = owner.Name
 			break
 		}
@@ -177,7 +177,7 @@ func (r *AntreaConfigReconciler) ReconcileAntreaConfigDataValue(
 	// prepare data values secret for AntreaConfig
 	antreaDataValuesSecret := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      util.GenerateDataValueSecretName(antreaConfig.Name, constants.AntreaAddonName),
+			Name:      util.GenerateDataValueSecretName(cluster.Name, constants.AntreaAddonName),
 			Namespace: antreaConfig.Namespace,
 		},
 	}
diff --git a/addons/controllers/clusterbootstrap_controller.go b/addons/controllers/clusterbootstrap_controller.go
index f1261702d0..123460ad89 100644
--- a/addons/controllers/clusterbootstrap_controller.go
+++ b/addons/controllers/clusterbootstrap_controller.go
@@ -12,15 +12,18 @@ import (
 
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	errorsutil "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/client-go/discovery"
 	cacheddiscovery "k8s.io/client-go/discovery/cached/memory"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/util/retry"
 	"k8s.io/utils/pointer"
 	clusterapiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	clusterapiutil "sigs.k8s.io/cluster-api/util"
@@ -36,6 +39,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
+	kappctrlv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1"
+	kapppkgiv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/packaging/v1alpha1"
+	kapppkgv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/apis/datapackaging/v1alpha1"
+	versions "github.com/vmware-tanzu/carvel-vendir/pkg/vendir/versions/v1alpha1"
 	addonconfig "github.com/vmware-tanzu/tanzu-framework/addons/pkg/config"
 	"github.com/vmware-tanzu/tanzu-framework/addons/pkg/constants"
 	"github.com/vmware-tanzu/tanzu-framework/addons/pkg/types"
@@ -49,8 +56,8 @@ type ClusterBootstrapReconciler struct {
 	client.Client
 	Log     logr.Logger
 	Scheme  *runtime.Scheme
+	Config  *addonconfig.ClusterBootstrapControllerConfig
 	context context.Context
-	Config  addonconfig.ClusterBootstrapControllerConfig
 
 	// internal properties
 	controller    controller.Controller
@@ -61,6 +68,7 @@ type ClusterBootstrapReconciler struct {
 	cachedDiscoveryClient discovery.CachedDiscoveryInterface
 	// cache for resolved api-resources so that look up is fast (cleared periodically)
 	providerGVR map[schema.GroupKind]*schema.GroupVersionResource
+	liveClient  client.Client
 }
 
 // NewClusterBootstrapReconciler returns a reconciler for ClusterBootstrap
@@ -69,7 +77,7 @@ func NewClusterBootstrapReconciler(c client.Client, log logr.Logger, scheme *run
 		Client: c,
 		Log:    log,
 		Scheme: scheme,
-		Config: *config,
+		Config: config,
 	}
 }
 
@@ -117,6 +125,11 @@ func (r *ClusterBootstrapReconciler) SetupWithManager(ctx context.Context, mgr c
 
 	go r.periodicGVRCachesClean()
 
+	r.liveClient, err = client.New(mgr.GetConfig(), client.Options{Scheme: mgr.GetScheme()})
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -162,45 +175,90 @@ func (r *ClusterBootstrapReconciler) Reconcile(ctx context.Context, req ctrl.Req
 }
 
 // reconcileNormal reconciles the ClusterBootstrap object
+// TODO: Fix the gocyclo linter issue: https://github.com/vmware-tanzu/tanzu-framework/issues/1761
+//nolint: gocyclo
 func (r *ClusterBootstrapReconciler) reconcileNormal(cluster *clusterapiv1beta1.Cluster, log logr.Logger) (ctrl.Result, error) {
 	// get or clone or patch from template
-	clusterBootstrap, err := r.createOrPatchclusterBootstrapFromTemplate(cluster, log)
+	clusterBootstrap, err := r.createOrPatchClusterBootstrapFromTemplate(cluster, log)
 	if err != nil {
-		return ctrl.Result{}, err
+		return ctrl.Result{Requeue: true}, err
 	}
 	if clusterBootstrap == nil {
 		return ctrl.Result{}, nil
 	}
 
 	// reconcile the proxy settings of the cluster
-	err = r.reconcileClusterProxyAndNetworkSettings(cluster, log)
-	if err != nil {
+	if err := r.reconcileClusterProxyAndNetworkSettings(cluster, log); err != nil {
 		return ctrl.Result{}, err
 	}
 
+	if cluster.Status.Phase != string(clusterapiv1beta1.ClusterPhaseProvisioned) {
+		r.Log.Info(fmt.Sprintf("cluster %s/%s does not have status phase %s", cluster.Namespace, cluster.Name, clusterapiv1beta1.ClusterPhaseProvisioned))
+		return ctrl.Result{}, nil
+	}
 	remoteClient, err := util.GetClusterClient(r.context, r.Client, r.Scheme, clusterapiutil.ObjectKey(cluster))
 	if err != nil {
 		log.Error(err, "Error getting remote cluster client")
+		return ctrl.Result{Requeue: true}, err
+	}
+
+	// Create a PackageInstall CR under the cluster namespace for deploying the kapp-controller on the remote cluster.
+	// We need kapp-controller to be deployed prior to CNI, CPI, CSI. This will be a no-op if the cluster object is mgmt
+	// cluster.
+	if err := r.createOrPatchKappPackageInstall(clusterBootstrap, cluster); err != nil {
+		// Return error if kapp-controller fails to be deployed, let reconciler try again
 		return ctrl.Result{}, err
 	}
-	// TODO handle kapp as a remote packageinstall from mgmt cluster
-	// handle cni, cpi, csi
-	// https://github.com/vmware-tanzu/tanzu-framework/issues/1587
 
-	// handle additionalPackages
-	for _, additionalPkg := range clusterBootstrap.Spec.AdditionalPackages {
-		// TODO packageinstall (rbac, serviceaccount, package, packageinstall)
-		// https://github.com/vmware-tanzu/tanzu-framework/issues/1589
-		secret, err := r.createOrPatchPackageInstallSecret(cluster, additionalPkg, remoteClient, clusterBootstrap.Namespace, log)
-		if err != nil {
-			log.Error(err, "failed to createOrPatchPackageInstallSecret")
-			return ctrl.Result{}, err
+	if err := r.reconcileSystemNamespace(remoteClient); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	// Create the ServiceAccount on remote cluster, so it could be referenced in PackageInstall CR for kapp-controller
+	// reconciliation.
+	if _, err := r.createOrPatchAddonServiceAccountOnRemote(cluster, remoteClient); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	// Create the ClusterRole on remote cluster, and bind it to the ServiceAccount created in above. kapp-controller
+	// reconciliation needs privileges.
+	if err := r.createOrPatchAddonRBACOnRemote(cluster, remoteClient); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	// Create or patch the resources for CNI, CPI, CSI to be running on the remote cluster.
+	// Those resources include Package CR, data value Secret, PackageInstall CR.
+	var corePackages []*runtanzuv1alpha3.ClusterBootstrapPackage
+	corePackages = append(corePackages, clusterBootstrap.Spec.CPI, clusterBootstrap.Spec.CSI)
+	corePackages = append(corePackages, clusterBootstrap.Spec.CNIs...)
+	for _, corePackage := range corePackages {
+		// The following nil check is redundant, we do not expect CNI, CPI, CSI to be nil and webhook should handle
+		// the validations against those fields. Having this nil check is mainly to allow local envtest run when
+		// any above component is missing.
+		if corePackage == nil {
+			continue
 		}
-		if secret != nil {
-			log.Info("created secret for package in cluster", "secret", secret)
+		// There are different ways to have all the resources created or patched on remote cluster. Current solution is
+		// to handle packages in sequence order. I.e., Create all resources for CNI first, and then CPI, CSI. It is also
+		// possible to create all resources in a different order or in parallel. We will consider to use goroutines to create
+		// all resources in parallel on remote cluster if there is performance issue from sequential ordering.
+		if err := r.createOrPatchAddonResourcesOnRemote(cluster, corePackage, remoteClient); err != nil {
+			// For core packages, we require all their creation or patching to succeed, so if error happens against any of the
+			// packages, we return error and let the reconciler retry again.
+			log.Error(err, fmt.Sprintf("unable to create or patch all the required resources for %s on cluster: %s/%s",
+				corePackage.RefName, cluster.Namespace, cluster.Name))
+			return ctrl.Result{RequeueAfter: time.Second * 10}, err
+		}
+	}
+
+	// Create or patch the resources for additionalPackages
+	for _, additionalPkg := range clusterBootstrap.Spec.AdditionalPackages {
+		if err := r.createOrPatchAddonResourcesOnRemote(cluster, additionalPkg, remoteClient); err != nil {
+			// Logging has been handled in createOrPatchAddonResourcesOnRemote()
+			return ctrl.Result{Requeue: true}, err
 		}
 		// set watches on provider objects in additional packages if not already set
-		if additionalPkg.ValuesFrom.ProviderRef != nil {
+		if additionalPkg.ValuesFrom != nil && additionalPkg.ValuesFrom.ProviderRef != nil {
 			if err := r.watchProvider(additionalPkg.ValuesFrom.ProviderRef, clusterBootstrap.Namespace, log); err != nil {
 				return ctrl.Result{}, err
 			}
@@ -210,15 +268,14 @@ func (r *ClusterBootstrapReconciler) reconcileNormal(cluster *clusterapiv1beta1.
 	return ctrl.Result{}, nil
 }
 
-// createOrPatchclusterBootstrapFromTemplate will get, clone or update a ClusterBootstrap associated with a cluster
-// all linked secret refs and object refs are cloned into the same namespace as clusterBootstrap
-func (r *ClusterBootstrapReconciler) createOrPatchclusterBootstrapFromTemplate(
-	cluster *clusterapiv1beta1.Cluster,
+// createOrPatchClusterBootstrapFromTemplate will get, clone or update a ClusterBootstrap associated with a cluster
+// all linked secret refs and object refs are cloned into the same namespace as ClusterBootstrap
+func (r *ClusterBootstrapReconciler) createOrPatchClusterBootstrapFromTemplate(cluster *clusterapiv1beta1.Cluster,
 	log logr.Logger) (*runtanzuv1alpha3.ClusterBootstrap, error) {
 
 	tkrName := cluster.Labels[constants.TKRLabelClassyClusters]
 	clusterBootstrapTemplate := &runtanzuv1alpha3.ClusterBootstrapTemplate{}
-	key := client.ObjectKey{Namespace: constants.TKGSystemNS, Name: tkrName}
+	key := client.ObjectKey{Namespace: r.Config.SystemNamespace, Name: tkrName}
 	if err := r.Client.Get(r.context, key, clusterBootstrapTemplate); err != nil {
 		log.Error(err, "unable to fetch ClusterBootstrapTemplate", "objectkey", key)
 		return nil, err
@@ -242,7 +299,7 @@ func (r *ClusterBootstrapReconciler) createOrPatchclusterBootstrapFromTemplate(
 		clusterBootstrap.Namespace = cluster.Namespace
 		clusterBootstrap.Spec = clusterBootstrapTemplate.Spec.DeepCopy()
 		// get selected CNI and populate clusterBootstrap.Spec.CNIs with it
-		cniPackage, err := r.getCNI(clusterBootstrapTemplate, cluster, log)
+		cniPackage, err := r.getCNIForClusterBootstrap(clusterBootstrapTemplate, cluster, log)
 		if err != nil {
 			return nil, err
 		}
@@ -250,7 +307,7 @@ func (r *ClusterBootstrapReconciler) createOrPatchclusterBootstrapFromTemplate(
 
 		secrets, providers, err := r.cloneSecretsAndProviders(cluster, clusterBootstrap, clusterBootstrapTemplate.Namespace, log)
 		if err != nil {
-			r.Log.Error(err, "unable to clone secrets, providers")
+			r.Log.Error(err, "unable to clone secrets or providers")
 			return nil, err
 		}
 
@@ -279,12 +336,13 @@ func (r *ClusterBootstrapReconciler) createOrPatchclusterBootstrapFromTemplate(
 			BlockOwnerDeletion: pointer.BoolPtr(true),
 		}
 		if err := r.ensureOwnerRef(&ownerRef, secrets, providers); err != nil {
-			r.Log.Error(err, "unable to ensure ownerref on created secrets and providers", "clusterBootstrap", clusterBootstrap)
+			r.Log.Error(err, fmt.Sprintf("unable to ensure ClusterBootstrap %s/%s as a ownerRef on created secrets and providers", clusterBootstrap.Namespace, clusterBootstrap.Name))
 			return nil, err
 		}
 
 		clusterBootstrap.Status.ResolvedTKR = tkrName
 		if err := r.Status().Update(r.context, clusterBootstrap); err != nil {
+			r.Log.Error(err, fmt.Sprintf("unable to update the status of ClusterBootstrap %s/%s", clusterBootstrap.Namespace, clusterBootstrap.Name))
 			return nil, err
 		}
 		r.Log.Info("cloned clusterBootstrap", "clusterBootstrap", clusterBootstrap)
@@ -300,79 +358,390 @@ func (r *ClusterBootstrapReconciler) createOrPatchclusterBootstrapFromTemplate(
 	return nil, errors.New("should not happen")
 }
 
-// createOrPatchPackageInstallSecret creates or patches or the secret used for PackageInstall in a cluster
-func (r *ClusterBootstrapReconciler) createOrPatchPackageInstallSecret(cluster *clusterapiv1beta1.Cluster,
-	pkg *runtanzuv1alpha3.ClusterBootstrapPackage, clusterClient client.Client, namespace string, log logr.Logger) (*corev1.Secret, error) {
+// createOrPatchKappPackageInstall contains the logic that create/update PackageInstall CR for kapp-controller on
+// mgmt cluster. The kapp-controller running on mgmt cluster reconciles the PackageInstall CR and creates kapp-controller resources
+// on remote workload cluster. This is required for a workload cluster and its corresponding package installations to be functional.
+func (r *ClusterBootstrapReconciler) createOrPatchKappPackageInstall(clusterBootstrap *runtanzuv1alpha3.ClusterBootstrap, cluster *clusterapiv1beta1.Cluster) error {
+	// Skip if the cluster object represents the management cluster
+	if _, exists := cluster.Labels[constants.ManagementClusterRoleLabel]; exists {
+		r.Log.Info(fmt.Sprintf("cluster %s/%s is management cluster, skip creating or patching the PackageInstall CR for kapp-controller", cluster.Namespace, cluster.Name))
+		return nil
+	}
 
-	secret := &corev1.Secret{}
+	// In order to create PackageInstall CR, we need to get the Package.Spec.RefName and Package.Spec.Version
+	packageRefName, packageVersion, err := util.GetPackageMetadata(r.context, r.liveClient, clusterBootstrap.Spec.Kapp.RefName, cluster.Namespace)
+	if packageRefName == "" || packageVersion == "" || err != nil {
+		// Package.Spec.RefName and Package.Spec.Version are required fields for Package CR. We do not expect them to be
+		// empty and error should not happen when fetching them from a Package CR.
+		r.Log.Error(err, fmt.Sprintf("unable to fetch Package.Spec.RefName or Package.Spec.Version from Package %s/%s",
+			cluster.Namespace, clusterBootstrap.Spec.Kapp.RefName))
+		return err
+	}
 
-	if pkg.ValuesFrom.SecretRef != "" {
-		key := client.ObjectKey{Namespace: namespace, Name: pkg.ValuesFrom.SecretRef}
-		if err := r.Get(r.context, key, secret); err != nil {
-			log.Error(err, "unable to fetch secret", "objectkey", key)
-			return nil, err
-		}
+	pkgi := &kapppkgiv1alpha1.PackageInstall{
+		ObjectMeta: metav1.ObjectMeta{
+			// The legacy addons controller uses <cluster name>-<addon name> convention for naming the PackageInstall CR.
+			// https://github.com/vmware-tanzu/tanzu-framework/blob/main/addons/controllers/package_reconciler.go#L195.
+			// util.GeneratePackageInstallName() follows the same pattern.
+			Name: util.GeneratePackageInstallName(cluster.Name, packageRefName),
+			// kapp-controller PackageInstall CR is installed under the same namespace as tanzuClusterBootstrap. The namespace
+			// is also the same as where the cluster belongs.
+			Namespace: cluster.Namespace,
+			OwnerReferences: []metav1.OwnerReference{
+				{
+					APIVersion: clusterapiv1beta1.GroupVersion.String(),
+					Kind:       cluster.Kind,
+					Name:       cluster.Name,
+					UID:        cluster.UID,
+				},
+			},
+		},
 	}
 
-	if pkg.ValuesFrom.ProviderRef != nil {
-		gvr, err := r.getGVR(schema.GroupKind{Group: *pkg.ValuesFrom.ProviderRef.APIGroup, Kind: pkg.ValuesFrom.ProviderRef.Kind})
+	pkgiMutateFn := func() error {
+		// TODO: Followup on the following fields, only add them if needed.
+		// https://github.com/vmware-tanzu/tanzu-framework/issues/1677
+		// if ipkg.ObjectMeta.Annotations == nil {
+		//	 ipkg.ObjectMeta.Annotations = make(map[string]string)
+		// }
+		// ipkg.ObjectMeta.Annotations[addontypes.YttMarkerAnnotation] = ""
+		pkgi.Spec.SyncPeriod = &metav1.Duration{Duration: r.Config.PkgiSyncPeriod}
+		pkgi.Spec.PackageRef = &kapppkgiv1alpha1.PackageRef{
+			// clusterBootstrap.Spec.Kapp.RefName is Package.Name. I.e., kapp-controller.tanzu.vmware.com.0.28.0+vmware.1-tkg.1-rc.1
+			// PackageInstall.Spec.PackageRef looks for the Package.Spec.refName which is a short name of the full Package.Name
+			// packageRefName and packageVersion are fetched from the Package CR.
+			RefName: packageRefName,
+			VersionSelection: &versions.VersionSelectionSemver{
+				Constraints: packageVersion,
+				Prereleases: &versions.VersionSelectionSemverPrereleases{},
+			},
+		}
+		// Adding the cluster reference to PackageInstall spec to instruct kapp-controller where to deploy
+		// the underlying resources
+		clusterKubeconfigDetails := util.GetClusterKubeconfigSecretDetails(cluster)
+		pkgi.Spec.Cluster = &kappctrlv1alpha1.AppCluster{
+			KubeconfigSecretRef: &kappctrlv1alpha1.AppClusterKubeconfigSecretRef{
+				Name: clusterKubeconfigDetails.Name,
+				Key:  clusterKubeconfigDetails.Key,
+			},
+		}
+		secretName, err := r.GetDataValueSecretNameFromBootstrapPackage(clusterBootstrap.Spec.Kapp, cluster.Namespace)
 		if err != nil {
-			log.Error(err, "failed to getGVR")
-			return nil, err
+			return err
 		}
-		provider, err := r.dynamicClient.Resource(*gvr).Namespace(namespace).Get(r.context, pkg.ValuesFrom.ProviderRef.Name, metav1.GetOptions{}, "status")
+		pkgi.Spec.Values = []kapppkgiv1alpha1.PackageInstallValues{
+			{SecretRef: &kapppkgiv1alpha1.PackageInstallValuesSecretRef{
+				Name: secretName},
+			},
+		}
+		return nil
+	}
 
-		if err != nil {
-			log.Error(err, "unable to fetch provider", "provider", pkg.ValuesFrom.ProviderRef, "gvr", gvr)
-			return nil, err
+	_, err = controllerutil.CreateOrPatch(r.context, r.Client, pkgi, pkgiMutateFn)
+	if err != nil {
+		r.Log.Error(err, fmt.Sprintf("unable to create or patch PackageInstall %s/%s for cluster: %s",
+			pkgi.Namespace, pkgi.Name, cluster.Name))
+		return err
+	}
+
+	r.Log.Info(fmt.Sprintf("created or patched the PackageInstall %s/%s for cluster %s", pkgi.Namespace, pkgi.Name, cluster.Name))
+	return nil
+}
+
+// createOrPatchPackageOnRemote creates the Package CR on remote cluster. In order to install a package on remote cluster
+// the Package CR needs to be present.
+// createOrPatchPackageOnRemote returns a tuple: (<remote-package>, <error>)
+func (r *ClusterBootstrapReconciler) createOrPatchPackageOnRemote(cluster *clusterapiv1beta1.Cluster,
+	cbPkg *runtanzuv1alpha3.ClusterBootstrapPackage, clusterClient client.Client) (*kapppkgv1alpha1.Package, error) {
+
+	var err error
+	// Create or patch Package CR on remote cluster
+	localPackage := &kapppkgv1alpha1.Package{}
+	key := client.ObjectKey{Namespace: cluster.Namespace, Name: cbPkg.RefName}
+	if err = r.liveClient.Get(r.context, key, localPackage); err != nil {
+		// If there is an error to get the Carvel Package CR from local cluster, nothing needs to be created/cloned on remote.
+		// Let the reconciler try again.
+		r.Log.Error(err, fmt.Sprintf("unable to create or patch Package %s on cluster %s/%s. Error occurs when getting Package %s from the management cluster",
+			key.String(), cluster.Namespace, cluster.Name, key.String()))
+		return nil, err
+	}
+	remotePackage := &kapppkgv1alpha1.Package{}
+	remotePackage.SetName(localPackage.Name)
+	// The Package CR on remote cluster needs to be under configured system namespace
+	remotePackage.SetNamespace(r.Config.SystemNamespace)
+	_, err = controllerutil.CreateOrPatch(r.context, clusterClient, remotePackage, func() error {
+		remotePackage.Spec = *localPackage.Spec.DeepCopy()
+		// TODO: Follow up to see if we need to preserve all the other fields, like annotations
+		// https://github.com/vmware-tanzu/tanzu-framework/issues/1678
+		return nil
+	})
+	if err != nil {
+		r.Log.Error(err, fmt.Sprintf("unable to create or patch Package resource %s/%s on cluster: %s/%s",
+			remotePackage.Namespace, remotePackage.Name, cluster.Namespace, cluster.Name))
+		return nil, err
+	}
+	return remotePackage, nil
+}
+
+// createOrPatchPackageInstallOnRemote creates or patches PackageInstall CR on remote cluster. The kapp-controller
+// running on remote cluster will reconcile it and deploy resources.
+func (r *ClusterBootstrapReconciler) createOrPatchPackageInstallOnRemote(cluster *clusterapiv1beta1.Cluster,
+	cbPkg *runtanzuv1alpha3.ClusterBootstrapPackage, remoteSecret *corev1.Secret, clusterClient client.Client) (*kapppkgiv1alpha1.PackageInstall, error) {
+
+	// In order to create PackageInstall CR, we need to get the Package.Spec.RefName and Package.Spec.Version
+	remotePackageRefName, remotePackageVersion, err := util.GetPackageMetadata(r.context, clusterClient, cbPkg.RefName, r.Config.SystemNamespace)
+	if remotePackageRefName == "" || remotePackageVersion == "" || err != nil {
+		// Package.Spec.RefName and Package.Spec.Version are required fields for Package CR. We do not expect them to be
+		// empty and error should not happen when fetching them from a Package CR.
+		r.Log.Error(err, fmt.Sprintf("unable to fetch Package.Spec.RefName or Package.Spec.Version from Package %s/%s on cluster %s/%s",
+			r.Config.SystemNamespace, cbPkg.RefName, cluster.Namespace, cluster.Name))
+		return nil, err
+	}
+
+	// Create PackageInstall CRs on the remote workload cluster, kapp-controller will take care of reconciling them
+	remotePkgi := &kapppkgiv1alpha1.PackageInstall{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      util.GeneratePackageInstallName(cluster.Name, remotePackageRefName),
+			Namespace: r.Config.SystemNamespace,
+		},
+	}
+
+	_, err = controllerutil.CreateOrPatch(r.context, clusterClient, remotePkgi, func() error {
+		remotePkgi.Spec.ServiceAccountName = r.Config.PkgiServiceAccount
+		// remotePackageRefName and remotePackageVersion are fetched from the Package CR on remote cluster.
+		remotePkgi.Spec.PackageRef = &kapppkgiv1alpha1.PackageRef{
+			RefName: remotePackageRefName,
+			VersionSelection: &versions.VersionSelectionSemver{
+				Constraints: remotePackageVersion,
+				Prereleases: &versions.VersionSelectionSemverPrereleases{},
+			},
 		}
-		secretName, found, err := unstructured.NestedString(provider.UnstructuredContent(), "status", "secretRef")
-		if err != nil {
-			log.Error(err, "unable to fetch secretRef in provider", "provider", provider)
-			return nil, err
+		if remoteSecret != nil {
+			// The nil remoteSecret means no data values for current ClusterBootstrapPackage are needed. And no remote secret
+			// object gets created. The PackageInstall CR should be created without specifying the spec.Values.
+			remotePkgi.Spec.Values = []kapppkgiv1alpha1.PackageInstallValues{
+				{SecretRef: &kapppkgiv1alpha1.PackageInstallValuesSecretRef{
+					Name: remoteSecret.Name},
+				},
+			}
 		}
-		if !found {
-			log.Info("provider status does not have secretRef", "provider", provider)
+		return nil
+	})
+	if err != nil {
+		r.Log.Error(err, fmt.Sprintf("unable to create or patch PackageInstall resource %s/%s on cluster: %s/%s",
+			remotePkgi.Namespace, remotePkgi.Name, cluster.Namespace, cluster.Name))
+		return nil, err
+	}
+
+	return remotePkgi, nil
+}
+
+func (r *ClusterBootstrapReconciler) reconcileSystemNamespace(clusterClient client.Client) error {
+	namespace := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: r.Config.SystemNamespace,
+		},
+	}
+
+	result, err := controllerutil.CreateOrPatch(r.context, clusterClient, namespace, nil)
+	if err != nil {
+		r.Log.Error(err, "Error creating or patching system namespace")
+		return err
+	}
+	if result != controllerutil.OperationResultNone {
+		r.Log.Info("created namespace", "namespace", r.Config.SystemNamespace)
+	}
+	return nil
+}
+
+// createOrPatchAddonServiceAccountOnRemote creates or patches the addon ServiceAccount on remote cluster.
+// The ServiceAccount will be referenced by the PackageInstall CR, so that kapp-controller on remote cluster could consume
+// for PackageInstall reconciliation.
+func (r *ClusterBootstrapReconciler) createOrPatchAddonServiceAccountOnRemote(cluster *clusterapiv1beta1.Cluster, clusterClient client.Client) (*corev1.ServiceAccount, error) {
+	serviceAccount := &corev1.ServiceAccount{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      r.Config.PkgiServiceAccount,
+			Namespace: r.Config.SystemNamespace,
+		},
+	}
+
+	r.Log.Info(fmt.Sprintf("creating or patching ServiceAccount %s/%s on cluster %s/%s",
+		serviceAccount.Namespace, serviceAccount.Name, cluster.Namespace, cluster.Name))
+
+	_, err := controllerutil.CreateOrPatch(r.context, clusterClient, serviceAccount, nil)
+	if err != nil {
+		if apierrors.IsAlreadyExists(err) {
+			// If the error is IsAlreadyExists, we ignore and return nil
 			return nil, nil
 		}
-		key := client.ObjectKey{Namespace: namespace, Name: secretName}
-		if err := r.Get(r.context, key, secret); err != nil {
-			log.Error(err, "unable to fetch secret", "objectkey", key)
-			return nil, err
+		r.Log.Error(err, fmt.Sprintf("unable to create or patch ServiceAccount %s/%s on cluster %s/%s",
+			serviceAccount.Namespace, serviceAccount.Name, cluster.Namespace, cluster.Name))
+		return nil, err
+	}
+
+	return serviceAccount, nil
+}
+
+// createOrPatchAddonRBACOnRemote creates or patches the ClusterRole, ClusterRoleBinding on remote cluster.
+// The ClusterRole is bound to the ServiceAccount which is referenced by PackageInstall CR, so that kapp-controller on remote
+// cluster could have privileges to lifecycle manage package resources.
+func (r *ClusterBootstrapReconciler) createOrPatchAddonRBACOnRemote(cluster *clusterapiv1beta1.Cluster, clusterClient client.Client) error {
+	addonRole := &rbacv1.ClusterRole{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: r.Config.PkgiClusterRole,
+		},
+	}
+
+	if _, err := controllerutil.CreateOrPatch(r.context, clusterClient, addonRole, func() error {
+		addonRole.Rules = []rbacv1.PolicyRule{
+			{
+				APIGroups: []string{"*"},
+				Verbs:     []string{"*"},
+				Resources: []string{"*"},
+			},
+		}
+		return nil
+	}); err != nil {
+		if !apierrors.IsAlreadyExists(err) {
+			r.Log.Error(err,
+				fmt.Sprintf("unable to create or patch ClusterRole %s/%s on cluster %s/%s",
+					addonRole.Namespace, addonRole.Name, cluster.Namespace, cluster.Name))
+			return err
+		}
+		r.Log.Info(fmt.Sprintf("ClusterRole %s already exists on cluster %s/%s. Nothing to create or patch.", addonRole.Name, cluster.Namespace, cluster.Name))
+	}
+
+	r.Log.Info(fmt.Sprintf("created or patched ClusterRole %s/%s on cluster %s/%s",
+		addonRole.Namespace, addonRole.Name, cluster.Namespace, cluster.Name))
+
+	addonRoleBinding := &rbacv1.ClusterRoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: r.Config.PkgiClusterRoleBinding,
+		},
+	}
+	if _, err := controllerutil.CreateOrPatch(r.context, clusterClient, addonRoleBinding, func() error {
+		addonRoleBinding.Subjects = []rbacv1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      r.Config.PkgiServiceAccount,
+				Namespace: r.Config.SystemNamespace,
+			},
+		}
+		addonRoleBinding.RoleRef = rbacv1.RoleRef{
+			APIGroup: "rbac.authorization.k8s.io",
+			Kind:     "ClusterRole",
+			Name:     r.Config.PkgiClusterRole,
 		}
+		return nil
+	}); err != nil {
+		if !apierrors.IsAlreadyExists(err) {
+			r.Log.Error(err, fmt.Sprintf("unable to create or patch ClusterRoleBinding %s/%s on cluster %s/%s",
+				addonRoleBinding.Namespace, addonRoleBinding.Name, cluster.Namespace, cluster.ClusterName))
+			return err
+		}
+		r.Log.Info(fmt.Sprintf("ClusterRoleBinding %s/%s already exists on cluster %s/%s. Nothing to create or patch.",
+			addonRoleBinding.Namespace, addonRole.Name, cluster.Namespace, cluster.Name))
+	}
+
+	return nil
+}
+
+// createOrPatchAddonResourcesOnRemote creates or patches the resources for a cluster bootstrap package on remote workload
+// cluster. The resources are [Package CR, Secret for PackageInstall, PackageInstall CR].
+func (r *ClusterBootstrapReconciler) createOrPatchAddonResourcesOnRemote(cluster *clusterapiv1beta1.Cluster,
+	cbPkg *runtanzuv1alpha3.ClusterBootstrapPackage, clusterClient client.Client) error {
+
+	remotePackage, err := r.createOrPatchPackageOnRemote(cluster, cbPkg, clusterClient)
+	if err != nil {
+		return err
 	}
+	r.Log.Info(fmt.Sprintf("created the Package CR %s on cluster %s/%s", remotePackage.Name, cluster.Namespace,
+		cluster.Name))
 
+	// Create or patch the data value secret on remote cluster. The data value secret has been generated by each
+	// addon config controller on local cluster.
+	remoteSecret, err := r.createOrPatchPackageInstallSecretOnRemote(cluster, cbPkg, clusterClient)
+	if err != nil {
+		// We expect there is NO error to create or patch the secret used for PackageInstall in a cluster.
+		// Logging has been handled by createOrPatchPackageInstallSecretOnRemote() already
+		return err
+	}
+	if remoteSecret != nil {
+		r.Log.Info(fmt.Sprintf("created or patched secret for package %s on cluster %s/%s", remotePackage.Name, cluster.Namespace,
+			cluster.Name))
+	}
+
+	pkgi, err := r.createOrPatchPackageInstallOnRemote(cluster, cbPkg, remoteSecret, clusterClient)
+	if err != nil {
+		return err
+	}
+	r.Log.Info(fmt.Sprintf("created or patched the PackageInstall CR %s/%s on cluster %s/%s",
+		pkgi.Namespace, pkgi.Name, cluster.Namespace, cluster.Name))
+
+	return nil
+}
+
+// createOrPatchPackageInstallSecretOnRemote creates or patches the secret used for PackageInstall in a cluster
+func (r *ClusterBootstrapReconciler) createOrPatchPackageInstallSecretOnRemote(cluster *clusterapiv1beta1.Cluster,
+	cbpkg *runtanzuv1alpha3.ClusterBootstrapPackage, clusterClient client.Client) (*corev1.Secret, error) {
+
+	secretName, err := r.GetDataValueSecretNameFromBootstrapPackage(cbpkg, cluster.Namespace)
+	if err != nil {
+		// logging has been handled in GetDataValueSecretNameFromBootstrapPackage()
+		return nil, err
+	}
+	if secretName == "" {
+		r.Log.Info(fmt.Sprintf("no data values secret is needed for ClusterBootstrapPackage: %s, nothing to be created or patched on cluster %s/%s",
+			cbpkg.RefName, cluster.Namespace, cluster.Name))
+		return nil, nil
+	}
+
+	localSecret := &corev1.Secret{}
+	key := client.ObjectKey{Namespace: cluster.Namespace, Name: secretName}
+	if err = r.Get(r.context, key, localSecret); err != nil {
+		r.Log.Error(err, "unable to fetch secret", "objectKey", key)
+		return nil, err
+	}
+
+	// TODO: This logic should be moved to cloneSecretsAndProviders()
+	// https://github.com/vmware-tanzu/tanzu-framework/issues/1729
 	// Add cluster and package labels to secrets if not already present
 	// This helps us to track the secrets in the watch and trigger Reconcile requests when these secrets are updated
-	patchedSecret := secret.DeepCopy()
-	if patchSecretWithLabels(patchedSecret, pkg.RefName, cluster.Name) {
-		if err := r.Patch(r.context, patchedSecret, client.MergeFrom(secret)); err != nil {
-			log.Error(err, "unable to patch secret labels for ", "secret", secret.Name)
+	patchedSecret := localSecret.DeepCopy()
+	if patchSecretWithLabels(patchedSecret, util.ParseStringForLabel(cbpkg.RefName), cluster.Name) {
+		if err := r.Patch(r.context, patchedSecret, client.MergeFrom(localSecret)); err != nil {
+			r.Log.Error(err, "unable to patch secret labels for ", "secret", localSecret.Name)
 			return nil, err
 		}
-		log.Info("Patched secrets with package and cluster labels to watch for changes")
+		r.Log.Info(fmt.Sprintf("patched the secret %s/%s with package and cluster labels", localSecret.Namespace, localSecret.Name))
 	}
 
-	// Now prepare the dataValuesSecret to send to target cluster
-	dataValuesSecret := &corev1.Secret{}
-	dataValuesSecret.Name = fmt.Sprintf("%s-%s-data-values", cluster.Name, packageShortName(pkg.RefName))
-	dataValuesSecret.Namespace = constants.TKGSystemNS
-	dataValuesSecret.Type = corev1.SecretTypeOpaque
+	packageRefName, _, err := util.GetPackageMetadata(r.context, r.liveClient, cbpkg.RefName, cluster.Namespace)
+	if err != nil {
+		r.Log.Error(err, fmt.Sprintf("unable to get Package CR %s/%s for its metadata", cluster.Namespace, cbpkg.RefName))
+		return nil, err
+	}
+
+	remoteSecret := &corev1.Secret{}
+	remoteSecret.Name = util.GenerateDataValueSecretName(cluster.Name, packageRefName)
+	// The secret will be created or patched under tkg-system namespace on remote cluster
+	remoteSecret.Namespace = r.Config.SystemNamespace
+	remoteSecret.Type = corev1.SecretTypeOpaque
 
 	dataValuesSecretMutateFn := func() error {
-		dataValuesSecret.Data = map[string][]byte{}
-		for k, v := range secret.Data {
-			dataValuesSecret.Data[k] = v
+		remoteSecret.Data = map[string][]byte{}
+		for k, v := range patchedSecret.Data {
+			remoteSecret.Data[k] = v
 		}
 		return nil
 	}
 
-	_, err := controllerutil.CreateOrPatch(r.context, clusterClient, dataValuesSecret, dataValuesSecretMutateFn)
+	_, err = controllerutil.CreateOrPatch(r.context, clusterClient, remoteSecret, dataValuesSecretMutateFn)
 	if err != nil {
-		log.Error(err, "Error creating or patching addon data values secret")
+		r.Log.Error(err, "error creating or patching addon data values secret")
 		return nil, err
 	}
-	return dataValuesSecret, nil
+	return remoteSecret, nil
 }
 
 // patchSecretWithLabels updates the secret by adding package and cluster labels
@@ -428,21 +797,30 @@ func (r *ClusterBootstrapReconciler) cloneSecretsAndProviders(cluster *clusterap
 
 // updateValues updates secretRef and/or providerRef
 func (r *ClusterBootstrapReconciler) updateValues(cluster *clusterapiv1beta1.Cluster, bootstrap *runtanzuv1alpha3.ClusterBootstrap,
-	pkg *runtanzuv1alpha3.ClusterBootstrapPackage, templateNS string, log logr.Logger) (*corev1.Secret, *unstructured.Unstructured, error) {
+	cbPkg *runtanzuv1alpha3.ClusterBootstrapPackage, cbTemplateNamespace string, log logr.Logger) (*corev1.Secret, *unstructured.Unstructured, error) {
+
+	packageRefName, _, err := util.GetPackageMetadata(r.context, r.liveClient, cbPkg.RefName, cluster.Namespace)
+	if packageRefName == "" || err != nil {
+		// Package.Spec.RefName and Package.Spec.Version are required fields for Package CR. We do not expect them to be
+		// empty and error should not happen when fetching them from a Package CR.
+		r.Log.Error(err, fmt.Sprintf("unable to fetch Package.Spec.RefName or Package.Spec.Version from Package %s/%s",
+			cluster.Namespace, cbPkg.RefName))
+		return nil, nil, err
+	}
 
-	if pkg.ValuesFrom == nil {
+	if cbPkg.ValuesFrom == nil {
 		return nil, nil, nil
 	}
-	if pkg.ValuesFrom.SecretRef != "" {
-		secret, err := r.updateValuesFromSecret(cluster, bootstrap, pkg, templateNS, log)
+	if cbPkg.ValuesFrom.SecretRef != "" {
+		secret, err := r.updateValuesFromSecret(cluster, bootstrap, cbPkg, cbTemplateNamespace, packageRefName, log)
 		if err != nil {
 			return nil, nil, err
 		}
 		return secret, nil, nil
 	}
 
-	if pkg.ValuesFrom.ProviderRef != nil {
-		provider, err := r.updateValuesFromProvider(cluster, bootstrap, pkg, templateNS, log)
+	if cbPkg.ValuesFrom.ProviderRef != nil {
+		provider, err := r.updateValuesFromProvider(cluster, bootstrap, cbPkg, cbTemplateNamespace, packageRefName, log)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -460,34 +838,41 @@ func (r *ClusterBootstrapReconciler) ensureOwnerRef(ownerRef *metav1.OwnerRefere
 		}
 		_, err := controllerutil.CreateOrPatch(r.context, r.Client, secret, ownerRefsMutateFn)
 		if err != nil {
+			r.Log.Error(err, fmt.Sprintf("unable to create or patch the secret %s/%s with ownerRef", secret.Namespace, secret.Name))
 			return err
 		}
 	}
 	for _, provider := range providers {
-		provider.SetOwnerReferences(clusterapiutil.EnsureOwnerRef(provider.GetOwnerReferences(), *ownerRef))
 		gvr, err := r.getGVR(provider.GroupVersionKind().GroupKind())
 		if err != nil {
+			r.Log.Error(err, fmt.Sprintf("unable to get GVR of provider %s/%s", provider.GetNamespace(), provider.GetName()))
 			return err
 		}
-
-		_, err = r.dynamicClient.Resource(*gvr).Namespace(provider.GetNamespace()).Update(r.context, provider, metav1.UpdateOptions{})
+		err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
+			// We need to get and update, otherwise there could have concurrency issue: ["the object has been modified; please
+			// apply your changes to the latest version and try again"]
+			newProvider, errGetProvider := r.dynamicClient.Resource(*gvr).Namespace(provider.GetNamespace()).Get(r.context, provider.GetName(), metav1.GetOptions{})
+			if errGetProvider != nil {
+				r.Log.Error(errGetProvider, fmt.Sprintf("unable to get provider %s/%s", provider.GetNamespace(), provider.GetName()))
+				return errGetProvider
+			}
+			newProvider = newProvider.DeepCopy()
+			newProvider.SetOwnerReferences(clusterapiutil.EnsureOwnerRef(provider.GetOwnerReferences(), *ownerRef))
+			_, errUpdateProvider := r.dynamicClient.Resource(*gvr).Namespace(newProvider.GetNamespace()).Update(r.context, newProvider, metav1.UpdateOptions{})
+			if errUpdateProvider != nil {
+				r.Log.Error(errUpdateProvider, fmt.Sprintf("unable to update provider %s/%s", provider.GetNamespace(), provider.GetName()))
+				return errUpdateProvider
+			}
+			return nil
+		})
 		if err != nil {
+			r.Log.Error(err, fmt.Sprintf("unable to update the OwnerRefrences for provider %s/%s", provider.GetNamespace(), provider.GetName()))
 			return err
 		}
 	}
 	return nil
 }
 
-func packageShortName(refName string) string {
-	if refName != "" {
-		refParts := strings.Split(refName, ".")
-		if len(refParts) > 0 {
-			return refParts[0]
-		}
-	}
-	return refName
-}
-
 // getGVR returns a GroupVersionResource for a GroupKind
 func (r *ClusterBootstrapReconciler) getGVR(gk schema.GroupKind) (*schema.GroupVersionResource, error) {
 	if gvr, ok := r.providerGVR[gk]; ok {
@@ -532,7 +917,7 @@ func (r *ClusterBootstrapReconciler) periodicGVRCachesClean() {
 
 // updateValuesFromSecret updates secretRef in valuesFrom
 func (r *ClusterBootstrapReconciler) updateValuesFromSecret(cluster *clusterapiv1beta1.Cluster, bootstrap *runtanzuv1alpha3.ClusterBootstrap,
-	pkg *runtanzuv1alpha3.ClusterBootstrapPackage, templateNS string, log logr.Logger) (*corev1.Secret, error) {
+	pkg *runtanzuv1alpha3.ClusterBootstrapPackage, templateNS, pkgRefName string, log logr.Logger) (*corev1.Secret, error) {
 
 	var newSecret *corev1.Secret
 	if pkg.ValuesFrom.SecretRef != "" {
@@ -544,42 +929,45 @@ func (r *ClusterBootstrapReconciler) updateValuesFromSecret(cluster *clusterapiv
 		}
 		newSecret = secret.DeepCopy()
 		newSecret.ObjectMeta.Reset()
-		newSecret.OwnerReferences = []metav1.OwnerReference{
-			{
-				APIVersion: clusterapiv1beta1.GroupVersion.String(),
-				Kind:       cluster.Kind,
-				Name:       cluster.Name,
-				UID:        cluster.UID,
-			},
-		}
-
-		// Add cluster and package labels to cloned secrets
-		if newSecret.Labels == nil {
-			newSecret.Labels = map[string]string{}
-		}
-
-		newSecret.Labels[types.PackageNameLabel] = pkg.RefName
-		newSecret.Labels[types.ClusterNameLabel] = cluster.Name
+		newSecret.Name = fmt.Sprintf("%s-%s-package", cluster.Name, pkgRefName)
+		newSecret.Namespace = bootstrap.Namespace
 
-		// Set secret.Type to ClusterBootstrapManagedSecret to enable us to Watch these secrets
-		newSecret.Type = constants.ClusterBootstrapManagedSecret
+		var createOrPatchErr error
+		_, createOrPatchErr = controllerutil.CreateOrPatch(r.context, r.Client, newSecret, func() error {
+			newSecret.OwnerReferences = []metav1.OwnerReference{
+				{
+					APIVersion: clusterapiv1beta1.GroupVersion.String(),
+					Kind:       cluster.Kind,
+					Name:       cluster.Name,
+					UID:        cluster.UID,
+				},
+			}
 
-		newSecret.Name = fmt.Sprintf("%s-%s-package", cluster.Name, packageShortName(pkg.RefName))
-		newSecret.Namespace = bootstrap.Namespace
-		if err := r.Create(r.context, newSecret); err != nil {
-			return nil, err
+			// Add cluster and package labels to cloned secrets
+			if newSecret.Labels == nil {
+				newSecret.Labels = map[string]string{}
+			}
+			newSecret.Labels[types.PackageNameLabel] = util.ParseStringForLabel(pkg.RefName)
+			newSecret.Labels[types.ClusterNameLabel] = cluster.Name
+			// Set secret.Type to ClusterBootstrapManagedSecret to enable us to Watch these secrets
+			newSecret.Type = constants.ClusterBootstrapManagedSecret
+			return nil
+		})
+		if createOrPatchErr != nil {
+			return nil, createOrPatchErr
 		}
+		r.Log.Info(fmt.Sprintf("created or patched Secret %s/%s", newSecret.Namespace, newSecret.Name))
 		pkg.ValuesFrom.SecretRef = newSecret.Name
 	}
-
 	return newSecret, nil
 }
 
 // updateValuesFromProvider updates providerRef in valuesFrom
 func (r *ClusterBootstrapReconciler) updateValuesFromProvider(cluster *clusterapiv1beta1.Cluster, bootstrap *runtanzuv1alpha3.ClusterBootstrap,
-	pkg *runtanzuv1alpha3.ClusterBootstrapPackage, templateNS string, log logr.Logger) (*unstructured.Unstructured, error) {
+	pkg *runtanzuv1alpha3.ClusterBootstrapPackage, cbTemplateNamespace, pkgRefName string, log logr.Logger) (*unstructured.Unstructured, error) {
 
 	var newProvider *unstructured.Unstructured
+	var createdOrUpdatedProvider *unstructured.Unstructured
 	valuesFrom := pkg.ValuesFrom
 	if valuesFrom.ProviderRef != nil {
 		gvr, err := r.getGVR(schema.GroupKind{Group: *valuesFrom.ProviderRef.APIGroup, Kind: valuesFrom.ProviderRef.Kind})
@@ -587,9 +975,9 @@ func (r *ClusterBootstrapReconciler) updateValuesFromProvider(cluster *clusterap
 			log.Error(err, "failed to getGVR")
 			return nil, err
 		}
-		provider, err := r.dynamicClient.Resource(*gvr).Namespace(templateNS).Get(r.context, valuesFrom.ProviderRef.Name, metav1.GetOptions{})
+		provider, err := r.dynamicClient.Resource(*gvr).Namespace(cbTemplateNamespace).Get(r.context, valuesFrom.ProviderRef.Name, metav1.GetOptions{})
 		if err != nil {
-			log.Error(err, "unable to fetch provider", "provider", valuesFrom.ProviderRef, "gvr", gvr)
+			log.Error(err, fmt.Sprintf("unable to fetch provider %s/%s", cbTemplateNamespace, valuesFrom.ProviderRef.Name), "gvr", gvr)
 			return nil, err
 		}
 		newProvider = provider.DeepCopy()
@@ -606,26 +994,50 @@ func (r *ClusterBootstrapReconciler) updateValuesFromProvider(cluster *clusterap
 		providerLabels := newProvider.GetLabels()
 		if providerLabels == nil {
 			newProvider.SetLabels(map[string]string{
-				types.PackageNameLabel: pkg.RefName,
+				// A package refName could contain characters that K8S does not like as a label value.
+				// For example, kapp-controller.tanzu.vmware.com.0.30.0+vmware.1-tkg.1 is a
+				// valid package refName, but it contains "+" that K8S complains. We parse the refName by replacing
+				// + to ---.
+				types.PackageNameLabel: util.ParseStringForLabel(pkg.RefName),
 				types.ClusterNameLabel: cluster.Name,
 			})
 		} else {
-			providerLabels[types.PackageNameLabel] = pkg.RefName
+			providerLabels[types.PackageNameLabel] = util.ParseStringForLabel(pkg.RefName)
 			providerLabels[types.ClusterNameLabel] = cluster.Name
 		}
 
-		newProvider.SetName(fmt.Sprintf("%s-%s-package", cluster.Name, packageShortName(pkg.RefName)))
-		log.Info("cloning provider", "provider", newProvider)
-		newProvider, err = r.dynamicClient.Resource(*gvr).Namespace(bootstrap.Namespace).Create(r.context, newProvider, metav1.CreateOptions{})
+		newProvider.SetName(fmt.Sprintf("%s-%s-package", cluster.Name, pkgRefName))
+		newProvider.SetNamespace(bootstrap.Namespace)
+		log.Info(fmt.Sprintf("cloning provider %s/%s to namespace %s", cbTemplateNamespace, newProvider.GetName(), bootstrap.Namespace), "gvr", gvr)
+		// newProvider and createdOrUpdatedProvider are different. The newProvider is the one we want apiserver to accept,
+		// however createdOrUpdatedProvider is the actual object pointer that apiserver has already created with several managed fields,
+		// The intent of this function is to return the actual object pointer that apiserver has created, so we should
+		// return createdOrUpdatedProvider instead of newProvider. Otherwise, when the caller wants to make changes to the
+		// created provider objects, there will be errors, i.e., [invalid: metadata.resourceVersion: Invalid value: 0x0: must be specified for an update]
+		createdOrUpdatedProvider, err = r.dynamicClient.Resource(*gvr).Namespace(bootstrap.Namespace).Create(r.context, newProvider, metav1.CreateOptions{})
 		if err != nil {
-			log.Error(err, "unable to clone provider", "provider", newProvider, "gvr", gvr)
-			return nil, err
+			// There are possibilities that current reconciliation loop fails due to various reasons, and during next reconciliation
+			// loop, it is possible that the provider resource has been created. In this case, we want to run update/patch.
+			if apierrors.IsAlreadyExists(err) {
+				// Setting the resource version is because it's been removed at L984 with 'unstructured.RemoveNestedField(newProvider.Object, "metadata")'
+				// apiserver requires that field to be present for concurrency control.
+				newProvider.SetResourceVersion(provider.GetResourceVersion())
+				createdOrUpdatedProvider, err = r.dynamicClient.Resource(*gvr).Namespace(bootstrap.Namespace).Update(r.context, newProvider, metav1.UpdateOptions{})
+				if err != nil {
+					log.Info(fmt.Sprintf("unable to updated provider %s/%s", newProvider.GetNamespace(), newProvider.GetName()), "gvr", gvr)
+					return nil, err
+				}
+			} else {
+				log.Error(err, fmt.Sprintf("unable to clone provider %s/%s", newProvider.GetNamespace(), newProvider.GetName()), "gvr", gvr)
+				return nil, err
+			}
 		}
 
-		valuesFrom.ProviderRef.Name = newProvider.GetName()
+		valuesFrom.ProviderRef.Name = createdOrUpdatedProvider.GetName()
+		log.Info(fmt.Sprintf("cloned provider %s/%s to namespace %s", createdOrUpdatedProvider.GetNamespace(), createdOrUpdatedProvider.GetName(), bootstrap.Namespace), "gvr", gvr)
 	}
 
-	return newProvider, nil
+	return createdOrUpdatedProvider, nil
 }
 
 // watchProvider will set a watch on the Type indicated by providerRef if not already watching
@@ -664,7 +1076,64 @@ func (r *ClusterBootstrapReconciler) watchProvider(providerRef *corev1.TypedLoca
 	)
 }
 
-func (r *ClusterBootstrapReconciler) getCNI(
+// GetDataValueSecretNameFromBootstrapPackage attempts to get the data value secret name associated with a ClusterBootstrapPackage.
+// Users have three ways to provide the data values for a ClusterBootstrapPackage: [Inline, SecretRef, ProviderRef], or
+// leave ClusterBootstrapPackage.ValuesFrom field as nil. If data values are provided by ProviderRef, the corresponding
+// controller needs to generate the secret object.
+//
+// Returns:
+// - string: The secret name which references to the Secret CR on mgmt cluster under a particular cluster namespace.
+// - error: whether there is error when getting the secret name.
+func (r *ClusterBootstrapReconciler) GetDataValueSecretNameFromBootstrapPackage(cbPkg *runtanzuv1alpha3.ClusterBootstrapPackage, clusterNamespace string) (string, error) {
+	// When valuesFrom is nil, we interpret it as no data values are needed for the package installation.
+	if cbPkg.ValuesFrom == nil {
+		r.Log.Info(fmt.Sprintf("no data values are provided to the ClusterBootstrapPackage.ValuesFrom field. ClusterBootstrapPackage.RefName: %s", cbPkg.RefName))
+		return "", nil
+	}
+
+	// TODO: Handle inline valueFrom. https://github.com/vmware-tanzu/tanzu-framework/issues/1694
+
+	if cbPkg.ValuesFrom.SecretRef != "" {
+		return cbPkg.ValuesFrom.SecretRef, nil
+	}
+
+	if cbPkg.ValuesFrom.ProviderRef != nil {
+		gvr, err := r.getGVR(schema.GroupKind{Group: *cbPkg.ValuesFrom.ProviderRef.APIGroup, Kind: cbPkg.ValuesFrom.ProviderRef.Kind})
+		if err != nil {
+			r.Log.Error(err, "unable to get GVR")
+			return "", err
+		}
+		provider, err := r.dynamicClient.Resource(*gvr).Namespace(clusterNamespace).Get(r.context, cbPkg.ValuesFrom.ProviderRef.Name, metav1.GetOptions{}, "status")
+		if err != nil {
+			r.Log.Error(err, "unable to fetch provider", "GVR", gvr)
+			return "", err
+		}
+		secretName, found, err := unstructured.NestedString(provider.UnstructuredContent(), "status", "secretRef")
+		if err != nil {
+			r.Log.Error(err, "unable to fetch secretRef in provider", "GVR", gvr)
+			return "", err
+		}
+		if !found {
+			// In this case, we expect the secretRef to be present under status subresource and its value gets updated by
+			// the corresponding controller. However, the config controller might not create the secret in time.
+			r.Log.Info("provider status does not have secretRef", "GVR", gvr)
+			return "", nil
+		}
+		return secretName, nil
+	}
+
+	// When valuesFrom is not nil, but either valuesFrom.Inline, valuesFrom.SecretRef, or valuesFrom.providerRef is empty or nil,
+	// we interpret it as the data value secret for that package has not been available yet. One of those three fields needs
+	// to be provided either by the user or the controller.
+	err := fmt.Errorf("unable to get the data value secret name from the ClusterBootstrapPackage.ValuesFrom field. "+
+		"ClusterBootstrapPackage.RefName: %s. One of the fields under ClusterBootstrapPackage.ValuesFrom is empty or nil",
+		cbPkg.RefName)
+	// The message in err object has sufficient information
+	r.Log.Error(err, "")
+	return "", err
+}
+
+func (r *ClusterBootstrapReconciler) getCNIForClusterBootstrap(
 	clusterBootstrapTemplate *runtanzuv1alpha3.ClusterBootstrapTemplate,
 	cluster *clusterapiv1beta1.Cluster,
 	log logr.Logger) (*runtanzuv1alpha3.ClusterBootstrapPackage, error) {
@@ -677,12 +1146,26 @@ func (r *ClusterBootstrapReconciler) getCNI(
 		return nil, err
 	}
 	if selectedCNI != "" {
+		var getPkgMetadataErrs []error
+		foundCNI := false
 		for _, cni := range clusterBootstrapTemplate.Spec.CNIs {
-			if selectedCNI == packageShortName(cni.RefName) {
+			// Package should be available in cluster namespace
+			pkgRefName, _, getPkgMetadataErr := util.GetPackageMetadata(r.context, r.liveClient, cni.RefName, cluster.Namespace)
+			if getPkgMetadataErr != nil {
+				getPkgMetadataErrs = append(getPkgMetadataErrs, getPkgMetadataErr)
+			}
+			// selectedCNI string from cluster.Topology.Variables could be any arbitrary string. I.e., antrea or antrea.tanzu.vmware.com
+			// A Carvel package refName could follow a different naming convention.
+			// When comparing the selectedCNI string with
+			if strings.HasPrefix(pkgRefName, selectedCNI) {
 				clusterBootstrapPackage = cni
+				foundCNI = true
 				break
 			}
 		}
+		if len(getPkgMetadataErrs) != 0 && !foundCNI {
+			return nil, errorsutil.NewAggregate(getPkgMetadataErrs)
+		}
 	} else {
 		if len(clusterBootstrapTemplate.Spec.CNIs) > 0 {
 			clusterBootstrapPackage = clusterBootstrapTemplate.Spec.CNIs[0]
@@ -690,7 +1173,6 @@ func (r *ClusterBootstrapReconciler) getCNI(
 			return nil, errors.New("no CNI was specified in the ClusterClass or in ClusterBootstrap.Spec.CNIs")
 		}
 	}
-
 	return clusterBootstrapPackage, nil
 }
 
@@ -704,10 +1186,6 @@ func (r *ClusterBootstrapReconciler) watchesForClusterBootstrap() []ClusterBoots
 			&source.Kind{Type: &corev1.Secret{}},
 			handler.EnqueueRequestsFromMapFunc(r.SecretsToClusters),
 		},
-		{
-			&source.Kind{Type: &corev1.Secret{}},
-			handler.EnqueueRequestsFromMapFunc(r.SecretsToClusters),
-		},
 	}
 }
 
@@ -725,23 +1203,23 @@ func (r *ClusterBootstrapReconciler) reconcileClusterProxyAndNetworkSettings(clu
 	// Individual config controllers are responsible for validating the info provided
 	HTTPProxy, err := util.ParseClusterVariableString(cluster, r.Config.HTTPProxyClusterClassVarName)
 	if err != nil {
-		log.Error(err, "Failed to fetch cluster HTTP proxy setting, defaulting to empty")
+		log.Error(err, "unable to fetch cluster HTTP proxy setting, defaulting to empty")
 	}
 	HTTPSProxy, err := util.ParseClusterVariableString(cluster, r.Config.HTTPSProxyClusterClassVarName)
 	if err != nil {
-		log.Error(err, "Failed to fetch cluster HTTPS proxy setting, defaulting to empty")
+		log.Error(err, "unable to fetch cluster HTTPS proxy setting, defaulting to empty")
 	}
 	NoProxy, err := util.ParseClusterVariableString(cluster, r.Config.NoProxyClusterClassVarName)
 	if err != nil {
-		log.Error(err, "Failed to fetch cluster no-proxy setting, defaulting to empty")
+		log.Error(err, "unable to fetch cluster no-proxy setting, defaulting to empty")
 	}
 	ProxyCACert, err := util.ParseClusterVariableString(cluster, r.Config.ProxyCACertClusterClassVarName)
 	if err != nil {
-		log.Error(err, "Failed to fetch cluster proxy CA certificate, defaulting to empty")
+		log.Error(err, "unable to fetch cluster proxy CA certificate, defaulting to empty")
 	}
 	IPFamily, err := util.ParseClusterVariableString(cluster, r.Config.IPFamilyClusterClassVarName)
 	if err != nil {
-		log.Error(err, "Failed to fetch cluster IP family, defaulting to empty")
+		log.Error(err, "unable to fetch cluster IP family, defaulting to empty")
 	}
 
 	if cluster.Annotations == nil {
@@ -756,7 +1234,7 @@ func (r *ClusterBootstrapReconciler) reconcileClusterProxyAndNetworkSettings(clu
 	log.Info("setting proxy and network configurations in Cluster annotation", types.HTTPProxyConfigAnnotation, HTTPProxy, types.HTTPSProxyConfigAnnotation, HTTPSProxy, types.NoProxyConfigAnnotation, NoProxy, types.ProxyCACertConfigAnnotation, ProxyCACert, types.IPFamilyConfigAnnotation, IPFamily)
 
 	if err := patchHelper.Patch(r.context, cluster); err != nil {
-		log.Error(err, "Error patching Cluster Annotation")
+		log.Error(err, "unable to patch Cluster Annotation")
 		return err
 	}
 
diff --git a/addons/controllers/kapp-controller/kappcontrollerconfig_controller.go b/addons/controllers/kapp-controller/kappcontrollerconfig_controller.go
index 096aebb7ed..b38d5c70c7 100644
--- a/addons/controllers/kapp-controller/kappcontrollerconfig_controller.go
+++ b/addons/controllers/kapp-controller/kappcontrollerconfig_controller.go
@@ -29,8 +29,6 @@ import (
 	runv1alpha3 "github.com/vmware-tanzu/tanzu-framework/apis/run/v1alpha3"
 )
 
-const KappControllerAddonName = "kapp-controller"
-
 // KappControllerConfigReconciler reconciles a KappControllerConfig object
 type KappControllerConfigReconciler struct {
 	client.Client
@@ -64,7 +62,7 @@ func (r *KappControllerConfigReconciler) Reconcile(ctx context.Context, req ctrl
 	clusterNamespacedName := req.NamespacedName
 	cluster := &clusterapiv1beta1.Cluster{}
 	for _, owner := range kappControllerConfig.OwnerReferences {
-		if owner.Kind == cluster.Kind {
+		if owner.Kind == constants.ClusterKind {
 			clusterNamespacedName.Name = owner.Name
 			break
 		}
@@ -158,7 +156,7 @@ func (r *KappControllerConfigReconciler) ReconcileKappControllerConfigNormal(
 	}
 
 	// update status.secretRef
-	dataValueSecretName := util.GenerateDataValueSecretName(kappControllerConfig.Name, KappControllerAddonName)
+	dataValueSecretName := util.GenerateDataValueSecretName(kappControllerConfig.Name, constants.KappControllerAddonName)
 	kappControllerConfig.Status.SecretRef = dataValueSecretName
 
 	return nil
@@ -173,7 +171,7 @@ func (r *KappControllerConfigReconciler) ReconcileKappControllerConfigDataValue(
 
 	dataValuesSecret := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      util.GenerateDataValueSecretName(kappControllerConfig.Name, KappControllerAddonName),
+			Name:      util.GenerateDataValueSecretName(kappControllerConfig.Name, constants.KappControllerAddonName),
 			Namespace: cluster.Namespace,
 			OwnerReferences: []metav1.OwnerReference{{
 				APIVersion: clusterapiv1beta1.GroupVersion.String(),
@@ -210,7 +208,7 @@ func (r *KappControllerConfigReconciler) ReconcileKappControllerConfigDataValue(
 		return err
 	}
 
-	log.Info(fmt.Sprintf("Resource %s data values secret %s", KappControllerAddonName, result))
+	log.Info(fmt.Sprintf("Resource %s data values secret %s", constants.KappControllerAddonName, result))
 
 	return nil
 }
diff --git a/addons/pkg/config/config.go b/addons/pkg/config/config.go
index b93745434d..0c34c9843c 100644
--- a/addons/pkg/config/config.go
+++ b/addons/pkg/config/config.go
@@ -28,4 +28,14 @@ type ClusterBootstrapControllerConfig struct {
 	NoProxyClusterClassVarName      string
 	ProxyCACertClusterClassVarName  string
 	IPFamilyClusterClassVarName     string
+	// The length of time to wait before kapp-controller's reconciliation
+	PkgiSyncPeriod time.Duration
+	// ServiceAccount name that will be used by kapp-controller to install underlying package contents
+	PkgiServiceAccount string
+	// The name that will be used to create ClusterRole contains all required rules for PkgiServiceAccount
+	PkgiClusterRole string
+	// The name of ClusterRoleBinding that will be used to bind PkgiClusterRole and PkgiServiceAccount
+	PkgiClusterRoleBinding string
+	// The namespace where the bootstrap objects will be created, i.e., tkg-system
+	SystemNamespace string
 }
diff --git a/addons/pkg/constants/constants.go b/addons/pkg/constants/constants.go
index 9caa9d3a6f..469373b26b 100644
--- a/addons/pkg/constants/constants.go
+++ b/addons/pkg/constants/constants.go
@@ -127,6 +127,13 @@ const (
 	// InfrastructureRefDocker is the docker infrastructure
 	InfrastructureRefDocker = "DockerCluster"
 
+	// KappControllerAddonName is the addon name of Kapp Controller
+	KappControllerAddonName = "kapp-controller"
+
+	// ManagementClusterRoleLabel is the label applied to cluster object that indicates the cluster object represents a
+	// mgmt cluster
+	ManagementClusterRoleLabel = "cluster-role.tkg.tanzu.vmware.com/management"
+
 	// SecretNameLogKey is the log key for Secrets
 	SecretNameLogKey = "secret-name"
 
diff --git a/addons/pkg/util/addon_util.go b/addons/pkg/util/addon_util.go
index 0283df918a..52c00ba4c0 100644
--- a/addons/pkg/util/addon_util.go
+++ b/addons/pkg/util/addon_util.go
@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"net"
 	"strconv"
+	"strings"
 
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v2"
@@ -18,6 +19,7 @@ import (
 
 	kappctrl "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1"
 	pkgiv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/packaging/v1alpha1"
+	kapppkgv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/apis/datapackaging/v1alpha1"
 	"github.com/vmware-tanzu/tanzu-framework/addons/pkg/constants"
 	addontypes "github.com/vmware-tanzu/tanzu-framework/addons/pkg/types"
 	infraconstants "github.com/vmware-tanzu/tanzu-framework/pkg/v1/tkg/constants"
@@ -67,14 +69,45 @@ func GenerateAppNameFromAddonSecret(addonSecret *corev1.Secret) string {
 	return addonName
 }
 
+// GeneratePackageInstallName is the util function to generate the PackageInstall CR name in a consistent manner.
+// clusterName is the name of cluster within which all resources associated with this PackageInstall CR is installed.
+// It does not necessarily
+// mean the PackageInstall CR will be installed in that cluster. I.e., the kapp-controller PackageInstall CR is installed
+// in the management cluster but is named after "<workload-cluster-name>-kapp-controller". It indicates that this kapp-controller
+// PackageInstall is for reconciling resources in a cluster named "<workload-cluster-name>".
+// addonName is the short name of a Tanzu addon with which the PackageInstall CR is associated.
+func GeneratePackageInstallName(clusterName, addonName string) string {
+	return fmt.Sprintf("%s-%s", clusterName, strings.Split(addonName, ".")[0])
+}
+
+func GetPackageMetadata(ctx context.Context, c client.Client, carvelPkgName, carvelPkgNamespace string) (string, string, error) {
+	pkg := &kapppkgv1alpha1.Package{}
+	if err := c.Get(ctx, client.ObjectKey{Name: carvelPkgName, Namespace: carvelPkgNamespace}, pkg); err != nil {
+		return "", "", err
+	}
+	return pkg.Spec.RefName, pkg.Spec.Version, nil
+}
+
+// ParseStringForLabel parse the package ref name to make it valid for K8S object labels.
+// A package ref name could contain some characters that are not allowed as a label value. The regex
+// used for validation is (([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?
+func ParseStringForLabel(s string) string {
+	// Replace + sign with ---
+	safeLabel := strings.ReplaceAll(s, "+", "---")
+	if len(safeLabel) <= 63 {
+		return safeLabel
+	}
+	return safeLabel[:63]
+}
+
 // GenerateAppSecretNameFromAddonSecret generates app secret name from addon secret
 func GenerateAppSecretNameFromAddonSecret(addonSecret *corev1.Secret) string {
 	return fmt.Sprintf("%s-data-values", GenerateAppNameFromAddonSecret(addonSecret))
 }
 
 // GenerateDataValueSecretName generates data value secret name from the cluster and the package name
-func GenerateDataValueSecretName(clusterName, pkgName string) string {
-	return fmt.Sprintf("%s-%s-data-values", clusterName, pkgName)
+func GenerateDataValueSecretName(clusterName, carvelPkgRefName string) string {
+	return fmt.Sprintf("%s-%s-data-values", clusterName, carvelPkgRefName)
 }
 
 // GenerateAppNamespaceFromAddonSecret generates app namespace from addons secret


Test patch:
diff --git a/addons/controllers/clusterbootstrap_controller_test.go b/addons/controllers/clusterbootstrap_controller_test.go
index 25522e6d1a..19b92adf80 100644
--- a/addons/controllers/clusterbootstrap_controller_test.go
+++ b/addons/controllers/clusterbootstrap_controller_test.go
@@ -6,29 +6,45 @@ package controllers
 import (
 	"fmt"
 	"os"
+	"strings"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	v1 "k8s.io/api/core/v1"
+
+	kapppkgiv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/packaging/v1alpha1"
+	kapppkgv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/apis/datapackaging/v1alpha1"
+	"github.com/vmware-tanzu/tanzu-framework/addons/pkg/constants"
+	addontypes "github.com/vmware-tanzu/tanzu-framework/addons/pkg/types"
+	"github.com/vmware-tanzu/tanzu-framework/addons/pkg/util"
+	"github.com/vmware-tanzu/tanzu-framework/addons/testutil"
+	runtanzuv1alpha3 "github.com/vmware-tanzu/tanzu-framework/apis/run/v1alpha3"
+
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	clusterapiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	clusterapiutil "sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/secret"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	"github.com/vmware-tanzu/tanzu-framework/addons/pkg/constants"
-	addontypes "github.com/vmware-tanzu/tanzu-framework/addons/pkg/types"
-	"github.com/vmware-tanzu/tanzu-framework/addons/testutil"
-	runtanzuv1alpha3 "github.com/vmware-tanzu/tanzu-framework/apis/run/v1alpha3"
 )
 
 var _ = Describe("ClusterBootstrap Reconciler", func() {
 	var (
 		clusterName             string
+		clusterNamespace        string
 		clusterResourceFilePath string
 	)
 
+	// Constants defined in testdata manifests
+	const (
+		foobarCarvelPackageRefName  = "foobar.example.com"
+		foobarCarvelPackageName     = "foobar.example.com.1.17.2"
+		foobar1CarvelPackageRefName = "foobar1.example.com"
+		foobar1CarvelPackageName    = "foobar1.example.com.1.17.2"
+	)
+
 	JustBeforeEach(func() {
 		// create cluster resources
 		By("Creating a cluster")
@@ -38,298 +54,386 @@ var _ = Describe("ClusterBootstrap Reconciler", func() {
 		Expect(testutil.CreateResources(f, cfg, dynamicClient)).To(Succeed())
 
 		By("Creating kubeconfig for cluster")
-		Expect(testutil.CreateKubeconfigSecret(cfg, clusterName, "default", k8sClient)).To(Succeed())
+		Expect(testutil.CreateKubeconfigSecret(cfg, clusterName, clusterNamespace, k8sClient)).To(Succeed())
 	})
 
 	AfterEach(func() {
 		By("Deleting kubeconfig for cluster")
 		key := client.ObjectKey{
-			Namespace: "default",
+			Namespace: clusterNamespace,
 			Name:      secret.Name(clusterName, secret.Kubeconfig),
 		}
-		s := &v1.Secret{}
+		s := &corev1.Secret{}
 		Expect(k8sClient.Get(ctx, key, s)).To(Succeed())
 		Expect(k8sClient.Delete(ctx, s)).To(Succeed())
 	})
 
-	Context("reconcileClusterBootstrapNormal", func() {
-
+	When("cluster is created with topology", func() {
 		BeforeEach(func() {
 			clusterName = "test-cluster-tcbt"
+			clusterNamespace = "cluster-namespace"
 			clusterResourceFilePath = "testdata/test-cluster-bootstrap-1.yaml"
 		})
-
-		It("Should create clone ClusterBootstrapTemplate and its related objects for the cluster, create package "+
-			"secret for foobar.example.com.1.17.2 and set CNI to the one specified using cluster variable", func() {
-			cluster := &clusterapiv1beta1.Cluster{}
-			Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: "default", Name: clusterName}, cluster)).To(Succeed())
-
-			clusterBootstrap := &runtanzuv1alpha3.ClusterBootstrap{}
-
-			// Verify ownerReference for cluster in cloned object
-			Eventually(func() bool {
-				err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), clusterBootstrap)
-				if err != nil {
-					return false
-				}
-				for _, ownerRef := range clusterBootstrap.OwnerReferences {
-					if ownerRef.UID == cluster.UID {
-						return true
+		Context("from a ClusterBootstrapTemplate", func() {
+			It("should create ClusterBootstrap CR and the related objects for the cluster", func() {
+
+				By("verifying CAPI cluster is created properly")
+				cluster := &clusterapiv1beta1.Cluster{}
+				Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: clusterNamespace, Name: clusterName}, cluster)).To(Succeed())
+				copiedCluster := cluster.DeepCopy()
+				copiedCluster.Status.Phase = string(clusterapiv1beta1.ClusterPhaseProvisioned)
+				Expect(k8sClient.Status().Update(ctx, copiedCluster)).To(Succeed())
+
+				By("ClusterBootstrap CR is created with correct ownerReference added")
+				clusterBootstrap := &runtanzuv1alpha3.ClusterBootstrap{}
+				// Verify ownerReference for cluster in cloned object
+				Eventually(func() bool {
+					err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), clusterBootstrap)
+					if err != nil {
+						return false
+					}
+					for _, ownerRef := range clusterBootstrap.OwnerReferences {
+						if ownerRef.UID == cluster.UID {
+							return true
+						}
 					}
-				}
-				return false
-			}, waitTimeout, pollingInterval).Should(BeTrue())
-
-			// Verify CNI is populated in the cloned object with the value from the cluster variables definitions
-			Expect(len(clusterBootstrap.Spec.CNIs)).NotTo(BeZero())
-			cni := clusterBootstrap.Spec.CNIs[0]
-			Expect(packageShortName(cni.RefName)).To(Equal("antrea"))
-
-			Expect(cni.RefName).To(Equal("antrea.tanzu.vmware.com.21.2.0--vmware.1-tkg.1-rc.1"))
-			Expect(*cni.ValuesFrom.ProviderRef.APIGroup).To(Equal("cni.tanzu.vmware.com"))
-			Expect(cni.ValuesFrom.ProviderRef.Kind).To(Equal("AntreaConfig"))
-			providerName := fmt.Sprintf("%s-antrea-package", clusterName)
-			Expect(cni.ValuesFrom.ProviderRef.Name).To(Equal(providerName))
-
-			// Verify ResolvedTKR
-			Eventually(func() bool {
-				err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), clusterBootstrap)
-				if err != nil {
 					return false
-				}
-				if clusterBootstrap.Status.ResolvedTKR == "v1.22.3" {
-					return true
-				}
-
-				return false
-			}, waitTimeout, pollingInterval).Should(BeTrue())
+				}, waitTimeout, pollingInterval).Should(BeTrue())
 
-			// Verify Proxy Configurations
-			Eventually(func() bool {
-				err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), clusterBootstrap)
-				if err != nil {
-					return false
-				}
+				By("verifying that ClusterBootstrap CR has status resolved to correct TKR")
+				// Verify ResolvedTKR
+				Eventually(func() bool {
+					err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), clusterBootstrap)
+					if err != nil {
+						return false
+					}
+					if clusterBootstrap.Status.ResolvedTKR == "v1.22.3" {
+						return true
+					}
 
-				err = k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster)
-				if err != nil {
 					return false
-				}
-
-				if cluster.Annotations != nil &&
-					cluster.Annotations[addontypes.HTTPProxyConfigAnnotation] == "foo.com" &&
-					cluster.Annotations[addontypes.HTTPSProxyConfigAnnotation] == "bar.com" &&
-					cluster.Annotations[addontypes.NoProxyConfigAnnotation] == "foobar.com" &&
-					cluster.Annotations[addontypes.ProxyCACertConfigAnnotation] == "dummyCertificate" &&
-					cluster.Annotations[addontypes.IPFamilyConfigAnnotation] == "ipv4" {
-					return true
-				}
-
-				return false
-			}, waitTimeout, pollingInterval).Should(BeTrue())
-
-			var gvr schema.GroupVersionResource
-			var object *unstructured.Unstructured
-			var fooPackage *runtanzuv1alpha3.ClusterBootstrapPackage
-			var foobar1Package *runtanzuv1alpha3.ClusterBootstrapPackage
+				}, waitTimeout, pollingInterval).Should(BeTrue())
 
-			// Verify providerRef exists and also the cloned provider object with ownerReferences to cluster and ClusterBootstrap
-			Eventually(func() bool {
-				Expect(len(clusterBootstrap.Spec.AdditionalPackages) > 1).To(BeTrue())
+				By("verifying that CNI has been populated properly")
+				// Verify CNI is populated in the cloned object with the value from the cluster variables definitions
+				Expect(len(clusterBootstrap.Spec.CNIs)).NotTo(BeZero())
+				cni := clusterBootstrap.Spec.CNIs[0]
+				Expect(strings.HasPrefix(cni.RefName, "antrea")).To(BeTrue())
 
-				fooPackage = clusterBootstrap.Spec.AdditionalPackages[1]
-				Expect(fooPackage.RefName == "foobar.example.com.1.17.2").To(BeTrue())
-				Expect(*fooPackage.ValuesFrom.ProviderRef.APIGroup == "run.tanzu.vmware.com").To(BeTrue())
-				Expect(fooPackage.ValuesFrom.ProviderRef.Kind == "FooBar").To(BeTrue())
+				Expect(cni.RefName).To(Equal("antrea.tanzu.vmware.com.1.2.3--vmware.1-tkg.1"))
+				Expect(*cni.ValuesFrom.ProviderRef.APIGroup).To(Equal("cni.tanzu.vmware.com"))
+				Expect(cni.ValuesFrom.ProviderRef.Kind).To(Equal("AntreaConfig"))
+				providerName := fmt.Sprintf("%s-antrea.tanzu.vmware.com-package", clusterName)
+				Expect(cni.ValuesFrom.ProviderRef.Name).To(Equal(providerName))
 
-				providerName := fmt.Sprintf("%s-foobar-package", clusterName)
-				Expect(fooPackage.ValuesFrom.ProviderRef.Name == providerName).To(BeTrue())
+				By("verifying that the proxy related annotations are populated to cluster object properly")
+				Eventually(func() bool {
+					err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster)
+					if err != nil {
+						return false
+					}
+					if cluster.Annotations != nil &&
+						cluster.Annotations[addontypes.HTTPProxyConfigAnnotation] == "foo.com" &&
+						cluster.Annotations[addontypes.HTTPSProxyConfigAnnotation] == "bar.com" &&
+						cluster.Annotations[addontypes.NoProxyConfigAnnotation] == "foobar.com" &&
+						cluster.Annotations[addontypes.ProxyCACertConfigAnnotation] == "dummyCertificate" &&
+						cluster.Annotations[addontypes.IPFamilyConfigAnnotation] == "ipv4" {
+						return true
+					}
+					return false
+				}, waitTimeout, pollingInterval).Should(BeTrue())
 
-				gvr = schema.GroupVersionResource{Group: "run.tanzu.vmware.com", Version: "v1alpha1", Resource: "foobars"}
-				var err error
-				object, err = dynamicClient.Resource(gvr).Namespace("default").Get(ctx, providerName, metav1.GetOptions{})
+				By("verifying that the providerRef from additionalPackages is cloned into cluster namespace and ownerReferences set properly")
+				var gvr schema.GroupVersionResource
+				var object *unstructured.Unstructured
+				// Verify providerRef exists and also the cloned provider object with ownerReferences to cluster and ClusterBootstrap
+				Eventually(func() bool {
+					Expect(len(clusterBootstrap.Spec.AdditionalPackages) > 0).To(BeTrue())
+
+					fooPackage := clusterBootstrap.Spec.AdditionalPackages[1]
+					Expect(fooPackage.RefName == foobarCarvelPackageName).To(BeTrue())
+					Expect(*fooPackage.ValuesFrom.ProviderRef.APIGroup == "run.tanzu.vmware.com").To(BeTrue())
+					Expect(fooPackage.ValuesFrom.ProviderRef.Kind == "FooBar").To(BeTrue())
+
+					providerName := fmt.Sprintf("%s-%s-package", clusterName, foobarCarvelPackageRefName)
+					Expect(fooPackage.ValuesFrom.ProviderRef.Name == providerName).To(BeTrue())
+
+					gvr = schema.GroupVersionResource{Group: "run.tanzu.vmware.com", Version: "v1alpha1", Resource: "foobars"}
+					var err error
+					object, err = dynamicClient.Resource(gvr).Namespace(clusterNamespace).Get(ctx, providerName, metav1.GetOptions{})
+
+					Expect(err).ToNot(HaveOccurred())
+
+					var foundClusterOwnerRef bool
+					var foundClusterBootstrapOwnerRef bool
+					var foundLabels bool
+					for _, ownerRef := range object.GetOwnerReferences() {
+						if ownerRef.UID == cluster.UID {
+							foundClusterOwnerRef = true
+						}
+						if ownerRef.UID == clusterBootstrap.UID {
+							foundClusterBootstrapOwnerRef = true
+						}
+					}
+					providerLabels := object.GetLabels()
+					if providerLabels[addontypes.ClusterNameLabel] == clusterName &&
+						providerLabels[addontypes.PackageNameLabel] == util.ParseStringForLabel(fooPackage.RefName) {
+						foundLabels = true
+					}
 
-				Expect(err).ToNot(HaveOccurred())
+					if foundClusterOwnerRef && foundClusterBootstrapOwnerRef && foundLabels {
+						return true
+					}
+					return false
+				}, waitTimeout, pollingInterval).Should(BeTrue())
 
-				var foundClusterOwnerRef bool
-				var foundClusterBootstrapOwnerRef bool
-				var foundLabels bool
-				for _, ownerRef := range object.GetOwnerReferences() {
-					if ownerRef.UID == cluster.UID {
-						foundClusterOwnerRef = true
+				By("verifying the cloned secret of an additional package which with a secretRef")
+				// Verify that secret object mapping to foobar1 package exists with ownerReferences to cluster and ClusterBootstrap
+				Eventually(func() bool {
+					// "foobar1.example.com" is the carvel package ref name
+					foobar1SecretName := fmt.Sprintf("%s-%s-package", cluster.Name, foobar1CarvelPackageRefName)
+					foobar1Secret := &corev1.Secret{}
+					if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: clusterNamespace, Name: foobar1SecretName}, foobar1Secret); err != nil {
+						return false
 					}
-					if ownerRef.UID == clusterBootstrap.UID {
-						foundClusterBootstrapOwnerRef = true
+					var foundClusterOwnerRef bool
+					var foundClusterBootstrapOwnerRef bool
+					var foundLabels bool
+					var foundType bool
+					for _, ownerRef := range foobar1Secret.GetOwnerReferences() {
+						if ownerRef.UID == cluster.UID {
+							foundClusterOwnerRef = true
+						}
+						if ownerRef.UID == clusterBootstrap.UID {
+							foundClusterBootstrapOwnerRef = true
+						}
 					}
-				}
-				providerLabels := object.GetLabels()
-				if providerLabels[addontypes.ClusterNameLabel] == clusterName &&
-					providerLabels[addontypes.PackageNameLabel] == fooPackage.RefName {
-					foundLabels = true
-				}
-
-				if foundClusterOwnerRef && foundClusterBootstrapOwnerRef && foundLabels {
-					return true
-				}
-				return false
-			}, waitTimeout, pollingInterval).Should(BeTrue())
-
-			// Verify that secret object mapping to foobar1 package exists with ownerReferences to cluster and ClusterBootstrap
-			Eventually(func() bool {
-				foobar1Package = clusterBootstrap.Spec.AdditionalPackages[0]
-
-				Expect(foobar1Package.RefName == "foobar1.example.com.1.17.2").To(BeTrue())
-				secretName := fmt.Sprintf("%s-foobar1-package", clusterName)
-				Expect(foobar1Package.ValuesFrom.SecretRef == secretName).To(BeTrue())
-
-				s := &v1.Secret{}
-				Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: "default", Name: secretName}, s)).To(Succeed())
-
-				var foundClusterOwnerRef bool
-				var foundClusterBootstrapOwnerRef bool
-				var foundLabels bool
-				var foundType bool
-				for _, ownerRef := range s.GetOwnerReferences() {
-					if ownerRef.UID == cluster.UID {
-						foundClusterOwnerRef = true
+					secretLabels := foobar1Secret.GetLabels()
+					if secretLabels[addontypes.ClusterNameLabel] == clusterName &&
+						secretLabels[addontypes.PackageNameLabel] == util.ParseStringForLabel(foobar1CarvelPackageName) {
+						// "foobar1.example.com.1.17.2" is clusterBootstrap.additionalPackages[0].refName
+						foundLabels = true
 					}
-					if ownerRef.UID == clusterBootstrap.UID {
-						foundClusterBootstrapOwnerRef = true
+					if foobar1Secret.Type == constants.ClusterBootstrapManagedSecret {
+						foundType = true
 					}
-				}
-				secretLabels := s.GetLabels()
-				if secretLabels[addontypes.ClusterNameLabel] == clusterName &&
-					secretLabels[addontypes.PackageNameLabel] == foobar1Package.RefName {
-					foundLabels = true
-				}
-				if s.Type == constants.ClusterBootstrapManagedSecret {
-					foundType = true
-				}
-				if foundClusterOwnerRef && foundClusterBootstrapOwnerRef && foundLabels && foundType {
-					return true
-				}
-
-				return false
-			}, waitTimeout, pollingInterval).Should(BeTrue())
+					if foundClusterOwnerRef && foundClusterBootstrapOwnerRef && foundLabels && foundType {
+						return true
+					}
+					return false
+				}, waitTimeout, pollingInterval).Should(BeTrue())
 
-			// Simulate a controller adding secretRef to provider status and
-			// verify that a data-values secret has been created for the package
-			By("patching foobar status resource with secret", func() {
-				s := &v1.Secret{}
-				s.Name = "foobar-data-values"
-				s.Namespace = "default"
-				s.Data = map[string][]byte{}
-				s.Data["values.yaml"] = []byte("foobar")
-				Expect(k8sClient.Create(ctx, s)).To(Succeed())
+				// Update the secret created before for foobar1
+				// Verify that the updated secret leads to an updated data-values secret
+				By("Updating the foobar1 package secret", func() {
+					// In test setup, foobar1 is an additional package with a secretRef as valuesFrom. The referenced secret
+					// is cloned into cluster namespace on mgmt cluster, and eventually get mirrored on workload cluster
+					// under tkg-system namespace.
+					// Any updates to the data value secret under cluster namespace should be eventually reflected on the
+					// workload cluster under tkg-system namespace.
+					s := &corev1.Secret{}
+					Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: clusterNamespace, Name: fmt.Sprintf("%s-%s-package", cluster.Name, foobar1CarvelPackageRefName)}, s)).To(Succeed())
+					s.Data["values.yaml"] = []byte("foobar1-updated")
+					Expect(k8sClient.Update(ctx, s)).To(Succeed())
+
+					Eventually(func() bool {
+						s := &corev1.Secret{}
+						if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: constants.TKGSystemNS, Name: util.GenerateDataValueSecretName(clusterName, foobar1CarvelPackageRefName)}, s); err != nil {
+							return false
+						}
+						if string(s.Data["values.yaml"]) != "foobar1-updated" {
+							return false
+						}
 
-				Expect(unstructured.SetNestedField(object.Object, s.Name, "status", "secretRef")).To(Succeed())
+						return true
+					}, waitTimeout, pollingInterval).Should(BeTrue())
+				})
+
+				// Simulate a controller adding secretRef to provider status and
+				// verify that a data-values secret has been created for the Foobar package
+				By("patching foobar provider object's status resource with a secret ref", func() {
+					s := &corev1.Secret{}
+					s.Name = util.GenerateDataValueSecretName(clusterName, foobarCarvelPackageRefName)
+					s.Namespace = clusterNamespace
+					s.Data = map[string][]byte{}
+					s.Data["values.yaml"] = []byte("foobar")
+					Expect(k8sClient.Create(ctx, s)).To(Succeed())
+
+					Expect(unstructured.SetNestedField(object.Object, s.Name, "status", "secretRef")).To(Succeed())
+
+					_, err := dynamicClient.Resource(gvr).Namespace(clusterNamespace).UpdateStatus(ctx, object, metav1.UpdateOptions{})
+					Expect(err).ToNot(HaveOccurred())
+
+					Eventually(func() bool {
+						s := &corev1.Secret{}
+						if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: constants.TKGSystemNS, Name: util.GenerateDataValueSecretName(clusterName, foobarCarvelPackageRefName)}, s); err != nil {
+							return false
+						}
+						if string(s.Data["values.yaml"]) != "foobar" {
+							return false
+						}
 
-				_, err := dynamicClient.Resource(gvr).Namespace("default").UpdateStatus(ctx, object, metav1.UpdateOptions{})
-				Expect(err).ToNot(HaveOccurred())
+						return true
+					}, waitTimeout, pollingInterval).Should(BeTrue())
+				})
 
+				By("verifying that kapp-controller PackageInstall CR is created under cluster namespace properly on the management cluster")
+				// Verify kapp-controller PackageInstall CR has been created under cluster namespace on management cluster
+				kappControllerPkgi := &kapppkgiv1alpha1.PackageInstall{}
 				Eventually(func() bool {
-					s := &v1.Secret{}
-					if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "tkg-system", Name: fmt.Sprintf("%s-foobar-data-values", clusterName)}, s); err != nil {
+					if err := k8sClient.Get(ctx,
+						client.ObjectKey{
+							Namespace: clusterNamespace,
+							Name:      util.GeneratePackageInstallName(clusterName, "kapp-controller.tanzu.vmware.com"),
+						}, kappControllerPkgi); err != nil {
 						return false
 					}
-					if string(s.Data["values.yaml"]) != "foobar" {
-						return false
-					}
-
 					return true
 				}, waitTimeout, pollingInterval).Should(BeTrue())
-			})
+				Expect(len(kappControllerPkgi.OwnerReferences) == 1).To(BeTrue())
+				Expect(kappControllerPkgi.OwnerReferences[0].APIVersion).To(Equal(clusterapiv1beta1.GroupVersion.String()))
+				Expect(kappControllerPkgi.OwnerReferences[0].Name).To(Equal(cluster.Name))
 
-			// Update the secret created before for foobar resource
-			// Verify that the updated secret leads to an updated data-values secret
-			By("Updating the foobar package provider secret", func() {
-				s := &v1.Secret{}
-				Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: "default", Name: "foobar-data-values"}, s)).To(Succeed())
-				s.OwnerReferences = []metav1.OwnerReference{
-					{
-						APIVersion: clusterapiv1beta1.GroupVersion.String(),
-						Kind:       "Cluster",
-						Name:       cluster.Name,
-						UID:        cluster.UID,
-					},
-				}
-				s.Data["values.yaml"] = []byte("foobar-updated")
-				Expect(k8sClient.Update(ctx, s)).To(Succeed())
+				remoteClient, err := util.GetClusterClient(ctx, k8sClient, scheme, clusterapiutil.ObjectKey(cluster))
+				Expect(err).NotTo(HaveOccurred())
+				Expect(remoteClient).NotTo(BeNil())
 
+				By("verifying that ServiceAccount, ClusterRole and ClusterRoleBinding are created on the workload cluster properly")
+				sa := &corev1.ServiceAccount{}
 				Eventually(func() bool {
-					s := &v1.Secret{}
-					if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "tkg-system", Name: fmt.Sprintf("%s-foobar-data-values", clusterName)}, s); err != nil {
+					if err := remoteClient.Get(ctx,
+						client.ObjectKey{Namespace: constants.TKGSystemNS, Name: "tanzu-addons-manager-sa"},
+						sa); err != nil {
 						return false
 					}
-					if string(s.Data["values.yaml"]) != "foobar-updated" {
+					return true
+				}, waitTimeout, pollingInterval).Should(BeTrue())
+				clusterRole := &rbacv1.ClusterRole{}
+				Eventually(func() bool {
+					if err := remoteClient.Get(ctx,
+						client.ObjectKey{Name: "tanzu-addons-manager-clusterrole"},
+						clusterRole); err != nil {
 						return false
 					}
-
 					return true
 				}, waitTimeout, pollingInterval).Should(BeTrue())
-			})
-
-			// Update the secret created before for foobar1
-			// Verify that the updated secret leads to an updated data-values secret
-			By("Updating the foobar1 package secret", func() {
-				s := &v1.Secret{}
-				secretName := fmt.Sprintf("%s-foobar1-package", clusterName)
-				Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: "default", Name: secretName}, s)).To(Succeed())
-				s.Data["values.yaml"] = []byte("foobar1-updated")
-				Expect(k8sClient.Update(ctx, s)).To(Succeed())
-
+				clusterRoleBinding := &rbacv1.ClusterRoleBinding{}
 				Eventually(func() bool {
-					s := &v1.Secret{}
-					if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "tkg-system", Name: fmt.Sprintf("%s-foobar1-data-values", clusterName)}, s); err != nil {
+					if err := remoteClient.Get(ctx,
+						client.ObjectKey{Name: "tanzu-addons-manager-clusterrolebinding"},
+						clusterRoleBinding); err != nil {
 						return false
 					}
-					if string(s.Data["values.yaml"]) != "foobar1-updated" {
-						return false
+					return true
+				}, waitTimeout, pollingInterval).Should(BeTrue())
+
+				By("verifying that Package CRs of additionalPackages are created on the workload cluster properly")
+				pkg := &kapppkgv1alpha1.Package{}
+				pkgRefNameMap := make(map[string]string)
+				Eventually(func() bool {
+					for _, clusterBootstrapPackage := range clusterBootstrap.Spec.AdditionalPackages {
+						if err := remoteClient.Get(ctx,
+							client.ObjectKey{Namespace: constants.TKGSystemNS, Name: clusterBootstrapPackage.RefName},
+							pkg); err != nil {
+							return false
+						}
+						pkgRefNameMap[clusterBootstrapPackage.RefName] = pkg.Spec.RefName
+					}
+					return true
+				}, waitTimeout, pollingInterval).Should(BeTrue())
+
+				By("verifying that the data value secret is created on the workload cluster properly")
+				remoteSecret := &corev1.Secret{}
+				Eventually(func() bool {
+					for _, clusterBootstrapPackage := range clusterBootstrap.Spec.AdditionalPackages {
+						if err := remoteClient.Get(ctx,
+							client.ObjectKey{
+								Namespace: constants.TKGSystemNS,
+								Name:      util.GenerateDataValueSecretName(cluster.Name, pkgRefNameMap[clusterBootstrapPackage.RefName]),
+							}, remoteSecret); err != nil {
+							return false
+						}
 					}
+					return true
+				}, waitTimeout, pollingInterval).Should(BeTrue())
 
+				By("verifying that the PackageInstall CRs are created on the workload cluster properly")
+				remotePkgi := &kapppkgiv1alpha1.PackageInstall{}
+				Eventually(func() bool {
+					for _, clusterBootstrapPackage := range clusterBootstrap.Spec.AdditionalPackages {
+						if err := remoteClient.Get(ctx,
+							client.ObjectKey{
+								Namespace: constants.TKGSystemNS,
+								Name:      util.GeneratePackageInstallName(clusterName, pkgRefNameMap[clusterBootstrapPackage.RefName]),
+							}, remotePkgi); err != nil {
+							return false
+						}
+					}
 					return true
 				}, waitTimeout, pollingInterval).Should(BeTrue())
+				Expect(remotePkgi.Spec.PackageRef.RefName).To(Equal(pkg.Spec.RefName))
+				Expect(len(remotePkgi.Spec.Values)).NotTo(BeZero())
+				Expect(remotePkgi.Spec.Values[0].SecretRef.Name).To(Equal(util.GenerateDataValueSecretName(cluster.Name, pkg.Spec.RefName)))
+
+				// Update the secret created before for foobar resource
+				// Verify that the updated secret leads to an updated data-values secret
+				By("Updating the foobar package provider secret", func() {
+					s := &corev1.Secret{}
+					Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: clusterNamespace, Name: util.GenerateDataValueSecretName(clusterName, foobarCarvelPackageRefName)}, s)).To(Succeed())
+					s.OwnerReferences = []metav1.OwnerReference{
+						{
+							APIVersion: clusterapiv1beta1.GroupVersion.String(),
+							Kind:       "Cluster",
+							Name:       cluster.Name,
+							UID:        cluster.UID,
+						},
+					}
+					s.Data["values.yaml"] = []byte("foobar-updated")
+					Expect(k8sClient.Update(ctx, s)).To(Succeed())
+
+					Eventually(func() bool {
+						s := &corev1.Secret{}
+						if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: constants.TKGSystemNS, Name: util.GenerateDataValueSecretName(clusterName, foobarCarvelPackageRefName)}, s); err != nil {
+							return false
+						}
+						if string(s.Data["values.yaml"]) != "foobar-updated" {
+							return false
+						}
+
+						return true
+					}, waitTimeout, pollingInterval).Should(BeTrue())
+				})
 			})
 		})
 	})
 
-	Context("reconcileClusterBootstrapNormal, CNI selection in case of no CNI cluster variable", func() {
-
+	When("cluster is created without topology", func() {
 		BeforeEach(func() {
 			clusterName = "test-cluster-tcbt-2"
+			clusterNamespace = "cluster-namespace-2"
 			clusterResourceFilePath = "testdata/test-cluster-bootstrap-2.yaml"
 		})
-
-		It("Should create cloned ClusterBootstrap for the cluster and set CNI to the first entry in the template (as cluster variable for CNI is not set)", func() {
-			cluster := &clusterapiv1beta1.Cluster{}
-			Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: "default", Name: clusterName}, cluster)).To(Succeed())
-
-			clusterBootstrap := &runtanzuv1alpha3.ClusterBootstrap{}
-
-			// Verify ownerReference for cluster in cloned object
-			Eventually(func() bool {
-				err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), clusterBootstrap)
-				if err != nil {
-					return false
-				}
-				for _, ownerRef := range clusterBootstrap.OwnerReferences {
-					if ownerRef.UID == cluster.UID {
-						return true
-					}
-				}
-				return false
-			}, waitTimeout, pollingInterval).Should(BeTrue())
-
-			// Verify CNI is populated in the cloned object with the the first entry in the template (as cluster variable for CNI is not set)
-			Expect(len(clusterBootstrap.Spec.CNIs)).To(Equal(1))
-			cni := clusterBootstrap.Spec.CNIs[0]
-			Expect(packageShortName(cni.RefName)).To(Equal("calico"))
-
-			Expect(cni.RefName).To(Equal("calico.tanzu.vmware.com.0.3.0--vmware.1-tkg.1-rc.1"))
-			Expect(*cni.ValuesFrom.ProviderRef.APIGroup).To(Equal("cni.tanzu.vmware.com"))
-			Expect(cni.ValuesFrom.ProviderRef.Kind).To(Equal("CalicoConfig"))
-			providerName := fmt.Sprintf("%s-calico-package", clusterName)
-			Expect(cni.ValuesFrom.ProviderRef.Name).To(Equal(providerName))
+		Context("from a ClusterBootstrapTemplate", func() {
+			It("should set CNI to the first entry in the template (as cluster variable for CNI is not set)", func() {
+				cluster := &clusterapiv1beta1.Cluster{}
+				Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: clusterNamespace, Name: clusterName}, cluster)).To(Succeed())
+				clusterBootstrap := &runtanzuv1alpha3.ClusterBootstrap{}
+				Eventually(func() bool {
+					err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), clusterBootstrap)
+					return err == nil
+				}, waitTimeout, pollingInterval).Should(BeTrue())
+				// Verify CNI is populated in the cloned object with the first entry in the template (as cluster variable for CNI is not set)
+				Expect(len(clusterBootstrap.Spec.CNIs)).To(Equal(1))
+				cni := clusterBootstrap.Spec.CNIs[0]
+				Expect(strings.HasPrefix(cni.RefName, "calico")).To(BeTrue())
+
+				Expect(cni.RefName).To(Equal("calico.tanzu.vmware.com.3.19.1--vmware.1-tkg.1"))
+				Expect(*cni.ValuesFrom.ProviderRef.APIGroup).To(Equal("cni.tanzu.vmware.com"))
+				Expect(cni.ValuesFrom.ProviderRef.Kind).To(Equal("CalicoConfig"))
+				providerName := fmt.Sprintf("%s-calico.tanzu.vmware.com-package", clusterName)
+				Expect(cni.ValuesFrom.ProviderRef.Name).To(Equal(providerName))
+			})
 		})
 	})
+
 })
diff --git a/addons/controllers/kappcontrollerconfig_controller_test.go b/addons/controllers/kappcontrollerconfig_controller_test.go
index 2b6d9e3833..818ebf346e 100644
--- a/addons/controllers/kappcontrollerconfig_controller_test.go
+++ b/addons/controllers/kappcontrollerconfig_controller_test.go
@@ -9,11 +9,12 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	v1 "k8s.io/api/core/v1"
 	clusterapiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	kappcontroller "github.com/vmware-tanzu/tanzu-framework/addons/controllers/kapp-controller"
+	"github.com/vmware-tanzu/tanzu-framework/addons/pkg/constants"
 	"github.com/vmware-tanzu/tanzu-framework/addons/pkg/util"
 	"github.com/vmware-tanzu/tanzu-framework/addons/testutil"
 	runv1alpha3 "github.com/vmware-tanzu/tanzu-framework/apis/run/v1alpha3"
@@ -93,7 +94,7 @@ var _ = Describe("KappControllerConfig Reconciler", func() {
 			Eventually(func() bool {
 				secretKey := client.ObjectKey{
 					Namespace: "default",
-					Name:      util.GenerateDataValueSecretName(kappConfigCRName, kappcontroller.KappControllerAddonName),
+					Name:      util.GenerateDataValueSecretName(kappConfigCRName, constants.KappControllerAddonName),
 				}
 				secret := &v1.Secret{}
 				err := k8sClient.Get(ctx, secretKey, secret)
@@ -137,7 +138,7 @@ var _ = Describe("KappControllerConfig Reconciler", func() {
 				if err != nil {
 					return false
 				}
-				Expect(config.Status.SecretRef).Should(Equal(util.GenerateDataValueSecretName(kappConfigCRName, kappcontroller.KappControllerAddonName)))
+				Expect(config.Status.SecretRef).Should(Equal(util.GenerateDataValueSecretName(kappConfigCRName, constants.KappControllerAddonName)))
 				return true
 			}, waitTimeout, pollingInterval).Should(BeTrue())
 
diff --git a/addons/controllers/suite_test.go b/addons/controllers/suite_test.go
index 7da8ac5a9c..9616b3a98e 100644
--- a/addons/controllers/suite_test.go
+++ b/addons/controllers/suite_test.go
@@ -32,6 +32,7 @@ import (
 
 	kappctrl "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1"
 	pkgiv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/packaging/v1alpha1"
+	kapppkgv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/apis/datapackaging/v1alpha1"
 	antrea "github.com/vmware-tanzu/tanzu-framework/addons/controllers/antrea"
 	calico "github.com/vmware-tanzu/tanzu-framework/addons/controllers/calico"
 	kappcontroller "github.com/vmware-tanzu/tanzu-framework/addons/controllers/kapp-controller"
@@ -127,6 +128,9 @@ var _ = BeforeSuite(func(done Done) {
 	err = pkgiv1alpha1.AddToScheme(scheme)
 	Expect(err).NotTo(HaveOccurred())
 
+	err = kapppkgv1alpha1.AddToScheme(scheme)
+	Expect(err).NotTo(HaveOccurred())
+
 	err = cniv1alpha1.AddToScheme(scheme)
 	Expect(err).NotTo(HaveOccurred())
 
@@ -205,7 +209,8 @@ var _ = BeforeSuite(func(done Done) {
 		Scheme: mgr.GetScheme(),
 	}).SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: 1})).To(Succeed())
 
-	bootstrapReconciler := NewClusterBootstrapReconciler(mgr.GetClient(),
+	bootstrapReconciler := NewClusterBootstrapReconciler(
+		mgr.GetClient(),
 		ctrl.Log.WithName("controllers").WithName("ClusterBootstrap"),
 		mgr.GetScheme(),
 		&addonconfig.ClusterBootstrapControllerConfig{
@@ -215,6 +220,11 @@ var _ = BeforeSuite(func(done Done) {
 			NoProxyClusterClassVarName:      constants.DefaultNoProxyClusterClassVarName,
 			ProxyCACertClusterClassVarName:  constants.DefaultProxyCaCertClusterClassVarName,
 			IPFamilyClusterClassVarName:     constants.DefaultIPFamilyClusterClassVarName,
+			SystemNamespace:                 constants.TKGSystemNS,
+			PkgiServiceAccount:              "tanzu-addons-manager-sa",
+			PkgiClusterRole:                 "tanzu-addons-manager-clusterrole",
+			PkgiClusterRoleBinding:          "tanzu-addons-manager-clusterrolebinding",
+			PkgiSyncPeriod:                  10 * time.Minute,
 		},
 	)
 	Expect(bootstrapReconciler.SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: 1})).To(Succeed())
diff --git a/addons/controllers/testdata/fake_package_crd.yaml b/addons/controllers/testdata/fake_package_crd.yaml
new file mode 100644
index 0000000000..85776a7b4e
--- /dev/null
+++ b/addons/controllers/testdata/fake_package_crd.yaml
@@ -0,0 +1,432 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: packages.data.packaging.carvel.dev
+spec:
+  group: data.packaging.carvel.dev
+  names:
+    kind: Package
+    listKind: PackageList
+    plural: packages
+    singular: package
+  scope: Namespaced
+  versions:
+    - name: v1alpha1
+      schema:
+        openAPIV3Schema:
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              properties:
+                capacityRequirementsDescription:
+                  type: string
+                licenses:
+                  items:
+                    type: string
+                  type: array
+                refName:
+                  type: string
+                releaseNotes:
+                  type: string
+                releasedAt:
+                  format: date-time
+                  nullable: true
+                  type: string
+                template:
+                  properties:
+                    spec:
+                      properties:
+                        canceled:
+                          description: Canceled when set to true will stop all active
+                            changes
+                          type: boolean
+                        cluster:
+                          properties:
+                            kubeconfigSecretRef:
+                              properties:
+                                key:
+                                  type: string
+                                name:
+                                  type: string
+                              type: object
+                            namespace:
+                              type: string
+                          type: object
+                        deploy:
+                          items:
+                            properties:
+                              kapp:
+                                properties:
+                                  delete:
+                                    properties:
+                                      rawOptions:
+                                        items:
+                                          type: string
+                                        type: array
+                                    type: object
+                                  inspect:
+                                    properties:
+                                      rawOptions:
+                                        items:
+                                          type: string
+                                        type: array
+                                    type: object
+                                  intoNs:
+                                    type: string
+                                  mapNs:
+                                    items:
+                                      type: string
+                                    type: array
+                                  rawOptions:
+                                    items:
+                                      type: string
+                                    type: array
+                                type: object
+                            type: object
+                          type: array
+                        fetch:
+                          items:
+                            properties:
+                              git:
+                                properties:
+                                  lfsSkipSmudge:
+                                    type: boolean
+                                  ref:
+                                    type: string
+                                  refSelection:
+                                    properties:
+                                      semver:
+                                        properties:
+                                          constraints:
+                                            type: string
+                                          prereleases:
+                                            properties:
+                                              identifiers:
+                                                items:
+                                                  type: string
+                                                type: array
+                                            type: object
+                                        type: object
+                                    type: object
+                                  secretRef:
+                                    description: 'Secret may include one or more keys:
+                                    ssh-privatekey, ssh-knownhosts'
+                                    properties:
+                                      name:
+                                        description: Object is expected to be within
+                                          same namespace
+                                        type: string
+                                    type: object
+                                  subPath:
+                                    type: string
+                                  url:
+                                    type: string
+                                type: object
+                              helmChart:
+                                properties:
+                                  name:
+                                    description: 'Example: stable/redis'
+                                    type: string
+                                  repository:
+                                    properties:
+                                      secretRef:
+                                        properties:
+                                          name:
+                                            description: Object is expected to be within
+                                              same namespace
+                                            type: string
+                                        type: object
+                                      url:
+                                        type: string
+                                    type: object
+                                  version:
+                                    type: string
+                                type: object
+                              http:
+                                properties:
+                                  secretRef:
+                                    description: 'Secret may include one or more keys:
+                                    username, password'
+                                    properties:
+                                      name:
+                                        description: Object is expected to be within
+                                          same namespace
+                                        type: string
+                                    type: object
+                                  sha256:
+                                    type: string
+                                  subPath:
+                                    type: string
+                                  url:
+                                    description: 'URL can point to one of following
+                                    formats: text, tgz, zip'
+                                    type: string
+                                type: object
+                              image:
+                                properties:
+                                  secretRef:
+                                    description: 'Secret may include one or more keys:
+                                    username, password, token. By default anonymous
+                                    access is used for authentication. TODO support
+                                    docker config formated secret'
+                                    properties:
+                                      name:
+                                        description: Object is expected to be within
+                                          same namespace
+                                        type: string
+                                    type: object
+                                  subPath:
+                                    type: string
+                                  tagSelection:
+                                    properties:
+                                      semver:
+                                        properties:
+                                          constraints:
+                                            type: string
+                                          prereleases:
+                                            properties:
+                                              identifiers:
+                                                items:
+                                                  type: string
+                                                type: array
+                                            type: object
+                                        type: object
+                                    type: object
+                                  url:
+                                    description: 'Example: username/app1-config:v0.1.0'
+                                    type: string
+                                type: object
+                              imgpkgBundle:
+                                properties:
+                                  image:
+                                    type: string
+                                  secretRef:
+                                    description: 'Secret may include one or more keys:
+                                    username, password, token. By default anonymous
+                                    access is used for authentication. TODO support
+                                    docker config formated secret'
+                                    properties:
+                                      name:
+                                        description: Object is expected to be within
+                                          same namespace
+                                        type: string
+                                    type: object
+                                  tagSelection:
+                                    properties:
+                                      semver:
+                                        properties:
+                                          constraints:
+                                            type: string
+                                          prereleases:
+                                            properties:
+                                              identifiers:
+                                                items:
+                                                  type: string
+                                                type: array
+                                            type: object
+                                        type: object
+                                    type: object
+                                type: object
+                              inline:
+                                properties:
+                                  paths:
+                                    additionalProperties:
+                                      type: string
+                                    type: object
+                                  pathsFrom:
+                                    items:
+                                      properties:
+                                        configMapRef:
+                                          properties:
+                                            directoryPath:
+                                              type: string
+                                            name:
+                                              type: string
+                                          type: object
+                                        secretRef:
+                                          properties:
+                                            directoryPath:
+                                              type: string
+                                            name:
+                                              type: string
+                                          type: object
+                                      type: object
+                                    type: array
+                                type: object
+                            type: object
+                          type: array
+                        noopDelete:
+                          description: When NoopDeletion set to true, App deletion should
+                            delete App CR but preserve App's associated resources
+                          type: boolean
+                        paused:
+                          description: Paused when set to true will ignore all pending
+                            changes, once it set back to false, pending changes will
+                            be applied
+                          type: boolean
+                        serviceAccountName:
+                          type: string
+                        syncPeriod:
+                          description: Controls frequency of app reconciliation
+                          type: string
+                        template:
+                          items:
+                            properties:
+                              helmTemplate:
+                                properties:
+                                  name:
+                                    type: string
+                                  namespace:
+                                    type: string
+                                  path:
+                                    type: string
+                                  valuesFrom:
+                                    items:
+                                      properties:
+                                        configMapRef:
+                                          properties:
+                                            name:
+                                              type: string
+                                          type: object
+                                        path:
+                                          type: string
+                                        secretRef:
+                                          properties:
+                                            name:
+                                              type: string
+                                          type: object
+                                      type: object
+                                    type: array
+                                type: object
+                              jsonnet:
+                                description: TODO implement jsonnet
+                                type: object
+                              kbld:
+                                properties:
+                                  paths:
+                                    items:
+                                      type: string
+                                    type: array
+                                type: object
+                              kustomize:
+                                description: TODO implement kustomize
+                                type: object
+                              sops:
+                                properties:
+                                  age:
+                                    properties:
+                                      privateKeysSecretRef:
+                                        properties:
+                                          name:
+                                            type: string
+                                        type: object
+                                    type: object
+                                  paths:
+                                    items:
+                                      type: string
+                                    type: array
+                                  pgp:
+                                    properties:
+                                      privateKeysSecretRef:
+                                        properties:
+                                          name:
+                                            type: string
+                                        type: object
+                                    type: object
+                                type: object
+                              ytt:
+                                properties:
+                                  fileMarks:
+                                    items:
+                                      type: string
+                                    type: array
+                                  ignoreUnknownComments:
+                                    type: boolean
+                                  inline:
+                                    properties:
+                                      paths:
+                                        additionalProperties:
+                                          type: string
+                                        type: object
+                                      pathsFrom:
+                                        items:
+                                          properties:
+                                            configMapRef:
+                                              properties:
+                                                directoryPath:
+                                                  type: string
+                                                name:
+                                                  type: string
+                                              type: object
+                                            secretRef:
+                                              properties:
+                                                directoryPath:
+                                                  type: string
+                                                name:
+                                                  type: string
+                                              type: object
+                                          type: object
+                                        type: array
+                                    type: object
+                                  paths:
+                                    items:
+                                      type: string
+                                    type: array
+                                  strict:
+                                    type: boolean
+                                  valuesFrom:
+                                    items:
+                                      properties:
+                                        configMapRef:
+                                          properties:
+                                            name:
+                                              type: string
+                                          type: object
+                                        path:
+                                          type: string
+                                        secretRef:
+                                          properties:
+                                            name:
+                                              type: string
+                                          type: object
+                                      type: object
+                                    type: array
+                                type: object
+                            type: object
+                          type: array
+                      type: object
+                  required:
+                    - spec
+                  type: object
+                valuesSchema:
+                  description: valuesSchema can be used to show template values that
+                    can be configured by users when a Package is installed in an OpenAPI
+                    schema format.
+                  properties:
+                    openAPIv3:
+                      nullable: true
+                      type: object
+                      x-kubernetes-preserve-unknown-fields: true
+                  type: object
+                version:
+                  type: string
+              type: object
+          required:
+            - spec
+          type: object
+      served: true
+      storage: true
+      subresources:
+        status: {}
diff --git a/addons/controllers/testdata/test-cluster-bootstrap-1.yaml b/addons/controllers/testdata/test-cluster-bootstrap-1.yaml
index 39e2223e49..2b8d5432e8 100644
--- a/addons/controllers/testdata/test-cluster-bootstrap-1.yaml
+++ b/addons/controllers/testdata/test-cluster-bootstrap-1.yaml
@@ -1,4 +1,9 @@
 ---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cluster-namespace
+---
 apiVersion: run.tanzu.vmware.com/v1alpha3
 kind: TanzuKubernetesRelease
 metadata:
@@ -10,13 +15,12 @@ spec:
     imageRepository: foo
   osImages: []
   bootstrapPackages: []
-
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: Cluster
 metadata:
   name: test-cluster-tcbt
-  namespace: default
+  namespace: cluster-namespace
   labels:
     tkg.tanzu.vmware.com/cluster-name: test-cluster-tcbt
     run.tanzu.vmware.com/tkr: v1.22.3
@@ -51,6 +55,13 @@ metadata:
   name: v1.22.3
   namespace: tkg-system
 spec:
+  kapp:
+    refName: kapp-controller.tanzu.vmware.com.0.30.0
+    valuesFrom:
+      providerRef:
+        apiGroup: run.tanzu.vmware.com
+        kind: KappControllerConfig
+        name: test-cluster-tcbt-kapp-controller-config
   additionalPackages:
     - refName: foobar1.example.com.1.17.2
       valuesFrom:
@@ -62,19 +73,18 @@ spec:
           kind: FooBar
           name: example-foo
   cnis:
-    - refName: "calico.tanzu.vmware.com.0.3.0--vmware.1-tkg.1-rc.1"
+    - refName: calico.tanzu.vmware.com.3.19.1--vmware.1-tkg.1
       valuesFrom:
         providerRef:
           apiGroup: cni.tanzu.vmware.com
           kind: CalicoConfig
           name: test-cluster-tcbt
-    - refName: "antrea.tanzu.vmware.com.21.2.0--vmware.1-tkg.1-rc.1"
+    - refName: antrea.tanzu.vmware.com.1.2.3--vmware.1-tkg.1
       valuesFrom:
         providerRef:
           apiGroup: cni.tanzu.vmware.com
           kind: AntreaConfig
           name: test-cluster-tcbt
-
 ---
 apiVersion: run.tanzu.vmware.com/v1alpha1
 kind: FooBar
@@ -111,3 +121,414 @@ spec:
   antrea:
     config:
       trafficEncapMode: encap
+---
+apiVersion: run.tanzu.vmware.com/v1alpha3
+kind: KappControllerConfig
+metadata:
+  name: test-cluster-tcbt-kapp-controller-config
+  namespace: tkg-system
+spec:
+  namespace: test-ns
+  kappController:
+    createNamespace: true
+    globalNamespace: tanzu-package-repo-global
+    deployment:
+      concurrency: 4
+      hostNetwork: true
+      priorityClassName: system-cluster-critical
+      apiPort: 10100
+      metricsBindAddress: "0"
+      tolerations:
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/master
+        - effect: NoSchedule
+          key: node.kubernetes.io/not-ready
+        - effect: NoSchedule
+          key: node.cloudprovider.kubernetes.io/uninitialized
+          value: "true"
+---
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
+metadata:
+  name: foobar.example.com.1.17.2
+  namespace: cluster-namespace
+spec:
+  refName: foobar.example.com
+  version: 1.17.2
+  releasedAt: "2021-05-13T18:00:00Z"
+  releaseNotes: foobar 1.17.2
+  capacityRequirementsDescription: Varies significantly based on cluster size. This should be tuned based on observed usage.
+  valuesSchema:
+    openAPIv3:
+      title: foobar.example.com.1.17.2+vmware.1-tkg.1 values schema
+      properties:
+        namespace:
+          type: string
+          description: The namespace in which to deploy fluent-bit.
+          default: tanzu-system-logging
+  licenses:
+    - 'VMware''s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/fluent-bit:v1.7.5_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=5m
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+---
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
+metadata:
+  name: foobar.example.com.1.17.2
+  namespace: tkg-system
+spec:
+  refName: foobar.example.com
+  version: 1.17.2
+  releasedAt: "2021-05-13T18:00:00Z"
+  releaseNotes: foobar 1.17.2
+  capacityRequirementsDescription: Varies significantly based on cluster size. This should be tuned based on observed usage.
+  valuesSchema:
+    openAPIv3:
+      title: foobar.example.com.1.17.2+vmware.1-tkg.1 values schema
+      properties:
+        namespace:
+          type: string
+          description: The namespace in which to deploy fluent-bit.
+          default: tanzu-system-logging
+  licenses:
+    - 'VMware''s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/fluent-bit:v1.7.5_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=5m
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+---
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
+metadata:
+  name: foobar1.example.com.1.17.2
+  namespace: cluster-namespace
+spec:
+  refName: foobar1.example.com
+  version: 1.17.2
+  releasedAt: "2021-05-13T18:00:00Z"
+  releaseNotes: foobar1 1.17.2
+  capacityRequirementsDescription: Varies significantly based on cluster size. This should be tuned based on observed usage.
+  valuesSchema:
+    openAPIv3:
+      title: foobar1.example.com.1.17.2+vmware.1-tkg.1 values schema
+      properties:
+        namespace:
+          type: string
+          description: The namespace in which to deploy fluent-bit.
+          default: tanzu-system-logging
+  licenses:
+    - 'VMware''s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/fluent-bit:v1.7.5_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=5m
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+---
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
+metadata:
+  name: foobar1.example.com.1.17.2
+  namespace: tkg-system
+spec:
+  refName: foobar1.example.com
+  version: 1.17.2
+  releasedAt: "2021-05-13T18:00:00Z"
+  releaseNotes: foobar1 1.17.2
+  capacityRequirementsDescription: Varies significantly based on cluster size. This should be tuned based on observed usage.
+  valuesSchema:
+    openAPIv3:
+      title: foobar1.example.com.1.17.2+vmware.1-tkg.1 values schema
+      properties:
+        namespace:
+          type: string
+          description: The namespace in which to deploy fluent-bit.
+          default: tanzu-system-logging
+  licenses:
+    - 'VMware''s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/fluent-bit:v1.7.5_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=5m
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+---
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
+metadata:
+  name: kapp-controller.tanzu.vmware.com.0.30.0
+  namespace: cluster-namespace
+spec:
+  refName: kapp-controller.tanzu.vmware.com
+  version: 0.30.0
+  releaseNotes: kapp-controller 0.30.0 https://github.com/vmware-tanzu/carvel-kapp-controller
+  licenses:
+    - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/kapp-controller:v0.30.0_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=30s
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+  releasedAt: "2021-12-20T10:59:32Z"
+  valuesSchema:
+    openAPIv3:
+      title: kapp-controller.tanzu.vmware.com.0.30.0+vmware.1-tkg.1 values schema
+---
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
+metadata:
+  name: kapp-controller.tanzu.vmware.com.0.30.0
+  namespace: tkg-system
+spec:
+  refName: kapp-controller.tanzu.vmware.com
+  version: 0.30.0
+  releaseNotes: kapp-controller 0.30.0 https://github.com/vmware-tanzu/carvel-kapp-controller
+  licenses:
+    - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/kapp-controller:v0.30.0_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=30s
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+  releasedAt: "2021-12-20T10:59:32Z"
+  valuesSchema:
+    openAPIv3:
+      title: kapp-controller.tanzu.vmware.com.0.30.0+vmware.1-tkg.1 values schema
+---
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
+metadata:
+  name: calico.tanzu.vmware.com.3.19.1--vmware.1-tkg.1
+  namespace: cluster-namespace
+spec:
+  refName: calico.tanzu.vmware.com
+  version: 3.19.1+vmware.1-tkg.1
+  releaseNotes: calico 3.19.1 https://docs.projectcalico.org/archive/v3.19/release-notes/
+  licenses:
+    - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/calico:v3.19.1_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=30s
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+  releasedAt: "2021-12-20T10:59:32Z"
+  valuesSchema:
+    openAPIv3:
+      title: calico.tanzu.vmware.com.3.19.1+vmware.1-tkg.1 values schema
+---
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
+metadata:
+  name: calico.tanzu.vmware.com.3.19.1--vmware.1-tkg.1
+  namespace: tkg-system
+spec:
+  refName: calico.tanzu.vmware.com
+  version: 3.19.1+vmware.1-tkg.1
+  releaseNotes: calico 3.19.1 https://docs.projectcalico.org/archive/v3.19/release-notes/
+  licenses:
+    - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/calico:v3.19.1_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=30s
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+  releasedAt: "2021-12-20T10:59:32Z"
+  valuesSchema:
+    openAPIv3:
+      title: calico.tanzu.vmware.com.3.19.1+vmware.1-tkg.1 values schema
+---
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
+metadata:
+  name: antrea.tanzu.vmware.com.1.2.3--vmware.1-tkg.1
+  namespace: cluster-namespace
+spec:
+  refName: antrea.tanzu.vmware.com
+  version: 1.2.3+vmware.1-tkg.1
+  releaseNotes: antrea 1.2.3 https://github.com/antrea-io/antrea/releases/tag/v1.2.3
+  licenses:
+    - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/antrea:v1.2.3_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=30s
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+  releasedAt: "2021-12-20T10:59:32Z"
+  valuesSchema:
+    openAPIv3:
+      title: antrea.tanzu.vmware.com.1.2.3+vmware.1-tkg.1 values schema
+---
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
+metadata:
+  name: antrea.tanzu.vmware.com.1.2.3--vmware.1-tkg.1
+  namespace: tkg-system
+spec:
+  refName: antrea.tanzu.vmware.com
+  version: 1.2.3+vmware.1-tkg.1
+  releaseNotes: antrea 1.2.3 https://github.com/antrea-io/antrea/releases/tag/v1.2.3
+  licenses:
+    - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/antrea:v1.2.3_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=30s
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+  releasedAt: "2021-12-20T10:59:32Z"
+  valuesSchema:
+    openAPIv3:
+      title: antrea.tanzu.vmware.com.1.2.3+vmware.1-tkg.1 values schema
diff --git a/addons/controllers/testdata/test-cluster-bootstrap-2.yaml b/addons/controllers/testdata/test-cluster-bootstrap-2.yaml
index d3f6e6d91e..11a3e15b1e 100644
--- a/addons/controllers/testdata/test-cluster-bootstrap-2.yaml
+++ b/addons/controllers/testdata/test-cluster-bootstrap-2.yaml
@@ -1,25 +1,17 @@
 ---
-apiVersion: run.tanzu.vmware.com/v1alpha3
-kind: TanzuKubernetesRelease
+apiVersion: v1
+kind: Namespace
 metadata:
-  name: v1.21.2---vmware.1-tkg.1
-spec:
-  version: v1.21.2---vmware.1-tkg.1
-  kubernetes:
-    version: v1.21.2---vmware.1-tkg.1
-    imageRepository: foo
-  osImages: []
-  bootstrapPackages: []
-
+  name: cluster-namespace-2
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: Cluster
 metadata:
   name: test-cluster-tcbt-2
-  namespace: default
+  namespace: cluster-namespace-2
   labels:
     tkg.tanzu.vmware.com/cluster-name: test-cluster-tcbt-2
-    run.tanzu.vmware.com/tkr: v1.21.2---vmware.1-tkg.1
+    run.tanzu.vmware.com/tkr: v1.22.3
 spec:
   infrastructureRef:
     kind: VSphereCluster
@@ -28,47 +20,195 @@ spec:
       cidrBlocks: [ "192.168.0.0/16","fd00:100:96::/48" ]
     services:
       cidrBlocks: [ "192.168.0.0/16","fd00:100:96::/48" ]
-  topology:
-    class: test-clusterclass-tcbt
-    version: v1.21.2---vmware.1-tkg.1
 ---
-apiVersion: run.tanzu.vmware.com/v1alpha3
-kind: ClusterBootstrapTemplate
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
 metadata:
-  name: v1.21.2---vmware.1-tkg.1
-  namespace: tkg-system
+  name: foobar.example.com.1.17.2
+  namespace: cluster-namespace-2
 spec:
-  cnis:
-    - refName: "calico.tanzu.vmware.com.0.3.0--vmware.1-tkg.1-rc.1"
-      valuesFrom:
-        providerRef:
-          apiGroup: cni.tanzu.vmware.com
-          kind: CalicoConfig
-          name: test-cluster-tcbt-2
-    - refName: "antrea.tanzu.vmware.com.21.2.0--vmware.1-tkg.1-rc.1"
-      valuesFrom:
-        providerRef:
-          apiGroup: cni.tanzu.vmware.com
-          kind: AntreaConfig
-          name: test-cluster-tcbt-2
-
+  refName: foobar.example.com
+  version: 1.17.2
+  releasedAt: "2021-05-13T18:00:00Z"
+  releaseNotes: foobar 1.17.2
+  capacityRequirementsDescription: Varies significantly based on cluster size. This should be tuned based on observed usage.
+  valuesSchema:
+    openAPIv3:
+      title: foobar.example.com.1.17.2+vmware.1-tkg.1 values schema
+      properties:
+        namespace:
+          type: string
+          description: The namespace in which to deploy fluent-bit.
+          default: tanzu-system-logging
+  licenses:
+    - 'VMware''s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/fluent-bit:v1.7.5_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=5m
+              - --kube-api-qps=20
+              - --kube-api-burst=30
 ---
-apiVersion: cni.tanzu.vmware.com/v1alpha1
-kind: CalicoConfig
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
 metadata:
-  name: test-cluster-tcbt-2
-  namespace: tkg-system
+  name: foobar1.example.com.1.17.2
+  namespace: cluster-namespace-2
 spec:
-  calico:
-    config:
-      vethMTU: 0
+  refName: foobar1.example.com
+  version: 1.17.2
+  releasedAt: "2021-05-13T18:00:00Z"
+  releaseNotes: foobar1 1.17.2
+  capacityRequirementsDescription: Varies significantly based on cluster size. This should be tuned based on observed usage.
+  valuesSchema:
+    openAPIv3:
+      title: foobar1.example.com.1.17.2+vmware.1-tkg.1 values schema
+      properties:
+        namespace:
+          type: string
+          description: The namespace in which to deploy fluent-bit.
+          default: tanzu-system-logging
+  licenses:
+    - 'VMware''s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/fluent-bit:v1.7.5_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=5m
+              - --kube-api-qps=20
+              - --kube-api-burst=30
 ---
-apiVersion: cni.tanzu.vmware.com/v1alpha1
-kind: AntreaConfig
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
 metadata:
-  name: test-cluster-tcbt-2
-  namespace: tkg-system
+  name: kapp-controller.tanzu.vmware.com.0.30.0
+  namespace: cluster-namespace-2
+spec:
+  refName: kapp-controller.tanzu.vmware.com
+  version: 0.30.0
+  releaseNotes: kapp-controller 0.30.0 https://github.com/vmware-tanzu/carvel-kapp-controller
+  licenses:
+    - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/kapp-controller:v0.30.0_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=30s
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+  releasedAt: "2021-12-20T10:59:32Z"
+  valuesSchema:
+    openAPIv3:
+      title: kapp-controller.tanzu.vmware.com.0.30.0+vmware.1-tkg.1 values schema
+---
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
+metadata:
+  name: calico.tanzu.vmware.com.3.19.1--vmware.1-tkg.1
+  namespace: cluster-namespace-2
+spec:
+  refName: calico.tanzu.vmware.com
+  version: 3.19.1+vmware.1-tkg.1
+  releaseNotes: calico 3.19.1 https://docs.projectcalico.org/archive/v3.19/release-notes/
+  licenses:
+    - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/calico:v3.19.1_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=30s
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+  releasedAt: "2021-12-20T10:59:32Z"
+  valuesSchema:
+    openAPIv3:
+      title: calico.tanzu.vmware.com.3.19.1+vmware.1-tkg.1 values schema
+---
+apiVersion: data.packaging.carvel.dev/v1alpha1
+kind: Package
+metadata:
+  name: antrea.tanzu.vmware.com.1.2.3--vmware.1-tkg.1
+  namespace: cluster-namespace-2
 spec:
-  antrea:
-    config:
-      trafficEncapMode: encap
+  refName: antrea.tanzu.vmware.com
+  version: 1.2.3+vmware.1-tkg.1
+  releaseNotes: antrea 1.2.3 https://github.com/antrea-io/antrea/releases/tag/v1.2.3
+  licenses:
+    - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)'
+  template:
+    spec:
+      fetch:
+        - imgpkgBundle:
+            image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/antrea:v1.2.3_vmware.1-tkg.1
+      template:
+        - ytt:
+            paths:
+              - config/
+            ignoreUnknownComments: true
+        - kbld:
+            paths:
+              - '-'
+              - .imgpkg/images.yml
+      deploy:
+        - kapp:
+            rawOptions:
+              - --wait-timeout=30s
+              - --kube-api-qps=20
+              - --kube-api-burst=30
+  releasedAt: "2021-12-20T10:59:32Z"
+  valuesSchema:
+    openAPIv3:
+      title: antrea.tanzu.vmware.com.1.2.3+vmware.1-tkg.1 values schema