Repository: vmware-tanzu/tanzu-framework (language: go)

Issue:
Enable delete of workload cluster on supervisor if TKR is not available
**Bug description**

Today in TKGs (vSphere with supervisor) user cannot delete TKC/Classy cluster when the TKR associated to the cluster is no longer available on the supervisor.

If the VMImage associated to a TKR is no longer available in the content library associated to the TKG service on supervisor, the TKR is no longer available on the supervisor cluster. 

In such cases, the addons controller does not have access to the TKR as a result blocking the reconcile loops which in turn blocks the delete cluster workflow.

This issue was fixed on the supervisor in guest-cluster-controller as part of https://gitlab.eng.vmware.com/core-build/guest-cluster-controller/-/merge_requests/958

A similar change needs to be done in addons controller to ensure the reconcile loops in addons controller can handle delete cluster workflow when TKR is not available on the supervisor

**Affected product area (please put an X in all that apply)**

- ( ) APIs
- ( ) Addons
- ( ) CLI
- ( ) Docs
- ( ) IAM
- ( ) Installation
- ( ) Plugin
- ( ) Security
- ( ) Test and Release
- ( ) User Experience
- ( ) Developer Experience

**Expected behavior**
Should be able to delete workload cluster in cases where TKR resource associated to the cluster is not available on the supervisor
**Steps to reproduce the bug**
On supervisor
1. create workload cluster (TKC/Classy) with say version 1.23.8
2. after successful bring up of the cluster, delete the associated vmimage from the content library on vSphere
3. Once the vmimage is removed from CL, the TKR definition is no longer available in the supervisor
4. Delete workload cluster (TKC/Classy cluster)

Delete of cluster is stuck as predelete hook of addons controller is stuck as the TKR resource is no longer available

**Version** (include the SHA if the version is not obvious)

**Environment where the bug was observed (cloud, OS, etc)**

**Relevant Debug Output (Logs, manifests, etc)**



Discussion / hints:
Hey @jvrahav! Thanks for opening your first issue. We appreciate your contribution and welcome you to our community! We are glad to have you here and to have your input on Tanzu Framework.

Fix patch:
diff --git a/addons/controllers/cluster_metadata_controller.go b/addons/controllers/cluster_metadata_controller.go
index e6fe0493e2..8d0d64c42a 100644
--- a/addons/controllers/cluster_metadata_controller.go
+++ b/addons/controllers/cluster_metadata_controller.go
@@ -92,7 +92,7 @@ func (r *ClusterMetadataReconciler) Reconcile(ctx context.Context, req reconcile
 		}
 	}
 
-	tkr, err := util.GetTKRByNameV1Alpha3(r.context, r.Client, tkrName)
+	tkr, err := util.GetTKRByNameV1Alpha3(r.context, r.Client, cluster, tkrName)
 	if err != nil {
 		log.Error(err, "unable to fetch TKR object", "name", tkrName)
 		return ctrl.Result{}, err
diff --git a/addons/controllers/clusterbootstrap_controller.go b/addons/controllers/clusterbootstrap_controller.go
index 4163e612f0..66602b0c1f 100644
--- a/addons/controllers/clusterbootstrap_controller.go
+++ b/addons/controllers/clusterbootstrap_controller.go
@@ -150,7 +150,7 @@ func (r *ClusterBootstrapReconciler) Reconcile(ctx context.Context, req ctrl.Req
 		return ctrl.Result{}, nil
 	}
 
-	tkr, err := util.GetTKRByNameV1Alpha3(r.context, r.Client, tkrName)
+	tkr, err := util.GetTKRByNameV1Alpha3(r.context, r.Client, cluster, tkrName)
 	if err != nil {
 		log.Error(err, "unable to fetch TKR object", "name", tkrName)
 		return ctrl.Result{}, err
diff --git a/addons/controllers/packageinstallstatus_controller.go b/addons/controllers/packageinstallstatus_controller.go
index 10fe78ff5e..5e02d9fece 100644
--- a/addons/controllers/packageinstallstatus_controller.go
+++ b/addons/controllers/packageinstallstatus_controller.go
@@ -138,7 +138,7 @@ func (r *PackageInstallStatusReconciler) Reconcile(_ context.Context, req reconc
 		return ctrl.Result{}, nil
 	}
 
-	tkr, err := util.GetTKRByNameV1Alpha3(r.ctx, r.Client, tkrName)
+	tkr, err := util.GetTKRByNameV1Alpha3(r.ctx, r.Client, cluster, tkrName)
 	if err != nil {
 		return ctrl.Result{}, errors.Wrapf(err, "unable to fetch TKR object '%s'", tkrName)
 	}
diff --git a/addons/pkg/constants/constants.go b/addons/pkg/constants/constants.go
index 35700c9367..407fa9be9e 100644
--- a/addons/pkg/constants/constants.go
+++ b/addons/pkg/constants/constants.go
@@ -369,6 +369,8 @@ const (
 	TKGDevPlan = "dev"
 	// TKGDevCCPan is the tkg plan name, mean that that cluster is in a dev env
 	TKGDevCCPan = "devcc"
+
+	TKRAnnotationKey = "run.tanzu.vmware.com/tkr-spec"
 )
 
 var (
diff --git a/addons/pkg/util/tkr_util.go b/addons/pkg/util/tkr_util.go
index ca9f959ede..d1cdb24b0f 100644
--- a/addons/pkg/util/tkr_util.go
+++ b/addons/pkg/util/tkr_util.go
@@ -4,10 +4,16 @@
 package util
 
 import (
+	"bytes"
+	"compress/gzip"
 	"context"
+	"encoding/base64"
 	"fmt"
+	"io"
 	"strings"
 
+	"gopkg.in/yaml.v3"
+
 	"github.com/pkg/errors"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	clusterapiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1"
@@ -38,7 +44,7 @@ func GetTKRByNameV1Alpha1(ctx context.Context, c client.Client, tkrName string)
 }
 
 // GetTKRByNameV1Alpha3 gets v1Alpha3 TKR object given a TKR name
-func GetTKRByNameV1Alpha3(ctx context.Context, c client.Client, tkrName string) (*runtanzuv1alpha3.TanzuKubernetesRelease, error) {
+func GetTKRByNameV1Alpha3(ctx context.Context, c client.Client, cluster *clusterapiv1beta1.Cluster, tkrName string) (*runtanzuv1alpha3.TanzuKubernetesRelease, error) {
 	tkrV1Alpha3 := &runtanzuv1alpha3.TanzuKubernetesRelease{}
 
 	if tkrName == "" {
@@ -49,6 +55,11 @@ func GetTKRByNameV1Alpha3(ctx context.Context, c client.Client, tkrName string)
 
 	if err := c.Get(ctx, tkrNamespaceName, tkrV1Alpha3); err != nil {
 		if apierrors.IsNotFound(err) {
+			tkrV1Alpha3, err = getTKRFromAnnotation(cluster.Annotations)
+			if tkrV1Alpha3 != nil {
+				return tkrV1Alpha3, nil
+			}
+
 			return nil, nil
 		}
 		return nil, err
@@ -72,7 +83,7 @@ func GetBootstrapPackageNameFromTKR(ctx context.Context, clt client.Client, pkgR
 	}
 
 	// get TKR object associated with the cluster
-	tkr, err := GetTKRByNameV1Alpha3(ctx, clt, tkrName)
+	tkr, err := GetTKRByNameV1Alpha3(ctx, clt, cluster, tkrName)
 	if err != nil || tkr == nil {
 		return "", pkgNamePrefix, fmt.Errorf("unable to fetch TKR object '%s'", tkrName)
 	}
@@ -90,3 +101,39 @@ func GetBootstrapPackageNameFromTKR(ctx context.Context, clt client.Client, pkgR
 
 	return "", pkgNamePrefix, fmt.Errorf("no bootstrap package prefixed with '%s' is found in the TKR object", pkgNamePrefix)
 }
+
+func getTKRFromAnnotation(tkc map[string]string) (*runtanzuv1alpha3.TanzuKubernetesRelease, error) {
+	controlPlaneTKR := &runtanzuv1alpha3.TanzuKubernetesRelease{}
+
+	tkrSpec, ok := tkc[constants.TKRAnnotationKey]
+	if !ok {
+		return nil, nil
+	}
+
+	unzipOut, err := gunzipAndBase64Decode(tkrSpec)
+	if err != nil {
+		return nil, err
+	}
+
+	err = yaml.Unmarshal(unzipOut, controlPlaneTKR)
+	if err != nil {
+		return nil, err
+	}
+
+	return controlPlaneTKR, nil
+}
+
+// GunzipAndBase64Decode extracts a gzip archive to a byte array
+func gunzipAndBase64Decode(input string) ([]byte, error) {
+	decodedData, err := base64.StdEncoding.DecodeString(input)
+	if err != nil {
+		return nil, err
+	}
+
+	r, err := gzip.NewReader(bytes.NewReader(decodedData))
+	if err != nil {
+		return nil, err
+	}
+
+	return io.ReadAll(r)
+}


Test patch:
diff --git a/addons/pkg/util/tkr_util_test.go b/addons/pkg/util/tkr_util_test.go
index 4c15c1f834..d713e643ed 100644
--- a/addons/pkg/util/tkr_util_test.go
+++ b/addons/pkg/util/tkr_util_test.go
@@ -4,9 +4,14 @@
 package util
 
 import (
+	"bytes"
+	"compress/gzip"
 	"context"
+	"encoding/base64"
 	"fmt"
 
+	"gopkg.in/yaml.v3"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/pkg/errors"
@@ -54,7 +59,8 @@ var _ = Describe("TKR utils", func() {
 			BeforeEach(func() {
 				crtCtl = &fakeclusterclient.CRTClusterClient{}
 				tkrName = ""
-				tkrV1Alpha3, err = GetTKRByNameV1Alpha3(ctx, crtCtl, tkrName)
+				cluster := &clusterapiv1beta1.Cluster{}
+				tkrV1Alpha3, err = GetTKRByNameV1Alpha3(ctx, crtCtl, cluster, tkrName)
 			})
 
 			It("should return nil TKR", func() {
@@ -81,8 +87,9 @@ var _ = Describe("TKR utils", func() {
 			BeforeEach(func() {
 				crtCtl = &fakeclusterclient.CRTClusterClient{}
 				tkrName = testTKR
+				cluster := &clusterapiv1beta1.Cluster{}
 				crtCtl.GetReturns(apierrors.NewNotFound(schema.GroupResource{Resource: "TanzuKubernetesRelease"}, testTKR))
-				tkrV1Alpha3, err = GetTKRByNameV1Alpha3(ctx, crtCtl, tkrName)
+				tkrV1Alpha3, err = GetTKRByNameV1Alpha3(ctx, crtCtl, cluster, tkrName)
 			})
 
 			It("should return nil TKR", func() {
@@ -110,8 +117,9 @@ var _ = Describe("TKR utils", func() {
 			BeforeEach(func() {
 				crtCtl = &fakeclusterclient.CRTClusterClient{}
 				tkrName = testTKR
+				cluster := &clusterapiv1beta1.Cluster{}
 				crtCtl.GetReturns(errors.New("some error"))
-				tkrV1Alpha3, err = GetTKRByNameV1Alpha3(ctx, crtCtl, tkrName)
+				tkrV1Alpha3, err = GetTKRByNameV1Alpha3(ctx, crtCtl, cluster, tkrName)
 			})
 
 			It("should return nil TKR", func() {
@@ -138,7 +146,8 @@ var _ = Describe("TKR utils", func() {
 			BeforeEach(func() {
 				crtCtl = &fakeclusterclient.CRTClusterClient{}
 				tkrName = testTKR
-				tkrV1Alpha3, err = GetTKRByNameV1Alpha3(ctx, crtCtl, tkrName)
+				cluster := &clusterapiv1beta1.Cluster{}
+				tkrV1Alpha3, err = GetTKRByNameV1Alpha3(ctx, crtCtl, cluster, tkrName)
 			})
 
 			It("should return nil TKR", func() {
@@ -146,6 +155,66 @@ var _ = Describe("TKR utils", func() {
 				Expect(tkrV1Alpha3).ShouldNot(BeNil())
 			})
 		})
+
+		When("tkr is available as annotation on cluster resource", func() {
+			BeforeEach(func() {
+				crtCtl = &fakeclusterclient.CRTClusterClient{}
+				tkrName = testTKR
+				var buf bytes.Buffer
+				out, _ := yaml.Marshal(tkrV1Alpha3)
+				w, _ := gzip.NewWriterLevel(&buf, gzip.BestCompression)
+				_, err = w.Write(out)
+				w.Close()
+				tkrString := base64.StdEncoding.EncodeToString(buf.Bytes())
+				annotation := make(map[string]string)
+				annotation["run.tanzu.vmware.com/tkr-spec"] = tkrString
+				cluster := &clusterapiv1beta1.Cluster{
+					TypeMeta: metav1.TypeMeta{
+						Kind:       "Cluster",
+						APIVersion: "v1beta1",
+					},
+					ObjectMeta: metav1.ObjectMeta{Name: testClusterName, Namespace: testNamespace, Labels: map[string]string{}, Annotations: annotation},
+				}
+				tkrV1Alpha3, err = getTKRFromAnnotation(cluster.Annotations)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(tkrV1Alpha3).NotTo(BeNil())
+			})
+
+			It("should return non nil TKR", func() {
+				Expect(err).ShouldNot(HaveOccurred())
+				Expect(tkrV1Alpha3).ShouldNot(BeNil())
+			})
+		})
+
+		When("tkr is not available in cluster", func() {
+			BeforeEach(func() {
+				crtCtl = &fakeclusterclient.CRTClusterClient{}
+				tkrName = testTKR
+				var buf bytes.Buffer
+				out, _ := yaml.Marshal(tkrV1Alpha3)
+				w, _ := gzip.NewWriterLevel(&buf, gzip.BestCompression)
+				_, err = w.Write(out)
+				w.Close()
+				tkrString := base64.StdEncoding.EncodeToString(buf.Bytes())
+				annotation := make(map[string]string)
+				annotation["run.tanzu.vmware.com/tkr-spec"] = tkrString
+				cluster := &clusterapiv1beta1.Cluster{
+					TypeMeta: metav1.TypeMeta{
+						Kind:       "Cluster",
+						APIVersion: "v1beta1",
+					},
+					ObjectMeta: metav1.ObjectMeta{Name: testClusterName, Namespace: testNamespace, Labels: map[string]string{}, Annotations: annotation},
+				}
+				tkrName = testTKR
+				crtCtl.GetReturns(apierrors.NewNotFound(schema.GroupResource{Resource: "TanzuKubernetesRelease"}, testTKR))
+				tkrV1Alpha3, err = GetTKRByNameV1Alpha3(ctx, crtCtl, cluster, tkrName)
+			})
+
+			It("should return tkr from annotation on cluster", func() {
+				Expect(err).ShouldNot(HaveOccurred())
+				Expect(tkrV1Alpha3).ShouldNot(BeNil())
+			})
+		})
 	})
 
 	Context("GetBootstrapPackageNameFromTKR()", func() {