Repository: vmware-tanzu/tanzu-framework (language: go)

Issue:
CRD Waiter in Tanzu Addons controller hangs on cached nil reads.
**Bug description**
Repeatedly testing `tanzu management-cluster create` locally on fresh kind clusters finds that the addons controller frequently stalls in its WaitforCRD function.

@randomvariable suggests that the problem is due to the lists retrieved hitting cached results, and the problem is resolved when the addon controller is kicked.

This significantly slows bootstrap time, and also puts the command at risk of timeout.

Log errors are as follows:
```
I1219 11:54:20.316919       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:54:25.320559       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:54:30.318875       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:54:35.322640       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:54:40.321588       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:54:45.323425       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:54:50.318498       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:54:55.322675       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:55:00.322242       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:55:05.322185       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:55:10.321237       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:55:15.322322       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:55:20.321640       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:55:25.321302       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:55:30.321227       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:55:35.322864       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:55:40.322856       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:55:45.322471       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:55:50.320092       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:55:55.321694       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:56:00.322489       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:56:05.321196       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:56:10.322231       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:56:15.322546       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:56:20.318506       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:56:25.321420       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:56:30.321818       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:56:35.322282       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:56:40.321323       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:56:45.321615       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:56:50.322382       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:56:55.321277       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:57:00.321635       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:57:05.322151       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:57:10.321376       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:57:15.318417       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:57:20.318782       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:57:25.318898       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:57:30.322333       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:57:35.321598       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:57:40.322232       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:57:45.321349       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:57:50.321868       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:57:55.322044       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
I1219 11:58:00.322399       1 logr.go:261] setup "msg"="error retrieving GroupVersion" "GroupVersion"="run.tanzu.vmware.com/v1alpha1"
```

**Affected product area (please put an X in all that apply)**

- ( ) APIs
- ( ) Addons
- ( ) CLI
- ( ) Docs
- ( ) IAM
- ( ) Installation
- ( ) Plugin
- ( ) Security
- ( ) Test and Release
- ( ) User Experience
- ( ) Developer Experience

**Expected behavior**

**Steps to reproduce the bug**

**Version** (include the SHA if the version is not obvious)

**Environment where the bug was observed (cloud, OS, etc)**

**Relevant Debug Output (Logs, manifests, etc)**



Discussion / hints:

Fix patch:
diff --git a/addons/controllers/addon_controller.go b/addons/controllers/addon_controller.go
index 53eb7782a9..be1a95f3a3 100644
--- a/addons/controllers/addon_controller.go
+++ b/addons/controllers/addon_controller.go
@@ -10,6 +10,7 @@ import (
 
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -592,7 +593,7 @@ func (r *AddonReconciler) GetAddonKappResourceReconciler(
 }
 
 // GetExternalCRDs returns all external custom resources that addon controller depends on
-func GetExternalCRDs() map[schema.GroupVersion]*sets.String {
+func GetExternalCRDs() (map[schema.GroupVersion]*sets.String, []client.Object) {
 	var crds = map[schema.GroupVersion]*sets.String{}
 	// cluster-api
 	clusterapiv1alpha3Resources := sets.NewString("clusters")
@@ -612,5 +613,40 @@ func GetExternalCRDs() map[schema.GroupVersion]*sets.String {
 	kapppkgv1alpha1Resources := sets.NewString("packageinstalls", "packagerepositories")
 	crds[kapppkg.SchemeGroupVersion] = &kapppkgv1alpha1Resources
 
-	return crds
+	// Add CustomResourceDefinition for the above apis
+	clusterCRD := &apiextensionsv1.CustomResourceDefinition{
+		ObjectMeta: metav1.ObjectMeta{Name: "clusters.cluster.x-k8s.io"},
+		Spec:       apiextensionsv1.CustomResourceDefinitionSpec{Group: "cluster.x-k8s.io", Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{Name: "v1beta1"}}},
+	}
+
+	kubeadmcontrolplaneCRD := &apiextensionsv1.CustomResourceDefinition{
+		ObjectMeta: metav1.ObjectMeta{Name: "kubeadmcontrolplanes.controlplane.cluster.x-k8s.io"},
+	}
+
+	PackageInstallCRD := &apiextensionsv1.CustomResourceDefinition{
+		ObjectMeta: metav1.ObjectMeta{Name: "packageinstalls.packaging.carvel.dev"},
+	}
+
+	tanzukubernetesreleaseCRD := &apiextensionsv1.CustomResourceDefinition{
+		ObjectMeta: metav1.ObjectMeta{Name: "tanzukubernetesreleases.run.tanzu.vmware.com"},
+	}
+
+	packagerepositorieCRD := &apiextensionsv1.CustomResourceDefinition{
+		ObjectMeta: metav1.ObjectMeta{Name: "packagerepositories.packaging.carvel.dev"},
+	}
+
+	appCRD := &apiextensionsv1.CustomResourceDefinition{
+		ObjectMeta: metav1.ObjectMeta{Name: "apps.kappctrl.k14s.io"},
+	}
+
+	initObjs := []client.Object{
+		clusterCRD.DeepCopy(),
+		kubeadmcontrolplaneCRD.DeepCopy(),
+		PackageInstallCRD.DeepCopy(),
+		tanzukubernetesreleaseCRD.DeepCopy(),
+		packagerepositorieCRD.DeepCopy(),
+		appCRD.DeepCopy(),
+	}
+
+	return crds, initObjs
 }
diff --git a/addons/main.go b/addons/main.go
index be00f507e3..3c9b88a686 100644
--- a/addons/main.go
+++ b/addons/main.go
@@ -187,23 +187,6 @@ func main() {
 		}()
 	}
 
-	crdwaiter := crdwait.CRDWaiter{
-		Ctx: ctx,
-		ClientSetFn: func() (kubernetes.Interface, error) {
-			return kubernetes.NewForConfig(ctrl.GetConfigOrDie())
-		},
-		Logger:       setupLog,
-		Scheme:       scheme,
-		PollInterval: constants.CRDWaitPollInterval,
-		PollTimeout:  constants.CRDWaitPollTimeout,
-	}
-	if err := crdwaiter.WaitForCRDs(controllers.GetExternalCRDs(),
-		&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: os.Getenv("POD_NAME"), Namespace: os.Getenv("POD_NAMESPACE")}},
-		constants.AddonControllerName,
-	); err != nil {
-		setupLog.Error(err, "unable to wait for CRDs")
-		os.Exit(1)
-	}
 	if flags.featureGateClusterBootstrap {
 		if err := os.MkdirAll(constants.WebhookCertDir, 0755); err != nil {
 			setupLog.Error(err, "unable to create directory for webhook certificates", "directory", constants.WebhookCertDir)
@@ -223,6 +206,32 @@ func main() {
 		SyncPeriod:             &flags.syncPeriod,
 		HealthProbeBindAddress: flags.healthdAddr,
 	})
+
+	clientSet, err := kubernetes.NewForConfig(mgr.GetConfig())
+	if err != nil {
+		setupLog.Error(err, "unable to create kubernetes clientset")
+		os.Exit(1)
+	}
+
+	crdwaiter := crdwait.CRDWaiter{
+		Ctx:          ctx,
+		APIReader:    mgr.GetAPIReader(),
+		ClientSet:    clientSet,
+		Logger:       setupLog,
+		Scheme:       scheme,
+		PollInterval: constants.CRDWaitPollInterval,
+		PollTimeout:  constants.CRDWaitPollTimeout,
+	}
+
+	crds, _ := controllers.GetExternalCRDs()
+	if err := crdwaiter.WaitForCRDs(crds,
+		&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: os.Getenv("POD_NAME"), Namespace: os.Getenv("POD_NAMESPACE")}},
+		constants.AddonControllerName,
+	); err != nil {
+		setupLog.Error(err, "unable to wait for CRDs")
+		os.Exit(1)
+	}
+
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")
 		os.Exit(1)
diff --git a/addons/pkg/crdwait/crd_wait.go b/addons/pkg/crdwait/crd_wait.go
index 01fd5a4df7..6ec9efe546 100644
--- a/addons/pkg/crdwait/crd_wait.go
+++ b/addons/pkg/crdwait/crd_wait.go
@@ -10,6 +10,9 @@ import (
 
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
+	apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -17,14 +20,14 @@ import (
 	"k8s.io/client-go/kubernetes"
 	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-type clientSetProvider func() (kubernetes.Interface, error)
-
 // CRDWaiter waits for api-resources and emits logs and events for missing resources
 type CRDWaiter struct {
 	Ctx           context.Context
-	ClientSetFn   clientSetProvider
+	ClientSet     kubernetes.Interface
+	APIReader     client.Reader
 	Logger        logr.Logger
 	Scheme        *runtime.Scheme
 	PollInterval  time.Duration
@@ -34,12 +37,7 @@ type CRDWaiter struct {
 
 // WaitForCRDs checks if CRDs are available by polling for resources.
 func (c *CRDWaiter) WaitForCRDs(crds map[schema.GroupVersion]*sets.String, object runtime.Object, controllerName string) error {
-	cs, err := c.ClientSetFn()
-	if err != nil {
-		return err
-	}
-
-	eventRecorder, eventBroadcaster := c.getEventRecorder(cs, controllerName)
+	eventRecorder, eventBroadcaster := c.getEventRecorder(c.ClientSet, controllerName)
 	go func() {
 		<-c.Ctx.Done()
 		eventBroadcaster.Shutdown()
@@ -47,11 +45,6 @@ func (c *CRDWaiter) WaitForCRDs(crds map[schema.GroupVersion]*sets.String, objec
 
 	poller := func() (done bool, err error) {
 		// Create new clientset for every invocation to avoid caching of resources
-		cs, err := c.ClientSetFn()
-		if err != nil {
-			return false, err
-		}
-
 		allFound := true
 		for gv, resources := range crds {
 			// All resources found, do nothing
@@ -59,21 +52,31 @@ func (c *CRDWaiter) WaitForCRDs(crds map[schema.GroupVersion]*sets.String, objec
 				delete(crds, gv)
 				continue
 			}
-
-			// Get the Resources for this GroupVersion
 			groupVersion := gv.String()
-
-			resourceList, err := cs.Discovery().ServerResourcesForGroupVersion(groupVersion)
-			if err != nil {
-				c.Logger.Info("error retrieving GroupVersion", "GroupVersion", groupVersion)
-				eventRecorder.Eventf(object, corev1.EventTypeWarning,
-					"Polling for GroupVersion", "The GroupVersion '%s' is not available yet", groupVersion)
-				return false, nil
-			}
-
-			// Remove each found resource from the resources set that we are waiting for
-			for i := 0; i < len(resourceList.APIResources); i++ {
-				resources.Delete(resourceList.APIResources[i].Name)
+			for _, resource := range resources.List() {
+				// Get the Resources for this GroupVersion
+
+				res := gv.WithResource(resource)
+				crd := unstructured.Unstructured{}
+				crd.SetGroupVersionKind(apiextensions.SchemeGroupVersion.WithKind("CustomResourceDefinition"))
+				name := res.GroupResource().String()
+				err := c.APIReader.Get(c.Ctx,
+					client.ObjectKey{Name: name},
+					&crd)
+
+				if apierrors.IsNotFound(err) {
+					c.Logger.Info("CRD not available yet", "name", name)
+					eventRecorder.Eventf(object, corev1.EventTypeWarning,
+						"Polling for CRD", "The CRD '%s' is not available yet", name)
+					return false, nil
+				}
+				if err != nil {
+					c.Logger.Error(err, "error retrieving CRD", "name", name)
+					eventRecorder.Eventf(object, corev1.EventTypeWarning,
+						"Error retrieving CRD %s", "Error trying to read CRD '%s'. Exiting", name)
+					return false, err
+				}
+				resources.Delete(resource)
 			}
 
 			// Still waiting on some resources in this group version


Test patch:
diff --git a/addons/controllers/suite_test.go b/addons/controllers/suite_test.go
index d4522cd40e..230c9bc42d 100644
--- a/addons/controllers/suite_test.go
+++ b/addons/controllers/suite_test.go
@@ -18,12 +18,14 @@ import (
 	. "github.com/onsi/gomega"
 	capociv1beta1 "github.com/oracle/cluster-api-provider-oci/api/v1beta1"
 	corev1 "k8s.io/api/core/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/selection"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/fake"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	capvv1beta1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1"
@@ -33,6 +35,7 @@ import (
 	controlplanev1beta1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	ctrlruntimefake "sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
@@ -234,6 +237,9 @@ var _ = BeforeSuite(func(done Done) {
 	err = topologyv1alpha1.AddToScheme(scheme)
 	Expect(err).NotTo(HaveOccurred())
 
+	err = apiextensionsv1.AddToScheme(scheme)
+	Expect(err).ToNot(HaveOccurred())
+
 	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme})
 	Expect(err).ToNot(HaveOccurred())
 	Expect(k8sClient).ToNot(BeNil())
@@ -262,18 +268,24 @@ var _ = BeforeSuite(func(done Done) {
 	Expect(err).ToNot(HaveOccurred())
 
 	ctx, cancel = context.WithCancel(ctx)
+
+	initObjs := []client.Object{}
+	fakeApiReader := ctrlruntimefake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).Build()
+
 	crdwaiter := crdwait.CRDWaiter{
-		Ctx: ctx,
-		ClientSetFn: func() (kubernetes.Interface, error) {
-			return kubernetes.NewForConfig(cfg)
-		},
+		Ctx:          ctx,
+		ClientSet:    fake.NewSimpleClientset(),
+		APIReader:    fakeApiReader,
 		Logger:       setupLog,
 		Scheme:       scheme,
 		PollInterval: constants.CRDWaitPollInterval,
 		PollTimeout:  constants.CRDWaitPollTimeout,
 	}
 
-	if err := crdwaiter.WaitForCRDs(GetExternalCRDs(),
+	crds, initObjs := GetExternalCRDs()
+	crdwaiter.APIReader = ctrlruntimefake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).Build()
+
+	if err := crdwaiter.WaitForCRDs(crds,
 		&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "test-pod", Namespace: "default"}},
 		constants.AddonControllerName,
 	); err != nil {
diff --git a/addons/pkg/crdwait/crd_wait_test.go b/addons/pkg/crdwait/crd_wait_test.go
index ee89f0c19c..59b950a981 100644
--- a/addons/pkg/crdwait/crd_wait_test.go
+++ b/addons/pkg/crdwait/crd_wait_test.go
@@ -11,6 +11,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -23,6 +24,8 @@ import (
 	"k8s.io/klog/v2/klogr"
 	clusterapiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	controlplanev1beta1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	ctrlruntimefake "sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	kappctrl "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1"
 	kapppkg "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/packaging/v1alpha1"
@@ -30,6 +33,7 @@ import (
 )
 
 var fakeClientSet *fake.Clientset
+var fakeApiReader client.Reader
 
 var _ = Describe("WaitForCRDs", func() {
 
@@ -37,7 +41,10 @@ var _ = Describe("WaitForCRDs", func() {
 	var crdWaiter CRDWaiter
 	var fakeRecorder *record.FakeRecorder
 	scheme = runtime.NewScheme()
-	_ = clientgoscheme.AddToScheme(scheme)
+	err := clientgoscheme.AddToScheme(scheme)
+	Expect(err).ToNot(HaveOccurred())
+	err = apiextensionsv1.AddToScheme(scheme)
+	Expect(err).ToNot(HaveOccurred())
 	pod := &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "foo",
@@ -46,11 +53,16 @@ var _ = Describe("WaitForCRDs", func() {
 
 	BeforeEach(func() {
 		fakeClientSet = fake.NewSimpleClientset(pod)
+
+		initObjs := []client.Object{pod.DeepCopy()}
+		fakeApiReader := ctrlruntimefake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).Build()
+
 		crdWaiter = CRDWaiter{
-			Ctx:         ctx,
-			ClientSetFn: getFakeClientSet,
-			Logger:      klogr.New(),
-			Scheme:      scheme,
+			Ctx:       ctx,
+			ClientSet: fakeClientSet,
+			APIReader: fakeApiReader,
+			Logger:    klogr.New(),
+			Scheme:    scheme,
 		}
 		fakeRecorder = record.NewFakeRecorder(50)
 		crdWaiter.eventRecorder = fakeRecorder
@@ -95,8 +107,14 @@ var _ = Describe("WaitForCRDs", func() {
 			crdWaiter.PollInterval = time.Second
 			crdWaiter.PollTimeout = time.Second
 
-			Expect(crdWaiter.WaitForCRDs(crds, pod, "foo")).NotTo(HaveOccurred())
+			clusterCRD := &apiextensionsv1.CustomResourceDefinition{
+				ObjectMeta: metav1.ObjectMeta{Name: "clusters.cluster.x-k8s.io"},
+				Spec:       apiextensionsv1.CustomResourceDefinitionSpec{Group: "cluster.x-k8s.io", Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{Name: "v1beta1"}}}}
+
+			initObjs := []client.Object{clusterCRD.DeepCopy(), pod.DeepCopy()}
+			crdWaiter.APIReader = ctrlruntimefake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).Build()
 
+			Expect(crdWaiter.WaitForCRDs(crds, pod, "foo")).NotTo(HaveOccurred())
 		})
 	})
 
@@ -138,6 +156,42 @@ var _ = Describe("WaitForCRDs", func() {
 			crdWaiter.PollInterval = time.Second
 			crdWaiter.PollTimeout = time.Second
 
+			clusterCRD := &apiextensionsv1.CustomResourceDefinition{
+				ObjectMeta: metav1.ObjectMeta{Name: "clusters.cluster.x-k8s.io"},
+				Spec:       apiextensionsv1.CustomResourceDefinitionSpec{Group: "cluster.x-k8s.io", Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{Name: "v1beta1"}}},
+			}
+
+			kubeadmcontrolplaneCRD := &apiextensionsv1.CustomResourceDefinition{
+				ObjectMeta: metav1.ObjectMeta{Name: "kubeadmcontrolplanes.controlplane.cluster.x-k8s.io"},
+			}
+
+			PackageInstallCRD := &apiextensionsv1.CustomResourceDefinition{
+				ObjectMeta: metav1.ObjectMeta{Name: "packageinstalls.packaging.carvel.dev"},
+			}
+
+			tanzukubernetesreleaseCRD := &apiextensionsv1.CustomResourceDefinition{
+				ObjectMeta: metav1.ObjectMeta{Name: "tanzukubernetesreleases.run.tanzu.vmware.com"},
+			}
+
+			packagerepositorieCRD := &apiextensionsv1.CustomResourceDefinition{
+				ObjectMeta: metav1.ObjectMeta{Name: "packagerepositories.packaging.carvel.dev"},
+			}
+
+			appCRD := &apiextensionsv1.CustomResourceDefinition{
+				ObjectMeta: metav1.ObjectMeta{Name: "apps.kappctrl.k14s.io"},
+			}
+
+			initObjs := []client.Object{
+				clusterCRD.DeepCopy(),
+				kubeadmcontrolplaneCRD.DeepCopy(),
+				PackageInstallCRD.DeepCopy(),
+				tanzukubernetesreleaseCRD.DeepCopy(),
+				packagerepositorieCRD.DeepCopy(),
+				appCRD.DeepCopy(),
+				pod.DeepCopy(),
+			}
+			crdWaiter.APIReader = ctrlruntimefake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).Build()
+
 			Expect(crdWaiter.WaitForCRDs(getCRDs(), pod, "foo")).NotTo(HaveOccurred())
 		})
 	})
@@ -153,14 +207,28 @@ var _ = Describe("WaitForCRDs", func() {
 				return true, nil, nil
 			})
 
+			clusterCRD := &apiextensionsv1.CustomResourceDefinition{
+				ObjectMeta: metav1.ObjectMeta{Name: "clusters.cluster.x-k8s.io"},
+				Spec:       apiextensionsv1.CustomResourceDefinitionSpec{Group: "cluster.x-k8s.io", Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{Name: "v1beta1"}}},
+			}
+			initObjs := []client.Object{clusterCRD.DeepCopy(), pod.DeepCopy()}
+			client := ctrlruntimefake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).Build()
+			err = client.Delete(ctx, clusterCRD)
+			Expect(err).ToNot(HaveOccurred())
+
 			go func() {
 				time.Sleep(time.Second * 3)
+
 				fakeClientSet.Resources = append(fakeClientSet.Resources,
 					&metav1.APIResourceList{GroupVersion: clusterapiv1beta1.GroupVersion.String(),
 						APIResources: []metav1.APIResource{
 							{Name: "clusters", Namespaced: true, Kind: "Cluster"},
 						},
 					})
+
+				err = client.Create(ctx, clusterCRD)
+				Expect(err).ToNot(HaveOccurred())
+				crdWaiter.APIReader = client
 			}()
 
 			crdWaiter.PollInterval = time.Second
@@ -182,29 +250,7 @@ var _ = Describe("WaitForCRDs", func() {
 			crdWaiter.PollTimeout = time.Second * 2
 
 			Expect(crdWaiter.WaitForCRDs(crds, pod, "foo")).To(HaveOccurred())
-			Expect(<-fakeRecorder.Events).To(ContainSubstring(fmt.Sprintf("The GroupVersion '%s' is not available yet", clusterapiv1beta1.GroupVersion.String())))
-
-		})
-	})
-
-	Context("when GroupVersion exists but not all api-resources exist ", func() {
-		It("should fail and emit events for missing api-resources", func() {
-			var crds = map[schema.GroupVersion]*sets.String{}
-			// cluster-api
-			clusterapiv1alpha3Resources := sets.NewString("clusters")
-			crds[clusterapiv1beta1.GroupVersion] = &clusterapiv1alpha3Resources
-
-			fakeClientSet.Resources = append(fakeClientSet.Resources,
-				&metav1.APIResourceList{GroupVersion: clusterapiv1beta1.GroupVersion.String(),
-					APIResources: []metav1.APIResource{
-						{Name: "foo", Namespaced: true, Kind: "Cluster"},
-					},
-				})
-			crdWaiter.PollInterval = time.Second
-			crdWaiter.PollTimeout = time.Second * 2
-
-			Expect(crdWaiter.WaitForCRDs(crds, pod, "foo")).To(HaveOccurred())
-			Expect(<-fakeRecorder.Events).To(ContainSubstring(fmt.Sprintf("The api-resources '[clusters]' in GroupVersion '%s' are not available yet", clusterapiv1beta1.GroupVersion.String())))
+			Expect(<-fakeRecorder.Events).To(ContainSubstring(fmt.Sprintf("The CRD 'clusters.cluster.x-k8s.io' is not available yet")))
 
 		})
 	})