Repository: vmware-tanzu/tanzu-framework (language: go)

Issue:
TC05: Ability to create CC guest cluster on supervisor with calico CNI and verify it is successful
Calico deployment (Guest cluster)


Discussion / hints:

Fix patch:
diff --git a/pkg/v1/tkg/client/cluster.go b/pkg/v1/tkg/client/cluster.go
index ba1a7431f5..d43c3edc51 100644
--- a/pkg/v1/tkg/client/cluster.go
+++ b/pkg/v1/tkg/client/cluster.go
@@ -237,13 +237,14 @@ func (c *TkgClient) waitForClusterCreation(regionalClusterClient clusterclient.C
 
 	c.WaitForAutoscalerDeployment(regionalClusterClient, options.ClusterName, options.TargetNamespace)
 
-	workloadClusterClient, err := clusterclient.NewClient(workloadClusterKubeconfigPath, kubeContext, clusterclient.Options{OperationTimeout: 15 * time.Minute})
-	if err != nil {
-		return errors.Wrap(err, "unable to create workload cluster client")
-	}
-
 	if !isTKGSCluster {
 		log.Info("Waiting for addons installation...")
+
+		workloadClusterClient, err := clusterclient.NewClient(workloadClusterKubeconfigPath, kubeContext, clusterclient.Options{OperationTimeout: 15 * time.Minute})
+		if err != nil {
+			return errors.Wrap(err, "unable to create workload cluster client")
+		}
+
 		if err := c.WaitForAddons(waitForAddonsOptions{
 			regionalClusterClient: regionalClusterClient,
 			workloadClusterClient: workloadClusterClient,


Test patch:
diff --git a/pkg/v1/tkg/test/tkgctl/tkgs/tkgs_suite_test.go b/pkg/v1/tkg/test/tkgctl/tkgs/tkgs_suite_test.go
index c0ba03d045..436c7bd285 100644
--- a/pkg/v1/tkg/test/tkgctl/tkgs/tkgs_suite_test.go
+++ b/pkg/v1/tkg/test/tkgctl/tkgs/tkgs_suite_test.go
@@ -90,9 +90,9 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 		Plan:       "dev",
 	}
 	tkgctlClient, err = tkgctl.New(tkgctlOptions)
-	var isTKGS bool
-	isTKGS, err = tkgctlClient.IsPacificRegionalCluster()
 	Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("failed to connect cluster with given input kube config file:%v and context:%v, reason: %v", e2eConfig.TKGSKubeconfigPath, e2eConfig.TKGSKubeconfigContext, err))
+	isTKGS, err := tkgctlClient.IsPacificRegionalCluster()
+	Expect(err).ShouldNot(HaveOccurred())
 	Expect(isTKGS).To(Equal(true), fmt.Sprintf("the input kube config file:%v with given context:%v is not TKGS cluster", e2eConfig.TKGSKubeconfigPath, e2eConfig.TKGSKubeconfigContext))
 
 	featureGateHelper := tkgctlClient.FeatureGateHelper()
diff --git a/pkg/v1/tkg/test/tkgctl/tkgs/tkgs_workload_cluster_test.go b/pkg/v1/tkg/test/tkgctl/tkgs/tkgs_workload_cluster_test.go
index 764e7c0342..efe569483b 100644
--- a/pkg/v1/tkg/test/tkgctl/tkgs/tkgs_workload_cluster_test.go
+++ b/pkg/v1/tkg/test/tkgctl/tkgs/tkgs_workload_cluster_test.go
@@ -17,7 +17,11 @@ import (
 	"github.com/vmware-tanzu/tanzu-framework/pkg/v1/tkg/tkgctl"
 )
 
-const TKC_KIND = "kind: TanzuKubernetesCluster"
+const (
+	TKC_KIND  = "kind: TanzuKubernetesCluster"
+	cniAntrea = "antrea"
+	cniCalico = "calico"
+)
 
 var _ = Describe("TKGS - Create TKC based workload cluster tests", func() {
 	var (
@@ -46,13 +50,31 @@ var _ = Describe("TKGS - Create TKC based workload cluster tests", func() {
 		AfterEach(func() {
 			defer os.Remove(clusterOptions.ClusterConfigFile)
 		})
+
 		Context("when cluster Plan is dev", func() {
 			BeforeEach(func() {
 				e2eConfig.WorkloadClusterOptions.ClusterPlan = "dev"
 			})
-			When("create cluster is invoked", func() {
+
+			When("create cluster is invoked with CNI Antrea", func() {
+				BeforeEach(func() {
+					clusterOptions.CniType = cniAntrea
+				})
+				AfterEach(func() {
+					err = tkgctlClient.DeleteCluster(deleteClusterOptions)
+				})
+				It("should create TKC Workload Cluster and delete it", func() {
+					Expect(err).To(BeNil())
+				})
+			})
+
+			When("create cluster is invoked with CNI Calico", func() {
+				BeforeEach(func() {
+					clusterOptions.CniType = cniCalico
+				})
 				AfterEach(func() {
 					err = tkgctlClient.DeleteCluster(deleteClusterOptions)
+					clusterOptions.CniType = cniAntrea
 				})
 				It("should create TKC Workload Cluster and delete it", func() {
 					Expect(err).To(BeNil())
@@ -82,15 +104,32 @@ var _ = Describe("TKGS - Create TKC based workload cluster tests", func() {
 				})
 			})
 		})
+
 		Context("when cluster Plan is prod", func() {
 			BeforeEach(func() {
 				clusterOptions.Plan = "prod"
 				clusterOptions.GenerateOnly = false
 			})
 
-			When("create cluster is invoked", func() {
+			When("create cluster is invoked with CNI Antrea", func() {
+				BeforeEach(func() {
+					clusterOptions.CniType = cniAntrea
+				})
+				AfterEach(func() {
+					err = tkgctlClient.DeleteCluster(deleteClusterOptions)
+				})
+				It("should create TKC Workload Cluster and delete it", func() {
+					Expect(err).To(BeNil())
+				})
+			})
+
+			When("create cluster is invoked with CNI Calico", func() {
+				BeforeEach(func() {
+					clusterOptions.CniType = cniCalico
+				})
 				AfterEach(func() {
 					err = tkgctlClient.DeleteCluster(deleteClusterOptions)
+					clusterOptions.CniType = cniAntrea
 				})
 				It("should create TKC Workload Cluster and delete it", func() {
 					Expect(err).To(BeNil())
diff --git a/pkg/v1/tkg/test/tkgctl/tkgs_cc/tkgs_cc_suite_test.go b/pkg/v1/tkg/test/tkgctl/tkgs_cc/tkgs_cc_suite_test.go
index 00bbafc88f..bb9bd43ce9 100644
--- a/pkg/v1/tkg/test/tkgctl/tkgs_cc/tkgs_cc_suite_test.go
+++ b/pkg/v1/tkg/test/tkgctl/tkgs_cc/tkgs_cc_suite_test.go
@@ -17,6 +17,7 @@ import (
 	"github.com/onsi/ginkgo/reporters"
 	. "github.com/onsi/gomega"
 	"github.com/pkg/errors"
+	"sigs.k8s.io/cluster-api/util"
 
 	"github.com/vmware-tanzu/tanzu-framework/pkg/v1/tkg/constants"
 	"github.com/vmware-tanzu/tanzu-framework/pkg/v1/tkg/test/framework"
@@ -90,9 +91,9 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 		Plan:       "dev",
 	}
 	tkgctlClient, err = tkgctl.New(tkgctlOptions)
-	var isTKGS bool
-	isTKGS, err = tkgctlClient.IsPacificRegionalCluster()
 	Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("failed to connect cluster with given input kube config file:%v and context:%v, reason: %v", e2eConfig.TKGSKubeconfigPath, e2eConfig.TKGSKubeconfigContext, err))
+	isTKGS, err := tkgctlClient.IsPacificRegionalCluster()
+	Expect(err).ShouldNot(HaveOccurred())
 	Expect(isTKGS).To(Equal(true), fmt.Sprintf("the input kube config file:%v with given context:%v is not TKGS cluster", e2eConfig.TKGSKubeconfigPath, e2eConfig.TKGSKubeconfigContext))
 
 	featureGateHelper := tkgctlClient.FeatureGateHelper()
@@ -156,3 +157,27 @@ func createClusterConfigFile(e2eConfig *framework.E2EConfig) string {
 	Expect(err).To(BeNil())
 	return clusterConfigFile
 }
+
+// getCalicoCNIClusterClassFile return temporary Cluster Class config file with Calico CNI
+func getCalicoCNIClusterClassFile(e2eConfig *framework.E2EConfig) string {
+	clusterConfigFile := e2eConfig.WorkloadClusterOptions.ClusterClassFilePath
+
+	_, err := os.Stat(clusterConfigFile)
+	Expect(err).To(BeNil())
+
+	yamlFile, err := os.ReadFile(clusterConfigFile)
+	Expect(err).To(BeNil())
+
+	f, err := os.CreateTemp("", "temp_calico_cni_cluster_config_"+util.RandomString(4)+".yaml") // nolint:gomnd
+	Expect(err).To(BeNil())
+
+	_, err = f.Write([]byte(fmt.Sprintf(cniCalicoCBResource, e2eConfig.WorkloadClusterOptions.ClusterName,
+		e2eConfig.WorkloadClusterOptions.Namespace, e2eConfig.TkrVersion, e2eConfig.WorkloadClusterOptions.ClusterName,
+		e2eConfig.WorkloadClusterOptions.Namespace, e2eConfig.WorkloadClusterOptions.ClusterName) + string(yamlFile)))
+	Expect(err).To(BeNil())
+
+	calicoCNIClusterConfigFile, err := filepath.Abs(f.Name())
+	Expect(err).To(BeNil())
+
+	return calicoCNIClusterConfigFile
+}
diff --git a/pkg/v1/tkg/test/tkgctl/tkgs_cc/tkgs_cc_workload_cluster_test.go b/pkg/v1/tkg/test/tkgctl/tkgs_cc/tkgs_cc_workload_cluster_test.go
index 512b5b71d8..6f61b6f3b2 100644
--- a/pkg/v1/tkg/test/tkgctl/tkgs_cc/tkgs_cc_workload_cluster_test.go
+++ b/pkg/v1/tkg/test/tkgctl/tkgs_cc/tkgs_cc_workload_cluster_test.go
@@ -14,7 +14,11 @@ import (
 	"github.com/vmware-tanzu/tanzu-framework/pkg/v1/tkg/tkgctl"
 )
 
-const TKC_KIND = "kind: TanzuKubernetesCluster"
+const (
+	TKC_KIND  = "kind: TanzuKubernetesCluster"
+	cniAntrea = "antrea"
+	cniCalico = "calico"
+)
 
 var _ = Describe("TKGS ClusterClass based workload cluster tests", func() {
 	var (
@@ -32,7 +36,7 @@ var _ = Describe("TKGS ClusterClass based workload cluster tests", func() {
 		err = tkgctlClient.CreateCluster(clusterOptions)
 	})
 
-	Context("when input file is cluster class based", func() {
+	Context("when input file is cluster class based with CNI Antrea", func() {
 		BeforeEach(func() {
 			clusterName, namespace = ValidateClusterClassConfigFile(e2eConfig.WorkloadClusterOptions.ClusterClassFilePath)
 			e2eConfig.WorkloadClusterOptions.Namespace = namespace
@@ -41,10 +45,38 @@ var _ = Describe("TKGS ClusterClass based workload cluster tests", func() {
 			clusterOptions.ClusterConfigFile = e2eConfig.WorkloadClusterOptions.ClusterClassFilePath
 			clusterOptions.ClusterName = e2eConfig.WorkloadClusterOptions.ClusterName
 			clusterOptions.Namespace = e2eConfig.WorkloadClusterOptions.Namespace
+			clusterOptions.CniType = cniAntrea
+		})
+
+		AfterEach(func() {
+			err = tkgctlClient.DeleteCluster(deleteClusterOptions)
+		})
+
+		It("should successfully create a cluster", func() {
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+
+	Context("when input file is cluster class based with CNI Calico", func() {
+		BeforeEach(func() {
+			clusterName, namespace = ValidateClusterClassConfigFile(e2eConfig.WorkloadClusterOptions.ClusterClassFilePath)
+			e2eConfig.WorkloadClusterOptions.Namespace = namespace
+			e2eConfig.WorkloadClusterOptions.ClusterName = clusterName
+			deleteClusterOptions = getDeleteClustersOptions(e2eConfig)
+			clusterOptions.ClusterName = e2eConfig.WorkloadClusterOptions.ClusterName
+			clusterOptions.Namespace = e2eConfig.WorkloadClusterOptions.Namespace
+			clusterOptions.CniType = cniCalico
+			// use a temporary cluster class config file with CalicoConfig and ClusterBoostrap resources to
+			// customize the CNI option as Calico
+			clusterOptions.ClusterConfigFile = getCalicoCNIClusterClassFile(e2eConfig)
 		})
 
 		AfterEach(func() {
 			err = tkgctlClient.DeleteCluster(deleteClusterOptions)
+			clusterOptions.CniType = cniAntrea
+			// remove the temporary cluster config file
+			os.Remove(clusterOptions.ClusterConfigFile)
+			clusterOptions.ClusterConfigFile = e2eConfig.WorkloadClusterOptions.ClusterClassFilePath
 		})
 
 		It("should successfully create a cluster", func() {
diff --git a/pkg/v1/tkg/test/tkgctl/tkgs_cc/utils.go b/pkg/v1/tkg/test/tkgctl/tkgs_cc/utils.go
new file mode 100644
index 0000000000..45b4be5ca8
--- /dev/null
+++ b/pkg/v1/tkg/test/tkgctl/tkgs_cc/utils.go
@@ -0,0 +1,38 @@
+// Copyright 2022 VMware, Inc. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package tkgs_cc
+
+// cniCalicoCBResource declares resources required along with the classy cluster
+// to deploy a workload cluster with CNI Calico
+const cniCalicoCBResource = `apiVersion: cni.tanzu.vmware.com/v1alpha1
+kind: CalicoConfig
+metadata:
+  name: %s
+  namespace: %s
+spec:
+  calico:
+    config:
+      vethMTU: 0
+---
+apiVersion: run.tanzu.vmware.com/v1alpha3
+kind: ClusterBootstrap
+metadata:
+  annotations:
+    tkg.tanzu.vmware.com/add-missing-fields-from-tkr: %s
+  name: %s
+  namespace: %s
+spec:
+  additionalPackages:
+  - refName: metrics-server*
+  - refName: secretgen-controller*
+  - refName: pinniped*
+  cni:
+    refName: calico*
+    valuesFrom:
+      providerRef:
+        apiGroup: cni.tanzu.vmware.com
+        kind: CalicoConfig
+        name: %s
+---
+`