Repository: argoproj-labs/applicationset (go)

Issue:
Files generator fails with LFS enabled repo due to missing git-lfs package
Attempting to use the files generator with a LFS enabled repo fails with the following:

```
time="2021-04-27T11:05:08Z" level=info msg="git fetch origin --tags --force" dir=/tmp/git@gitlab.com_REPO_NAME execID=pvFKK
time="2021-04-27T11:05:10Z" level=info msg=Trace args="[git fetch origin --tags --force]" dir=/tmp/git@gitlab.com_REPO_NAME operation_name="exec git" time_ms=1449.213518
time="2021-04-27T11:05:10Z" level=info msg="git lfs ls-files -n" dir=/tmp/git@gitlab.com_REPO_NAME execID=Z1Bab
time="2021-04-27T11:05:10Z" level=error msg="`git lfs ls-files -n` failed exit status 1: git: 'lfs' is not a git command. See 'git --help'.\n\nThe most similar command is\n\tlog" execID=Z1Bab
time="2021-04-27T11:05:10Z" level=info msg=Trace args="[git lfs ls-files -n]" dir=/tmp/git@gitlab.com_REPO_NAME operation_name="exec git" time_ms=6.200604
t
```

Installing `git-lfs` on the pod of course resolves the issue, I'll submit a PR to add it to the container.

Pull request description:
Add git-lfs to resolve error using files generator
Adds git-lfs to resolve #214 

Patch:
diff --git a/Dockerfile b/Dockerfile
index 57cfcde..755ff5a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -21,7 +21,7 @@ FROM ubuntu:20.10
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get upgrade -y && \
-  apt-get install -y git-all gpg && \
+  apt-get install -y git-all git-lfs gpg && \
   rm -r /var/lib/apt/lists /var/cache/apt/archives
 
 # Add Argo CD helper scripts that are required by 'github.com/argoproj/argo-cd/util/git' package
diff --git a/go.mod b/go.mod
index 4e9ff03..5854193 100644
--- a/go.mod
+++ b/go.mod
@@ -16,6 +16,7 @@ require (
 	k8s.io/api v0.19.2
 	k8s.io/apimachinery v0.19.2
 	k8s.io/client-go v11.0.1-0.20190816222228-6d55c1b1f1ca+incompatible
+	k8s.io/utils v0.0.0-20200912215256-4140de9c8800
 	sigs.k8s.io/controller-runtime v0.7.0
 )
 
diff --git a/main.go b/main.go
index 6adda40..2781e6e 100644
--- a/main.go
+++ b/main.go
@@ -147,6 +147,7 @@ func main() {
 		Renderer:         &utils.Render{},
 		Policy:           policyObj,
 		ArgoAppClientset: appclientset,
+		KubeClientset:    k8s,
 		ArgoDB:           argoCDDB,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "ApplicationSet")
diff --git a/pkg/controllers/applicationset_controller.go b/pkg/controllers/applicationset_controller.go
index d1064da..a612105 100644
--- a/pkg/controllers/applicationset_controller.go
+++ b/pkg/controllers/applicationset_controller.go
@@ -1,6 +1,4 @@
 /*
-
-
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
@@ -29,11 +27,11 @@ import (
 	"github.com/argoproj-labs/applicationset/pkg/utils"
 	"github.com/argoproj/argo-cd/common"
 	argov1alpha1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
-	"github.com/argoproj/argo-cd/util/argo"
 	"github.com/argoproj/argo-cd/util/db"
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
@@ -69,6 +67,7 @@ type ApplicationSetReconciler struct {
 	Generators       map[string]generators.Generator
 	ArgoDB           db.ArgoDB
 	ArgoAppClientset appclientset.Interface
+	KubeClientset    kubernetes.Interface
 	utils.Policy
 	utils.Renderer
 }
@@ -163,7 +162,7 @@ func (r *ApplicationSetReconciler) validateGeneratedApplications(ctx context.Con
 			return err
 		}
 
-		if err := argoutil.ValidateDestination(ctx, &app.Spec.Destination, r.ArgoDB); err != nil {
+		if err := utils.ValidateDestination(ctx, &app.Spec.Destination, r.KubeClientset, namespace); err != nil {
 			return fmt.Errorf("application destination spec is invalid: %s", err.Error())
 		}
 
@@ -172,7 +171,7 @@ func (r *ApplicationSetReconciler) validateGeneratedApplications(ctx context.Con
 			return err
 		}
 		if len(conditions) > 0 {
-			return fmt.Errorf("application spec is invalid: %s", argo.FormatAppConditions(conditions))
+			return fmt.Errorf("application spec is invalid: %s", argoutil.FormatAppConditions(conditions))
 		}
 
 	}
@@ -573,7 +572,7 @@ func (r *ApplicationSetReconciler) removeFinalizerOnInvalidDestination(ctx conte
 
 	// If the destination is invalid (for example the cluster is no longer defined), then remove
 	// the application finalizers to avoid triggering Argo CD bug #5817
-	if err := argoutil.ValidateDestination(ctx, &app.Spec.Destination, r.ArgoDB); err != nil {
+	if err := utils.ValidateDestination(ctx, &app.Spec.Destination, r.KubeClientset, applicationSet.Namespace); err != nil {
 
 		// Filter out the Argo CD finalizer from the finalizer list
 		newFinalizers := []string{}
diff --git a/pkg/generators/cluster.go b/pkg/generators/cluster.go
index be89a57..1920728 100644
--- a/pkg/generators/cluster.go
+++ b/pkg/generators/cluster.go
@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/argoproj/argo-cd/util/db"
 	"github.com/argoproj/argo-cd/util/settings"
 	log "github.com/sirupsen/logrus"
 
@@ -15,6 +14,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	argoprojiov1alpha1 "github.com/argoproj-labs/applicationset/api/v1alpha1"
+	"github.com/argoproj-labs/applicationset/pkg/utils"
 )
 
 const (
@@ -32,7 +32,6 @@ type ClusterGenerator struct {
 	// namespace is the Argo CD namespace
 	namespace       string
 	settingsManager *settings.SettingsManager
-	argoDB          db.ArgoDB
 }
 
 func NewClusterGenerator(c client.Client, ctx context.Context, clientset kubernetes.Interface, namespace string) Generator {
@@ -45,7 +44,6 @@ func NewClusterGenerator(c client.Client, ctx context.Context, clientset kuberne
 		clientset:       clientset,
 		namespace:       namespace,
 		settingsManager: settingsManager,
-		argoDB:          db.NewDB(namespace, settingsManager, clientset),
 	}
 	return g
 }
@@ -74,7 +72,7 @@ func (g *ClusterGenerator) GenerateParams(
 	ignoreLocalClusters := len(appSetGenerator.Clusters.Selector.MatchExpressions) > 0 || len(appSetGenerator.Clusters.Selector.MatchLabels) > 0
 
 	// ListCluster from Argo CD's util/db package will include the local cluster in the list of clusters
-	clustersFromArgoCD, err := g.argoDB.ListClusters(g.ctx)
+	clustersFromArgoCD, err := utils.ListClusters(g.ctx, g.clientset, g.namespace)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/utils/clusterUtils.go b/pkg/utils/clusterUtils.go
new file mode 100644
index 0000000..987111d
--- /dev/null
+++ b/pkg/utils/clusterUtils.go
@@ -0,0 +1,186 @@
+package utils
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/argoproj/argo-cd/common"
+	appv1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
+	log "github.com/sirupsen/logrus"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/utils/pointer"
+)
+
+// The contents of this file are from
+// github.com/argoproj/argo-cd/util/db/cluster.go
+//
+// The main difference is that ListClusters(...) calls the kubeclient directly,
+// via `g.clientset.CoreV1().Secrets`, rather than using the `db.listClusterSecrets()``
+// which appears to have a race condition on when it is called.
+//
+// I was reminded of this issue that I opened, which might be related:
+// https://github.com/argoproj/argo-cd/issues/4755
+//
+// I hope to upstream this change in some form, so that we do not need to worry about
+// Argo CD changing the logic on us.
+
+var (
+	localCluster = appv1.Cluster{
+		Name:            "in-cluster",
+		Server:          common.KubernetesInternalAPIServerAddr,
+		ConnectionState: appv1.ConnectionState{Status: appv1.ConnectionStatusSuccessful},
+	}
+	initLocalCluster sync.Once
+)
+
+const (
+	ArgoCDSecretTypeLabel   = "argocd.argoproj.io/secret-type"
+	ArgoCDSecretTypeCluster = "cluster"
+)
+
+func ListClusters(ctx context.Context, clientset kubernetes.Interface, namespace string) (*appv1.ClusterList, error) {
+
+	clusterSecretsList, err := clientset.CoreV1().Secrets(namespace).List(ctx, metav1.ListOptions{LabelSelector: common.LabelKeySecretType + "=" + common.LabelValueSecretTypeCluster})
+	if err != nil {
+		return nil, err
+	}
+
+	if clusterSecretsList == nil {
+		return nil, nil
+	}
+
+	clusterSecrets := clusterSecretsList.Items
+
+	clusterList := appv1.ClusterList{
+		Items: make([]appv1.Cluster, len(clusterSecrets)),
+	}
+	hasInClusterCredentials := false
+	for i, clusterSecret := range clusterSecrets {
+		cluster := *secretToCluster(&clusterSecret)
+		clusterList.Items[i] = cluster
+		if cluster.Server == common.KubernetesInternalAPIServerAddr {
+			hasInClusterCredentials = true
+		}
+	}
+	if !hasInClusterCredentials {
+		localCluster := getLocalCluster(clientset)
+		if localCluster != nil {
+			clusterList.Items = append(clusterList.Items, *localCluster)
+		}
+	}
+	return &clusterList, nil
+}
+
+func getLocalCluster(clientset kubernetes.Interface) *appv1.Cluster {
+	initLocalCluster.Do(func() {
+		info, err := clientset.Discovery().ServerVersion()
+		if err == nil {
+			localCluster.ServerVersion = fmt.Sprintf("%s.%s", info.Major, info.Minor)
+			localCluster.ConnectionState = appv1.ConnectionState{Status: appv1.ConnectionStatusSuccessful}
+		} else {
+			localCluster.ConnectionState = appv1.ConnectionState{
+				Status:  appv1.ConnectionStatusFailed,
+				Message: err.Error(),
+			}
+		}
+	})
+	cluster := localCluster.DeepCopy()
+	now := metav1.Now()
+	cluster.ConnectionState.ModifiedAt = &now
+	return cluster
+}
+
+// secretToCluster converts a secret into a Cluster object
+func secretToCluster(s *corev1.Secret) *appv1.Cluster {
+	var config appv1.ClusterConfig
+	if len(s.Data["config"]) > 0 {
+		err := json.Unmarshal(s.Data["config"], &config)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	var namespaces []string
+	for _, ns := range strings.Split(string(s.Data["namespaces"]), ",") {
+		if ns = strings.TrimSpace(ns); ns != "" {
+			namespaces = append(namespaces, ns)
+		}
+	}
+	var refreshRequestedAt *metav1.Time
+	if v, found := s.Annotations[common.AnnotationKeyRefresh]; found {
+		requestedAt, err := time.Parse(time.RFC3339, v)
+		if err != nil {
+			log.Warnf("Error while parsing date in cluster secret '%s': %v", s.Name, err)
+		} else {
+			refreshRequestedAt = &metav1.Time{Time: requestedAt}
+		}
+	}
+	var shard *int64
+	if shardStr := s.Data["shard"]; shardStr != nil {
+		if val, err := strconv.Atoi(string(shardStr)); err != nil {
+			log.Warnf("Error while parsing shard in cluster secret '%s': %v", s.Name, err)
+		} else {
+			shard = pointer.Int64Ptr(int64(val))
+		}
+	}
+	cluster := appv1.Cluster{
+		ID:                 string(s.UID),
+		Server:             strings.TrimRight(string(s.Data["server"]), "/"),
+		Name:               string(s.Data["name"]),
+		Namespaces:         namespaces,
+		Config:             config,
+		RefreshRequestedAt: refreshRequestedAt,
+		Shard:              shard,
+	}
+	return &cluster
+}
+
+// ValidateDestination checks:
+// if we used destination name we infer the server url
+// if we used both name and server then we return an invalid spec error
+func ValidateDestination(ctx context.Context, dest *appv1.ApplicationDestination, clientset kubernetes.Interface, namespace string) error {
+	if dest.Name != "" {
+		if dest.Server == "" {
+			server, err := getDestinationServer(ctx, dest.Name, clientset, namespace)
+			if err != nil {
+				return fmt.Errorf("unable to find destination server: %v", err)
+			}
+			if server == "" {
+				return fmt.Errorf("application references destination cluster %s which does not exist", dest.Name)
+			}
+			dest.SetInferredServer(server)
+		} else {
+			if !dest.IsServerInferred() {
+				return fmt.Errorf("application destination can't have both name and server defined: %s %s", dest.Name, dest.Server)
+			}
+		}
+	}
+	return nil
+}
+
+func getDestinationServer(ctx context.Context, clusterName string, clientset kubernetes.Interface, namespace string) (string, error) {
+
+	clusterList, err := ListClusters(ctx, clientset, namespace)
+	if err != nil {
+		return "", err
+	}
+	var servers []string
+	for _, c := range clusterList.Items {
+		if c.Name == clusterName {
+			servers = append(servers, c.Server)
+		}
+	}
+	if len(servers) > 1 {
+		return "", fmt.Errorf("there are %d clusters with the same name: %v", len(servers), servers)
+	} else if len(servers) == 0 {
+		return "", fmt.Errorf("there are no clusters with this name: %s", clusterName)
+	}
+	return servers[0], nil
+}


Test patch:
diff --git a/pkg/controllers/applicationset_controller_test.go b/pkg/controllers/applicationset_controller_test.go
index 723435a..50a06b8 100644
--- a/pkg/controllers/applicationset_controller_test.go
+++ b/pkg/controllers/applicationset_controller_test.go
@@ -11,17 +11,15 @@ import (
 
 	"github.com/argoproj-labs/applicationset/pkg/generators"
 	"github.com/argoproj/argo-cd/common"
-	"github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
 	argov1alpha1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
 	"github.com/sirupsen/logrus"
-	log "github.com/sirupsen/logrus"
 	logtest "github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/record"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 	crtclient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -29,6 +27,7 @@ import (
 	argoprojiov1alpha1 "github.com/argoproj-labs/applicationset/api/v1alpha1"
 	appclientset "github.com/argoproj/argo-cd/pkg/client/clientset/versioned/fake"
 	dbmocks "github.com/argoproj/argo-cd/util/db/mocks"
+	kubefake "k8s.io/client-go/kubernetes/fake"
 )
 
 type generatorMock struct {
@@ -163,7 +162,8 @@ func TestExtractApplications(t *testing.T) {
 				Generators: map[string]generators.Generator{
 					"List": &generatorMock,
 				},
-				Renderer: &rendererMock,
+				Renderer:      &rendererMock,
+				KubeClientset: kubefake.NewSimpleClientset(),
 			}
 
 			got, err := r.generateApplications(argoprojiov1alpha1.ApplicationSet{
@@ -274,7 +274,8 @@ func TestMergeTemplateApplications(t *testing.T) {
 				Generators: map[string]generators.Generator{
 					"List": &generatorMock,
 				},
-				Renderer: &rendererMock,
+				Renderer:      &rendererMock,
+				KubeClientset: kubefake.NewSimpleClientset(),
 			}
 
 			got, _ := r.generateApplications(argoprojiov1alpha1.ApplicationSet{
@@ -806,7 +807,7 @@ func TestCreateOrUpdateInCluster(t *testing.T) {
 
 		t.Run(c.name, func(t *testing.T) {
 
-			initObjs := []client.Object{&c.appSet}
+			initObjs := []crtclient.Object{&c.appSet}
 
 			for _, a := range c.existingApps {
 				err = controllerutil.SetControllerReference(&c.appSet, &a, scheme)
@@ -849,18 +850,12 @@ func TestRemoveFinalizerOnInvalidDestination(t *testing.T) {
 	err = argov1alpha1.AddToScheme(scheme)
 	assert.Nil(t, err)
 
-	myCluster := argov1alpha1.Cluster{
-		Server: "https://kubernetes.default.svc",
-		Name:   "my-cluster2",
-	}
-
 	for _, c := range []struct {
 		// name is human-readable test name
 		name               string
 		existingFinalizers []string
 		expectedFinalizers []string
 	}{
-
 		{
 			name:               "no finalizers",
 			existingFinalizers: []string{},
@@ -871,7 +866,6 @@ func TestRemoveFinalizerOnInvalidDestination(t *testing.T) {
 			existingFinalizers: []string{common.ResourcesFinalizerName},
 			expectedFinalizers: nil,
 		},
-
 		{
 			name:               "contains only non-argo finalizer",
 			existingFinalizers: []string{"non-argo-finalizer"},
@@ -912,25 +906,37 @@ func TestRemoveFinalizerOnInvalidDestination(t *testing.T) {
 				},
 			}
 
-			initObjs := []client.Object{&app, &appSet}
+			initObjs := []crtclient.Object{&app, &appSet}
 
 			client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).Build()
+			secret := &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "my-secret",
+					Namespace: "namespace",
+					Labels: map[string]string{
+						generators.ArgoCDSecretTypeLabel: generators.ArgoCDSecretTypeCluster,
+					},
+				},
+				Data: map[string][]byte{
+					// Since this test requires the cluster to be an invalid destination, we
+					// always return a cluster named 'my-cluster2' (different from app 'my-cluster', above)
+					"name":   []byte("mycluster2"),
+					"server": []byte("https://kubernetes.default.svc"),
+					"config": []byte("{\"username\":\"foo\",\"password\":\"foo\"}"),
+				},
+			}
 
-			// Always return a cluster named 'my-cluster2' (different from app 'my-cluster')
-			dbMock := &dbmocks.ArgoDB{}
-			dbMock.On("GetCluster", mock.Anything, "https://kubernetes.default.svc").Return(&myCluster, nil)
-			dbMock.On("ListClusters", mock.Anything).Return(&v1alpha1.ClusterList{Items: []argov1alpha1.Cluster{
-				myCluster,
-			}}, nil)
+			objects := append([]runtime.Object{}, secret)
+			kubeclientset := kubefake.NewSimpleClientset(objects...)
 
 			r := ApplicationSetReconciler{
-				Client:   client,
-				Scheme:   scheme,
-				Recorder: record.NewFakeRecorder(10),
-				ArgoDB:   dbMock,
+				Client:        client,
+				Scheme:        scheme,
+				Recorder:      record.NewFakeRecorder(10),
+				KubeClientset: kubeclientset,
 			}
 
-			appLog := log.WithFields(log.Fields{"app": app.Name, "appSet": ""})
+			appLog := logrus.WithFields(logrus.Fields{"app": app.Name, "appSet": ""})
 
 			appInputParam := app.DeepCopy()
 
@@ -1113,7 +1119,7 @@ func TestCreateApplications(t *testing.T) {
 			},
 		},
 	} {
-		initObjs := []client.Object{&c.appSet}
+		initObjs := []crtclient.Object{&c.appSet}
 		for _, a := range c.existsApps {
 			err = controllerutil.SetControllerReference(&c.appSet, &a, scheme)
 			assert.Nil(t, err)
@@ -1255,7 +1261,7 @@ func TestDeleteInCluster(t *testing.T) {
 			},
 		},
 	} {
-		initObjs := []client.Object{&c.appSet}
+		initObjs := []crtclient.Object{&c.appSet}
 		for _, a := range c.existingApps {
 			temp := a
 			err = controllerutil.SetControllerReference(&c.appSet, &temp, scheme)
@@ -1941,9 +1947,27 @@ func TestValidateGeneratedApplications(t *testing.T) {
 
 		t.Run(cc.name, func(t *testing.T) {
 
+			secret := &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "my-secret",
+					Namespace: "namespace",
+					Labels: map[string]string{
+						generators.ArgoCDSecretTypeLabel: generators.ArgoCDSecretTypeCluster,
+					},
+				},
+				Data: map[string][]byte{
+					"name":   []byte("my-cluster"),
+					"server": []byte("https://kubernetes.default.svc"),
+					"config": []byte("{\"username\":\"foo\",\"password\":\"foo\"}"),
+				},
+			}
+
+			objects := append([]runtime.Object{}, secret)
+			kubeclientset := kubefake.NewSimpleClientset(objects...)
+
 			argoDBMock := dbmocks.ArgoDB{}
 			argoDBMock.On("GetCluster", mock.Anything, "https://kubernetes.default.svc").Return(&myCluster, nil)
-			argoDBMock.On("ListClusters", mock.Anything).Return(&v1alpha1.ClusterList{Items: []argov1alpha1.Cluster{
+			argoDBMock.On("ListClusters", mock.Anything).Return(&argov1alpha1.ClusterList{Items: []argov1alpha1.Cluster{
 				myCluster,
 			}}, nil)
 
@@ -1959,6 +1983,7 @@ func TestValidateGeneratedApplications(t *testing.T) {
 				Generators:       map[string]generators.Generator{},
 				ArgoDB:           &argoDBMock,
 				ArgoAppClientset: appclientset.NewSimpleClientset(argoObjs...),
+				KubeClientset:    kubeclientset,
 			}
 
 			appSetInfo := argoprojiov1alpha1.ApplicationSet{}
diff --git a/pkg/utils/clusterUtils_test.go b/pkg/utils/clusterUtils_test.go
new file mode 100644
index 0000000..06c7d37
--- /dev/null
+++ b/pkg/utils/clusterUtils_test.go
@@ -0,0 +1,176 @@
+package utils
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/argoproj-labs/applicationset/test/e2e/fixture/applicationsets/utils"
+	argoappv1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes/fake"
+	kubetesting "k8s.io/client-go/testing"
+)
+
+const (
+	fakeNamespace = "fake-ns"
+)
+
+// From Argo CD util/db/cluster_test.go
+func Test_secretToCluster(t *testing.T) {
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "mycluster",
+			Namespace: fakeNamespace,
+		},
+		Data: map[string][]byte{
+			"name":   []byte("test"),
+			"server": []byte("http://mycluster"),
+			"config": []byte("{\"username\":\"foo\"}"),
+		},
+	}
+	cluster := secretToCluster(secret)
+	assert.Equal(t, *cluster, argoappv1.Cluster{
+		Name:   "test",
+		Server: "http://mycluster",
+		Config: argoappv1.ClusterConfig{
+			Username: "foo",
+		},
+	})
+}
+
+// From Argo CD util/db/cluster_test.go
+func Test_secretToCluster_NoConfig(t *testing.T) {
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "mycluster",
+			Namespace: fakeNamespace,
+		},
+		Data: map[string][]byte{
+			"name":   []byte("test"),
+			"server": []byte("http://mycluster"),
+		},
+	}
+	cluster := secretToCluster(secret)
+	assert.Equal(t, *cluster, argoappv1.Cluster{
+		Name:   "test",
+		Server: "http://mycluster",
+	})
+}
+
+func createClusterSecret(secretName string, clusterName string, clusterServer string) *corev1.Secret {
+
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      secretName,
+			Namespace: utils.ArgoCDNamespace,
+			Labels: map[string]string{
+				ArgoCDSecretTypeLabel: ArgoCDSecretTypeCluster,
+			},
+		},
+		Data: map[string][]byte{
+			"name":   []byte(clusterName),
+			"server": []byte(clusterServer),
+			"config": []byte("{\"username\":\"foo\",\"password\":\"foo\"}"),
+		},
+	}
+
+	return secret
+
+}
+
+// From util/argo/argo_test.go
+// (ported to use kubeclientset)
+func TestValidateDestination(t *testing.T) {
+
+	t.Run("Validate destination with server url", func(t *testing.T) {
+
+		dest := argoappv1.ApplicationDestination{
+			Server:    "https://127.0.0.1:6443",
+			Namespace: "default",
+		}
+
+		appCond := ValidateDestination(context.Background(), &dest, nil, fakeNamespace)
+		assert.Nil(t, appCond)
+		assert.False(t, dest.IsServerInferred())
+	})
+
+	t.Run("Validate destination with server name", func(t *testing.T) {
+		dest := argoappv1.ApplicationDestination{
+			Name: "minikube",
+		}
+
+		secret := createClusterSecret("my-secret", "minikube", "https://127.0.0.1:6443")
+		objects := []runtime.Object{}
+		objects = append(objects, secret)
+		kubeclientset := fake.NewSimpleClientset(objects...)
+
+		appCond := ValidateDestination(context.Background(), &dest, kubeclientset, utils.ArgoCDNamespace)
+		assert.Nil(t, appCond)
+		assert.Equal(t, "https://127.0.0.1:6443", dest.Server)
+		assert.True(t, dest.IsServerInferred())
+	})
+
+	t.Run("Error when having both server url and name", func(t *testing.T) {
+		dest := argoappv1.ApplicationDestination{
+			Server:    "https://127.0.0.1:6443",
+			Name:      "minikube",
+			Namespace: "default",
+		}
+
+		err := ValidateDestination(context.Background(), &dest, nil, utils.ArgoCDNamespace)
+		assert.Equal(t, "application destination can't have both name and server defined: minikube https://127.0.0.1:6443", err.Error())
+		assert.False(t, dest.IsServerInferred())
+	})
+
+	t.Run("List clusters fails", func(t *testing.T) {
+		dest := argoappv1.ApplicationDestination{
+			Name: "minikube",
+		}
+		kubeclientset := fake.NewSimpleClientset()
+
+		kubeclientset.PrependReactor("list", "*", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) {
+			return true, nil, errors.New("an error occurred")
+		})
+
+		err := ValidateDestination(context.Background(), &dest, kubeclientset, utils.ArgoCDNamespace)
+		assert.Equal(t, "unable to find destination server: an error occurred", err.Error())
+		assert.False(t, dest.IsServerInferred())
+	})
+
+	t.Run("Destination cluster does not exist", func(t *testing.T) {
+		dest := argoappv1.ApplicationDestination{
+			Name: "minikube",
+		}
+
+		secret := createClusterSecret("dind", "dind", "https://127.0.0.1:6443")
+		objects := []runtime.Object{}
+		objects = append(objects, secret)
+		kubeclientset := fake.NewSimpleClientset(objects...)
+
+		err := ValidateDestination(context.Background(), &dest, kubeclientset, utils.ArgoCDNamespace)
+		assert.Equal(t, "unable to find destination server: there are no clusters with this name: minikube", err.Error())
+		assert.False(t, dest.IsServerInferred())
+	})
+
+	t.Run("Validate too many clusters with the same name", func(t *testing.T) {
+		dest := argoappv1.ApplicationDestination{
+			Name: "dind",
+		}
+
+		secret := createClusterSecret("dind", "dind", "https://127.0.0.1:2443")
+		secret2 := createClusterSecret("dind2", "dind", "https://127.0.0.1:8443")
+
+		objects := []runtime.Object{}
+		objects = append(objects, secret, secret2)
+		kubeclientset := fake.NewSimpleClientset(objects...)
+
+		err := ValidateDestination(context.Background(), &dest, kubeclientset, utils.ArgoCDNamespace)
+		assert.Equal(t, "unable to find destination server: there are 2 clusters with the same name: [https://127.0.0.1:2443 https://127.0.0.1:8443]", err.Error())
+		assert.False(t, dest.IsServerInferred())
+	})
+
+}
diff --git a/test/e2e/applicationset/cluster_e2e_test.go b/test/e2e/applicationset/cluster_e2e_test.go
index abfad15..8ec3c13 100644
--- a/test/e2e/applicationset/cluster_e2e_test.go
+++ b/test/e2e/applicationset/cluster_e2e_test.go
@@ -220,3 +220,170 @@ func TestClusterGeneratorWithLocalCluster(t *testing.T) {
 		})
 	}
 }
+
+func TestSimpleClusterGeneratorAddingCluster(t *testing.T) {
+
+	expectedAppTemplate := argov1alpha1.Application{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Application",
+			APIVersion: "argoproj.io/v1alpha1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "{{name}}-guestbook",
+			Namespace:  utils.ArgoCDNamespace,
+			Finalizers: []string{"resources-finalizer.argocd.argoproj.io"},
+		},
+		Spec: argov1alpha1.ApplicationSpec{
+			Project: "default",
+			Source: argov1alpha1.ApplicationSource{
+				RepoURL:        "https://github.com/argoproj/argocd-example-apps.git",
+				TargetRevision: "HEAD",
+				Path:           "guestbook",
+			},
+			Destination: argov1alpha1.ApplicationDestination{
+				Name:      "{{name}}",
+				Namespace: "guestbook",
+			},
+		},
+	}
+
+	expectedAppCluster1 := *expectedAppTemplate.DeepCopy()
+	expectedAppCluster1.Spec.Destination.Name = "cluster1"
+	expectedAppCluster1.ObjectMeta.Name = "cluster1-guestbook"
+
+	expectedAppCluster2 := *expectedAppTemplate.DeepCopy()
+	expectedAppCluster2.Spec.Destination.Name = "cluster2"
+	expectedAppCluster2.ObjectMeta.Name = "cluster2-guestbook"
+
+	Given(t).
+		// Create a ClusterGenerator-based ApplicationSet
+		When().
+		CreateClusterSecret("my-secret", "cluster1", "https://kubernetes.default.svc").
+		Create(v1alpha1.ApplicationSet{ObjectMeta: metav1.ObjectMeta{
+			Name: "simple-cluster-generator",
+		},
+			Spec: v1alpha1.ApplicationSetSpec{
+				Template: v1alpha1.ApplicationSetTemplate{
+					ApplicationSetTemplateMeta: v1alpha1.ApplicationSetTemplateMeta{Name: "{{name}}-guestbook"},
+					Spec: argov1alpha1.ApplicationSpec{
+						Project: "default",
+						Source: argov1alpha1.ApplicationSource{
+							RepoURL:        "https://github.com/argoproj/argocd-example-apps.git",
+							TargetRevision: "HEAD",
+							Path:           "guestbook",
+						},
+						Destination: argov1alpha1.ApplicationDestination{
+							Name: "{{name}}",
+							// Server:    "{{server}}",
+							Namespace: "guestbook",
+						},
+					},
+				},
+				Generators: []v1alpha1.ApplicationSetGenerator{
+					{
+						Clusters: &v1alpha1.ClusterGenerator{
+							Selector: metav1.LabelSelector{
+								MatchLabels: map[string]string{
+									"argocd.argoproj.io/secret-type": "cluster",
+								},
+							},
+						},
+					},
+				},
+			},
+		}).Then().Expect(ApplicationsExist([]argov1alpha1.Application{expectedAppCluster1})).
+
+		// Update the ApplicationSet template namespace, and verify it updates the Applications
+		When().
+		CreateClusterSecret("my-secret2", "cluster2", "https://kubernetes.default.svc").
+		Then().Expect(ApplicationsExist([]argov1alpha1.Application{expectedAppCluster1, expectedAppCluster2})).
+
+		// Delete the ApplicationSet, and verify it deletes the Applications
+		When().
+		Delete().Then().Expect(ApplicationsDoNotExist([]argov1alpha1.Application{expectedAppCluster1, expectedAppCluster2}))
+}
+
+// There is a bug in ArgoCD that prevents this test from passing: https://github.com/argoproj/argo-cd/issues/5817
+func TestSimpleClusterGeneratorDeletingCluster(t *testing.T) {
+
+	expectedAppTemplate := argov1alpha1.Application{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Application",
+			APIVersion: "argoproj.io/v1alpha1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "{{name}}-guestbook",
+			Namespace:  utils.ArgoCDNamespace,
+			Finalizers: []string{"resources-finalizer.argocd.argoproj.io"},
+		},
+		Spec: argov1alpha1.ApplicationSpec{
+			Project: "default",
+			Source: argov1alpha1.ApplicationSource{
+				RepoURL:        "https://github.com/argoproj/argocd-example-apps.git",
+				TargetRevision: "HEAD",
+				Path:           "guestbook",
+			},
+			Destination: argov1alpha1.ApplicationDestination{
+				Name:      "{{name}}",
+				Namespace: "guestbook",
+			},
+		},
+	}
+
+	expectedAppCluster1 := *expectedAppTemplate.DeepCopy()
+	expectedAppCluster1.Spec.Destination.Name = "cluster1"
+	expectedAppCluster1.ObjectMeta.Name = "cluster1-guestbook"
+
+	expectedAppCluster2 := *expectedAppTemplate.DeepCopy()
+	expectedAppCluster2.Spec.Destination.Name = "cluster2"
+	expectedAppCluster2.ObjectMeta.Name = "cluster2-guestbook"
+
+	Given(t).
+		// Create a ClusterGenerator-based ApplicationSet
+		When().
+		CreateClusterSecret("my-secret", "cluster1", "https://kubernetes.default.svc").
+		CreateClusterSecret("my-secret2", "cluster2", "https://kubernetes.default.svc").
+		Create(v1alpha1.ApplicationSet{ObjectMeta: metav1.ObjectMeta{
+			Name: "simple-cluster-generator",
+		},
+			Spec: v1alpha1.ApplicationSetSpec{
+				Template: v1alpha1.ApplicationSetTemplate{
+					ApplicationSetTemplateMeta: v1alpha1.ApplicationSetTemplateMeta{Name: "{{name}}-guestbook"},
+					Spec: argov1alpha1.ApplicationSpec{
+						Project: "default",
+						Source: argov1alpha1.ApplicationSource{
+							RepoURL:        "https://github.com/argoproj/argocd-example-apps.git",
+							TargetRevision: "HEAD",
+							Path:           "guestbook",
+						},
+						Destination: argov1alpha1.ApplicationDestination{
+							Name: "{{name}}",
+							// Server:    "{{server}}",
+							Namespace: "guestbook",
+						},
+					},
+				},
+				Generators: []v1alpha1.ApplicationSetGenerator{
+					{
+						Clusters: &v1alpha1.ClusterGenerator{
+							Selector: metav1.LabelSelector{
+								MatchLabels: map[string]string{
+									"argocd.argoproj.io/secret-type": "cluster",
+								},
+							},
+						},
+					},
+				},
+			},
+		}).Then().Expect(ApplicationsExist([]argov1alpha1.Application{expectedAppCluster1, expectedAppCluster2})).
+
+		// Update the ApplicationSet template namespace, and verify it updates the Applications
+		When().
+		DeleteClusterSecret("my-secret2").
+		Then().Expect(ApplicationsExist([]argov1alpha1.Application{expectedAppCluster1})).
+		Expect(ApplicationsDoNotExist([]argov1alpha1.Application{expectedAppCluster2})).
+
+		// Delete the ApplicationSet, and verify it deletes the Applications
+		When().
+		Delete().Then().Expect(ApplicationsDoNotExist([]argov1alpha1.Application{expectedAppCluster1}))
+}
diff --git a/test/e2e/fixture/applicationsets/actions.go b/test/e2e/fixture/applicationsets/actions.go
index 75117f8..af875cd 100644
--- a/test/e2e/fixture/applicationsets/actions.go
+++ b/test/e2e/fixture/applicationsets/actions.go
@@ -67,7 +67,8 @@ func (a *Actions) CreateClusterSecret(secretName string, clusterName string, clu
 		},
 	}
 
-	_, err := utils.KubeClientset.CoreV1().Secrets(secret.Namespace).Create(context.Background(), secret, metav1.CreateOptions{})
+	fixtureClient := utils.GetE2EFixtureK8sClient()
+	_, err := fixtureClient.KubeClientset.CoreV1().Secrets(secret.Namespace).Create(context.Background(), secret, metav1.CreateOptions{})
 
 	a.describeAction = fmt.Sprintf("creating cluster Secret '%s'", secretName)
 	a.lastOutput, a.lastError = "", err
@@ -76,6 +77,18 @@ func (a *Actions) CreateClusterSecret(secretName string, clusterName string, clu
 	return a
 }
 
+// DeleteClusterSecret deletes a faux cluster secret
+func (a *Actions) DeleteClusterSecret(secretName string) *Actions {
+
+	err := utils.GetE2EFixtureK8sClient().KubeClientset.CoreV1().Secrets(utils.ArgoCDNamespace).Delete(context.Background(), secretName, metav1.DeleteOptions{})
+
+	a.describeAction = fmt.Sprintf("deleting cluster Secret '%s'", secretName)
+	a.lastOutput, a.lastError = "", err
+	a.verifyAction()
+
+	return a
+}
+
 // Create creates an ApplicationSet using the provided value
 func (a *Actions) Create(appSet v1alpha1.ApplicationSet) *Actions {
 	a.context.t.Helper()
@@ -83,7 +96,8 @@ func (a *Actions) Create(appSet v1alpha1.ApplicationSet) *Actions {
 	appSet.APIVersion = "argoproj.io/v1alpha1"
 	appSet.Kind = "ApplicationSet"
 
-	newResource, err := utils.AppSetClientset.Create(context.Background(), utils.MustToUnstructured(&appSet), metav1.CreateOptions{})
+	fixtureClient := utils.GetE2EFixtureK8sClient()
+	newResource, err := fixtureClient.AppSetClientset.Create(context.Background(), utils.MustToUnstructured(&appSet), metav1.CreateOptions{})
 
 	if err == nil {
 		a.context.name = newResource.GetName()
@@ -100,8 +114,10 @@ func (a *Actions) Create(appSet v1alpha1.ApplicationSet) *Actions {
 func (a *Actions) Delete() *Actions {
 	a.context.t.Helper()
 
+	fixtureClient := utils.GetE2EFixtureK8sClient()
+
 	deleteProp := metav1.DeletePropagationForeground
-	err := utils.AppSetClientset.Delete(context.Background(), a.context.name, metav1.DeleteOptions{PropagationPolicy: &deleteProp})
+	err := fixtureClient.AppSetClientset.Delete(context.Background(), a.context.name, metav1.DeleteOptions{PropagationPolicy: &deleteProp})
 	a.describeAction = fmt.Sprintf("Deleting ApplicationSet '%s' %v", a.context.name, err)
 	a.lastOutput, a.lastError = "", err
 	a.verifyAction()
@@ -113,7 +129,8 @@ func (a *Actions) Delete() *Actions {
 func (a *Actions) get() (*v1alpha1.ApplicationSet, error) {
 	appSet := v1alpha1.ApplicationSet{}
 
-	newResource, err := utils.AppSetClientset.Get(context.Background(), a.context.name, metav1.GetOptions{})
+	fixtureClient := utils.GetE2EFixtureK8sClient()
+	newResource, err := fixtureClient.AppSetClientset.Get(context.Background(), a.context.name, metav1.GetOptions{})
 	if err != nil {
 		return nil, err
 	}
@@ -142,7 +159,8 @@ func (a *Actions) Update(toUpdate func(*v1alpha1.ApplicationSet)) *Actions {
 		toUpdate(appSet)
 		a.describeAction = fmt.Sprintf("updating ApplicationSet '%s'", appSet.Name)
 
-		_, err = utils.AppSetClientset.Update(context.Background(), utils.MustToUnstructured(&appSet), metav1.UpdateOptions{})
+		fixtureClient := utils.GetE2EFixtureK8sClient()
+		_, err = fixtureClient.AppSetClientset.Update(context.Background(), utils.MustToUnstructured(&appSet), metav1.UpdateOptions{})
 	}
 	a.lastOutput, a.lastError = "", err
 	a.verifyAction()
diff --git a/test/e2e/fixture/applicationsets/consequences.go b/test/e2e/fixture/applicationsets/consequences.go
index c2e43f9..ab9df2b 100644
--- a/test/e2e/fixture/applicationsets/consequences.go
+++ b/test/e2e/fixture/applicationsets/consequences.go
@@ -66,7 +66,9 @@ func (c *Consequences) app(name string) *argov1alpha1.Application {
 }
 
 func (c *Consequences) apps() []argov1alpha1.Application {
-	list, err := utils.AppClientset.ArgoprojV1alpha1().Applications(utils.ArgoCDNamespace).List(context.Background(), v1.ListOptions{})
+
+	fixtureClient := utils.GetE2EFixtureK8sClient()
+	list, err := fixtureClient.AppClientset.ArgoprojV1alpha1().Applications(utils.ArgoCDNamespace).List(context.Background(), v1.ListOptions{})
 	errors.CheckError(err)
 
 	if list == nil {
diff --git a/test/e2e/fixture/applicationsets/utils/fixture.go b/test/e2e/fixture/applicationsets/utils/fixture.go
index 91f9913..b112f4d 100644
--- a/test/e2e/fixture/applicationsets/utils/fixture.go
+++ b/test/e2e/fixture/applicationsets/utils/fixture.go
@@ -6,6 +6,7 @@ import (
 	"os"
 	"regexp"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -30,26 +31,59 @@ const (
 )
 
 var (
-	id               string
+	id string
+
+	// call GetClientVars() to retrieve the Kubernetes client data for E2E test fixtures
+	clientInitialized  sync.Once
+	internalClientVars *E2EFixtureK8sClient
+)
+
+// E2EFixtureK8sClient contains Kubernetes clients initialized from local k8s configuration
+type E2EFixtureK8sClient struct {
 	KubeClientset    kubernetes.Interface
 	DynamicClientset dynamic.Interface
 	AppClientset     appclientset.Interface
 	AppSetClientset  dynamic.ResourceInterface
-)
+}
+
+// GetE2EFixtureK8sClient initializes the Kubernetes clients (if needed), and returns the most recently initalized value.
+// Note: this requires a local Kubernetes configuration (for example, while running the E2E tests).
+func GetE2EFixtureK8sClient() *E2EFixtureK8sClient {
+
+	// Initialize the Kubernetes clients only on first use
+	clientInitialized.Do(func() {
+
+		// set-up variables
+		config := getKubeConfig("", clientcmd.ConfigOverrides{})
 
+		internalClientVars = &E2EFixtureK8sClient{
+			AppClientset:     appclientset.NewForConfigOrDie(config),
+			DynamicClientset: dynamic.NewForConfigOrDie(config),
+			KubeClientset:    kubernetes.NewForConfigOrDie(config),
+		}
+
+		internalClientVars.AppSetClientset = internalClientVars.DynamicClientset.Resource(v1alpha1.GroupVersion.WithResource("applicationsets")).Namespace(ArgoCDNamespace)
+
+	})
+	return internalClientVars
+}
+
+// EnsureCleanSlate ensures that the Kubernetes resources on the cluster are are in a 'clean' state, before a test is run.
 func EnsureCleanState(t *testing.T) {
 
 	start := time.Now()
 
-	policy := v1.DeletePropagationBackground
+	fixtureClient := GetE2EFixtureK8sClient()
+
+	policy := v1.DeletePropagationForeground
 	// delete resources
 	// kubectl delete applicationsets --all
-	CheckError(AppSetClientset.DeleteCollection(context.Background(), v1.DeleteOptions{PropagationPolicy: &policy}, v1.ListOptions{}))
+	CheckError(fixtureClient.AppSetClientset.DeleteCollection(context.Background(), v1.DeleteOptions{PropagationPolicy: &policy}, v1.ListOptions{}))
 	// kubectl delete apps --all
-	CheckError(AppClientset.ArgoprojV1alpha1().Applications(ArgoCDNamespace).DeleteCollection(context.Background(), v1.DeleteOptions{PropagationPolicy: &policy}, v1.ListOptions{}))
+	CheckError(fixtureClient.AppClientset.ArgoprojV1alpha1().Applications(ArgoCDNamespace).DeleteCollection(context.Background(), v1.DeleteOptions{PropagationPolicy: &policy}, v1.ListOptions{}))
 
 	// kubectl delete secrets -l e2e.argoproj.io=true
-	CheckError(KubeClientset.CoreV1().Secrets(ArgoCDNamespace).DeleteCollection(context.Background(),
+	CheckError(fixtureClient.KubeClientset.CoreV1().Secrets(ArgoCDNamespace).DeleteCollection(context.Background(),
 		v1.DeleteOptions{PropagationPolicy: &policy}, v1.ListOptions{LabelSelector: TestingLabel + "=true"}))
 
 	// remove tmp dir
@@ -78,14 +112,6 @@ func init() {
 
 	// ensure we log all shell execs
 	log.SetLevel(log.DebugLevel)
-
-	// set-up variables
-	config := getKubeConfig("", clientcmd.ConfigOverrides{})
-	AppClientset = appclientset.NewForConfigOrDie(config)
-	KubeClientset = kubernetes.NewForConfigOrDie(config)
-	DynamicClientset = dynamic.NewForConfigOrDie(config)
-	AppSetClientset = DynamicClientset.Resource(v1alpha1.GroupVersion.WithResource("applicationsets")).Namespace(ArgoCDNamespace)
-
 }
 
 // PrettyPrintJson is a utility function for debugging purposes