Repository: argoproj-labs/applicationset
Pull request #91 (2021-01-11 23:38:23)

feat(cluster-generator): support arbitrary key:value pairs
Per the cluster generator filters example, we should allow [arbitrary key:value pairs](https://github.com/argoproj-labs/applicationset/blob/master/examples/filters.yaml#L17) that can be [used for templating](https://github.com/argoproj-labs/applicationset/blob/master/examples/filters.yaml#L40).

Depends on #93 

Signed-off-by: Michael Goodness <michael.goodness@mlb.com>


mgoodness: Premature PR, as the CRD will have to be regenerated with the addition to `ClusterGenerator`. That's complicated by the fact that the project was bootstrapped using kubebuilder v1. I'm in the process of migrating to v2 - a separate PR.
mgoodness: As reported in Slack, something in the e2e tests doesn't like this new ApplicationSet CRD generated using `apiextensions.k8s.io/v1`. If I generate the CRD using `apiextensions.k8s.io/v1beta` (as in #97), everything is fine.

Can confirm that the `apiextensions.k8s.io/v1` CRD works fine in my cluster.
mgoodness: Punting on the `apiextensions.k8s.io/v1` for now, in the interest of getting this feature merged.
mgoodness: > Thanks @mgoodness, still needs a quick rebase but LGTM!

Go go go, before someone sneaks in another change!

jgwest: Merged, thanks again... sorry for the code churn, going forward we should have fewer conflicts now that we are close to having things nailed down in the generated manifests. :smile: 

Patch:
diff --git a/Dockerfile b/Dockerfile
index 7babdf9..0c37411 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -20,8 +20,8 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-w -s" -a -o applic
 
 FROM debian:10-slim
 RUN apt-get update && apt-get upgrade -y && \
-    apt-get install -y git-all && \
-    rm -r /var/lib/apt/lists /var/cache/apt/archives
+  apt-get install -y git-all && \
+  rm -r /var/lib/apt/lists /var/cache/apt/archives
 
 WORKDIR /
 COPY --from=builder /workspace/applicationset-controller /usr/local/bin/
diff --git a/api/v1alpha1/applicationset_types.go b/api/v1alpha1/applicationset_types.go
index e709fb0..1c95f52 100644
--- a/api/v1alpha1/applicationset_types.go
+++ b/api/v1alpha1/applicationset_types.go
@@ -77,6 +77,9 @@ type ClusterGenerator struct {
 	// for matching the selector.
 	Selector metav1.LabelSelector   `json:"selector,omitempty"`
 	Template ApplicationSetTemplate `json:"template,omitempty"`
+
+	// Values contains key/value pairs which are passed directly as parameters to the template
+	Values map[string]string `json:"values,omitempty"`
 }
 
 type GitGenerator struct {
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index fe9a8d8..a4e1d97 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -193,6 +193,13 @@ func (in *ClusterGenerator) DeepCopyInto(out *ClusterGenerator) {
 	*out = *in
 	in.Selector.DeepCopyInto(&out.Selector)
 	in.Template.DeepCopyInto(&out.Template)
+	if in.Values != nil {
+		in, out := &in.Values, &out.Values
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterGenerator.
@@ -243,6 +250,11 @@ func (in *GitGenerator) DeepCopyInto(out *GitGenerator) {
 		*out = make([]GitDirectoryGeneratorItem, len(*in))
 		copy(*out, *in)
 	}
+	if in.Files != nil {
+		in, out := &in.Files, &out.Files
+		*out = make([]GitFileGeneratorItem, len(*in))
+		copy(*out, *in)
+	}
 	in.Template.DeepCopyInto(&out.Template)
 }
 
diff --git a/examples/filters.yaml b/examples/filters.yaml
index 38a712f..295d2dc 100644
--- a/examples/filters.yaml
+++ b/examples/filters.yaml
@@ -37,7 +37,7 @@ spec:
     spec:
       source:
         repoURL: https://github.com/infra-team/cluster-deployments.git
-        targetRevision: "{{version}}"
+        targetRevision: "{{values.version}}"
         chart: guestbook
         helm:
           parameters:
diff --git a/manifests/base/deployment.yaml b/manifests/base/deployment.yaml
index 9b4e3e7..65c0c05 100644
--- a/manifests/base/deployment.yaml
+++ b/manifests/base/deployment.yaml
@@ -18,7 +18,7 @@ spec:
       containers:
         - command:
             - applicationset-controller
-          image: registry.cn-hangzhou.aliyuncs.com/appcenter/argocd-applicationset:v0.1.0
+          image: argoprojlabs/argocd-applicationset:latest
           imagePullPolicy: Always
           name: argocd-applicationset-controller
           env:
diff --git a/manifests/crds/argoproj.io_applicationsets.yaml b/manifests/crds/argoproj.io_applicationsets.yaml
index 71b2b0f..f4e2e06 100644
--- a/manifests/crds/argoproj.io_applicationsets.yaml
+++ b/manifests/crds/argoproj.io_applicationsets.yaml
@@ -463,6 +463,12 @@ spec:
                         - metadata
                         - spec
                         type: object
+                      values:
+                        additionalProperties:
+                          type: string
+                        description: Values contains key/value pairs which are passed
+                          directly as parameters to the template
+                        type: object
                     type: object
                   git:
                     properties:
@@ -475,6 +481,15 @@ spec:
                           - path
                           type: object
                         type: array
+                      files:
+                        items:
+                          properties:
+                            path:
+                              type: string
+                          required:
+                          - path
+                          type: object
+                        type: array
                       repoURL:
                         type: string
                       requeueAfterSeconds:
diff --git a/pkg/generators/cluster.go b/pkg/generators/cluster.go
index 1c141b6..75bc622 100644
--- a/pkg/generators/cluster.go
+++ b/pkg/generators/cluster.go
@@ -69,7 +69,7 @@ func (g *ClusterGenerator) GenerateParams(
 	// For each matching cluster secret
 	res := make([]map[string]string, len(clusterSecretList.Items))
 	for i, cluster := range clusterSecretList.Items {
-		params := make(map[string]string, len(cluster.ObjectMeta.Annotations)+len(cluster.ObjectMeta.Labels)+2)
+		params := make(map[string]string, len(appSetGenerator.Clusters.Values)+len(cluster.ObjectMeta.Annotations)+len(cluster.ObjectMeta.Labels)+2)
 		params["name"] = string(cluster.Data["name"])
 		params["server"] = string(cluster.Data["server"])
 		for key, value := range cluster.ObjectMeta.Annotations {
@@ -78,6 +78,9 @@ func (g *ClusterGenerator) GenerateParams(
 		for key, value := range cluster.ObjectMeta.Labels {
 			params[fmt.Sprintf("metadata.labels.%s", key)] = value
 		}
+		for key, value := range appSetGenerator.Clusters.Values {
+			params[fmt.Sprintf("values.%s", key)] = value
+		}
 		log.WithField("cluster", cluster.Name).Info("matched cluster secret")
 
 		res[i] = params


Test patch:
diff --git a/pkg/generators/cluster_test.go b/pkg/generators/cluster_test.go
index 3d6d5e0..0d48074 100644
--- a/pkg/generators/cluster_test.go
+++ b/pkg/generators/cluster_test.go
@@ -82,12 +82,14 @@ func TestGenerateParams(t *testing.T) {
 	}
 	testCases := []struct {
 		selector      metav1.LabelSelector
+		values        map[string]string
 		expected      []map[string]string
 		clientError   bool
 		expectedError error
 	}{
 		{
 			metav1.LabelSelector{},
+			nil,
 			[]map[string]string{
 				{"name": "cHJvZHVjdGlvbi0wMQ==", "server": "https://production-01.example.com", "metadata.labels.environment": "production", "metadata.labels.org": "bar", "metadata.labels.argocd.argoproj.io/secret-type": "cluster", "metadata.annotations.foo.argoproj.io": "production"},
 				{"name": "c3RhZ2luZy0wMQ==", "server": "https://staging-01.example.com", "metadata.labels.environment": "staging", "metadata.labels.org": "foo", "metadata.labels.argocd.argoproj.io/secret-type": "cluster", "metadata.annotations.foo.argoproj.io": "staging"},
@@ -101,8 +103,11 @@ func TestGenerateParams(t *testing.T) {
 					"environment": "production",
 				},
 			},
+			map[string]string{
+				"foo": "bar",
+			},
 			[]map[string]string{
-				{"name": "cHJvZHVjdGlvbi0wMQ==", "server": "https://production-01.example.com", "metadata.labels.environment": "production", "metadata.labels.org": "bar", "metadata.labels.argocd.argoproj.io/secret-type": "cluster", "metadata.annotations.foo.argoproj.io": "production"},
+				{"values.foo": "bar", "name": "cHJvZHVjdGlvbi0wMQ==", "server": "https://production-01.example.com", "metadata.labels.environment": "production", "metadata.labels.org": "bar", "metadata.labels.argocd.argoproj.io/secret-type": "cluster", "metadata.annotations.foo.argoproj.io": "production"},
 			},
 			false,
 			nil,
@@ -120,9 +125,12 @@ func TestGenerateParams(t *testing.T) {
 					},
 				},
 			},
+			map[string]string{
+				"foo": "bar",
+			},
 			[]map[string]string{
-				{"name": "cHJvZHVjdGlvbi0wMQ==", "server": "https://production-01.example.com", "metadata.labels.environment": "production", "metadata.labels.org": "bar", "metadata.labels.argocd.argoproj.io/secret-type": "cluster", "metadata.annotations.foo.argoproj.io": "production"},
-				{"name": "c3RhZ2luZy0wMQ==", "server": "https://staging-01.example.com", "metadata.labels.environment": "staging", "metadata.labels.org": "foo", "metadata.labels.argocd.argoproj.io/secret-type": "cluster", "metadata.annotations.foo.argoproj.io": "staging"},
+				{"values.foo": "bar", "name": "c3RhZ2luZy0wMQ==", "server": "https://staging-01.example.com", "metadata.labels.environment": "staging", "metadata.labels.org": "foo", "metadata.labels.argocd.argoproj.io/secret-type": "cluster", "metadata.annotations.foo.argoproj.io": "staging"},
+				{"values.foo": "bar", "name": "cHJvZHVjdGlvbi0wMQ==", "server": "https://production-01.example.com", "metadata.labels.environment": "production", "metadata.labels.org": "bar", "metadata.labels.argocd.argoproj.io/secret-type": "cluster", "metadata.annotations.foo.argoproj.io": "production"},
 			},
 			false,
 			nil,
@@ -143,8 +151,11 @@ func TestGenerateParams(t *testing.T) {
 					"org": "foo",
 				},
 			},
+			map[string]string{
+				"name": "baz",
+			},
 			[]map[string]string{
-				{"name": "c3RhZ2luZy0wMQ==", "server": "https://staging-01.example.com", "metadata.labels.environment": "staging", "metadata.labels.org": "foo", "metadata.labels.argocd.argoproj.io/secret-type": "cluster", "metadata.annotations.foo.argoproj.io": "staging"},
+				{"values.name": "baz", "name": "c3RhZ2luZy0wMQ==", "server": "https://staging-01.example.com", "metadata.labels.environment": "staging", "metadata.labels.org": "foo", "metadata.labels.argocd.argoproj.io/secret-type": "cluster", "metadata.annotations.foo.argoproj.io": "staging"},
 			},
 			false,
 			nil,
@@ -152,6 +163,7 @@ func TestGenerateParams(t *testing.T) {
 		{
 			metav1.LabelSelector{},
 			nil,
+			nil,
 			true,
 			errors.New("could not list Secrets"),
 		},
@@ -169,6 +181,7 @@ func TestGenerateParams(t *testing.T) {
 		got, err := clusterGenerator.GenerateParams(&argoprojiov1alpha1.ApplicationSetGenerator{
 			Clusters: &argoprojiov1alpha1.ClusterGenerator{
 				Selector: testCase.selector,
+				Values:   testCase.values,
 			},
 		})
 
diff --git a/test/e2e/applicationset/applicationset_test.go b/test/e2e/applicationset/applicationset_test.go
index 00b4ed9..0b20c96 100644
--- a/test/e2e/applicationset/applicationset_test.go
+++ b/test/e2e/applicationset/applicationset_test.go
@@ -8,7 +8,6 @@ import (
 	"github.com/argoproj-labs/applicationset/test/e2e/fixture/applicationsets/utils"
 	argov1alpha1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func TestSimpleListGenerator(t *testing.T) {
@@ -39,12 +38,12 @@ func TestSimpleListGenerator(t *testing.T) {
 
 	Given(t).
 		// Create a ListGenerator-based ApplicationSet
-		When().Create(v1alpha1.ApplicationSet{ObjectMeta: v1.ObjectMeta{
+		When().Create(v1alpha1.ApplicationSet{ObjectMeta: metav1.ObjectMeta{
 		Name: "simple-list-generator",
 	},
 		Spec: v1alpha1.ApplicationSetSpec{
 			Template: v1alpha1.ApplicationSetTemplate{
-				ObjectMeta: v1.ObjectMeta{Name: "{{cluster}}-guestbook"},
+				ObjectMeta: metav1.ObjectMeta{Name: "{{cluster}}-guestbook"},
 				Spec: argov1alpha1.ApplicationSpec{
 					Project: "default",
 					Source: argov1alpha1.ApplicationSource{
@@ -117,12 +116,12 @@ func TestSimpleClusterGenerator(t *testing.T) {
 		// Create a ClusterGenerator-based ApplicationSet
 		When().
 		CreateClusterSecret("my-secret", "cluster1", "https://kubernetes.default.svc").
-		Create(v1alpha1.ApplicationSet{ObjectMeta: v1.ObjectMeta{
+		Create(v1alpha1.ApplicationSet{ObjectMeta: metav1.ObjectMeta{
 			Name: "simple-cluster-generator",
 		},
 			Spec: v1alpha1.ApplicationSetSpec{
 				Template: v1alpha1.ApplicationSetTemplate{
-					ObjectMeta: v1.ObjectMeta{Name: "{{name}}-guestbook"},
+					ObjectMeta: metav1.ObjectMeta{Name: "{{name}}-guestbook"},
 					Spec: argov1alpha1.ApplicationSpec{
 						Project: "default",
 						Source: argov1alpha1.ApplicationSource{
@@ -140,7 +139,7 @@ func TestSimpleClusterGenerator(t *testing.T) {
 				Generators: []v1alpha1.ApplicationSetGenerator{
 					{
 						Clusters: &v1alpha1.ClusterGenerator{
-							Selector: v1.LabelSelector{
+							Selector: metav1.LabelSelector{
 								MatchLabels: map[string]string{
 									"argocd.argoproj.io/secret-type": "cluster",
 								},
@@ -204,12 +203,12 @@ func TestSimpleGitGenerator(t *testing.T) {
 	Given(t).
 		When().
 		// Create a GitGenerator-based ApplicationSet
-		Create(v1alpha1.ApplicationSet{ObjectMeta: v1.ObjectMeta{
+		Create(v1alpha1.ApplicationSet{ObjectMeta: metav1.ObjectMeta{
 			Name: "simple-git-generator",
 		},
 			Spec: v1alpha1.ApplicationSetSpec{
 				Template: v1alpha1.ApplicationSetTemplate{
-					ObjectMeta: v1.ObjectMeta{Name: "{{path.basename}}"},
+					ObjectMeta: metav1.ObjectMeta{Name: "{{path.basename}}"},
 					Spec: argov1alpha1.ApplicationSpec{
 						Project: "default",
 						Source: argov1alpha1.ApplicationSource{