Repository: rancher/fleet
Pull request #671 (2021-12-04 01:01:38)

Sprig Templating for Helm Values with Inputs from Cluster Resource
This PR introduces the ability for fleet.yaml's `helm.values` object to contain go template strings in either the keys or values. The context of the go template also includes the cluster labels (for convenient migration from the existing `global.fleet.clusterLabels.FOO` users.

Cluster specific template variables can be added to `Cluster` CR under a new optional key in `spec.templateContext`

Related issues/PRs
- rancher/fleet#507 but adds the per-cluster custom template context
- rancher/fleet#375 
- rancher/fleet#355 

rajiteh: @gregsidelinger in case you're interested I am planning to propose an enhancement on top of this templating system to introduce secret management via vals. I have a PR here with a basic implementation, docs pending. IMO this gives an elegant way to template and as well as resolve secrets per release/cluster basis.

 https://github.com/rajiteh/fleet/pull/3/files
prachidamle: @rajiteh I am not clear on how would the Cluster specific template variables be added to a Cluster CR under spec.templateContext? Is the idea that the cluster Admins need to add this data to the target clusters? I am wondering if this templateContext data added to the Fleet target cluster CRs will get overwritten when synced with the management.cattle.io.cluster objects in Rancher. 
rajiteh: > @rajiteh I am not clear on how would the Cluster specific template variables be added to a Cluster CR under spec.templateContext? Is the idea that the cluster Admins need to add this data to the target clusters? I am wondering if this templateContext data added to the Fleet target cluster CRs will get overwritten when synced with the management.cattle.io.cluster objects in Rancher.

Yes, the idea is that admins will add these values. I can't speak to Rancher use case unfortunately as I'm only using fleet standalone. 

PS: I will address your feedback over the weekend!
gregsidelinger: I have not seen rancher touch the templateContext on the fleet cluster in my setup.  Once set rancher left everything alone when it synced labels from the management cluster.
rajiteh: @prachidamle hello! sorry for the delay, please have a look when you get a chance. I believe I address all the requested efeedback.
k0da: Hello, any updates?

thardeck: @rajiteh is there something you would miss in the implementation of #1035? It would be nice if you can comment there so we can find an implementation which covers most requirements. Thanks in advance.
rajiteh: FYI: I have updated the fork with the latest code from master! Love to get the ball rolling on this!! :) 
manno: @rajiteh Let's get this merged. Do you want to rename the context variable and rebase this PR against current master, or should we try to schedule this?
rajiteh: @manno I made all the changes and fixed the build!
manno: > @manno I made all the changes and fixed the build!
> 
> PS: Rebasing seems to be a little painful, we could squash the commits instead perhaps?

Yes, that would be great. 
thardeck: @rajiteh There is still a small linting issue: https://github.com/rancher/fleet/actions/runs/3576792840/jobs/6029393464
```
  Error: `getClusterAndBundle` - `clusterYaml` always receives `clusterYamlWithTemplateValues` (`"apiVersion: fleet.cattle.io/v1alpha1\nkind: Cluster\nmetadata:\n  na...`) (unparam)
```

Patch:
diff --git a/.gitignore b/.gitignore
index 0054359d..904e6f3c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,3 +11,4 @@
 docs/fleet-agent/
 docs/fleet-cli/
 docs/fleet-controller/
+^fleet$
diff --git a/charts/fleet-crd/templates/crds.yaml b/charts/fleet-crd/templates/crds.yaml
index c580c30f..0aa39412 100644
--- a/charts/fleet-crd/templates/crds.yaml
+++ b/charts/fleet-crd/templates/crds.yaml
@@ -117,6 +117,8 @@ spec:
                   chart:
                     nullable: true
                     type: string
+                  disablePreProcess:
+                    type: boolean
                   force:
                     type: boolean
                   maxHistory:
@@ -488,6 +490,8 @@ spec:
                         chart:
                           nullable: true
                           type: string
+                        disablePreProcess:
+                          type: boolean
                         force:
                           type: boolean
                         maxHistory:
@@ -997,6 +1001,8 @@ spec:
                       chart:
                         nullable: true
                         type: string
+                      disablePreProcess:
+                        type: boolean
                       force:
                         type: boolean
                       maxHistory:
@@ -1144,6 +1150,8 @@ spec:
                       chart:
                         nullable: true
                         type: string
+                      disablePreProcess:
+                        type: boolean
                       force:
                         type: boolean
                       maxHistory:
@@ -1784,6 +1792,10 @@ spec:
                 type: string
               redeployAgentGeneration:
                 type: integer
+              templateValues:
+                nullable: true
+                type: object
+                x-kubernetes-preserve-unknown-fields: true
             type: object
           status:
             properties:
diff --git a/go.mod b/go.mod
index e0f193b5..55611af4 100644
--- a/go.mod
+++ b/go.mod
@@ -33,7 +33,8 @@ replace (
 )
 
 require (
-	github.com/Masterminds/semver/v3 v3.2.0
+	github.com/Masterminds/semver/v3 v3.1.1
+	github.com/Masterminds/sprig/v3 v3.2.2
 	github.com/cheggaaa/pb v1.0.29
 	github.com/davecgh/go-spew v1.1.1
 	github.com/evanphx/json-patch v5.6.0+incompatible
@@ -93,7 +94,6 @@ require (
 	github.com/BurntSushi/toml v1.1.0 // indirect
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
-	github.com/Masterminds/sprig/v3 v3.2.2 // indirect
 	github.com/Masterminds/squirrel v1.5.3 // indirect
 	github.com/Microsoft/go-winio v0.6.0 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20220623141421-5afb4c282135 // indirect
diff --git a/go.sum b/go.sum
index b566eb07..af2f808c 100644
--- a/go.sum
+++ b/go.sum
@@ -115,9 +115,8 @@ github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy86
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
 github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
 github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
+github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
-github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g=
-github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
 github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
 github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8=
 github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk=
diff --git a/pkg/apis/fleet.cattle.io/v1alpha1/bundle.go b/pkg/apis/fleet.cattle.io/v1alpha1/bundle.go
index 8e91f765..f3a7d9e3 100644
--- a/pkg/apis/fleet.cattle.io/v1alpha1/bundle.go
+++ b/pkg/apis/fleet.cattle.io/v1alpha1/bundle.go
@@ -237,6 +237,9 @@ type HelmOptions struct {
 
 	// Atomic sets the --atomic flag when Helm is performing an upgrade
 	Atomic bool `json:"atomic,omitempty"`
+
+	// DisablePreProcess disables template processing in values
+	DisablePreProcess bool `json:"disablePreProcess,omitempty"`
 }
 
 // Define helm values that can come from configmap, secret or external. Credit: https://github.com/fluxcd/helm-operator/blob/0cfea875b5d44bea995abe7324819432070dfbdc/pkg/apis/helm.fluxcd.io/v1/types_helmrelease.go#L439
diff --git a/pkg/apis/fleet.cattle.io/v1alpha1/target.go b/pkg/apis/fleet.cattle.io/v1alpha1/target.go
index 88ce191b..6ddde35e 100644
--- a/pkg/apis/fleet.cattle.io/v1alpha1/target.go
+++ b/pkg/apis/fleet.cattle.io/v1alpha1/target.go
@@ -70,6 +70,9 @@ type ClusterSpec struct {
 	// AgentNamespace defaults to the system namespace, e.g. cattle-fleet-system
 	AgentNamespace string `json:"agentNamespace,omitempty"`
 	PrivateRepoURL string `json:"privateRepoURL,omitempty"`
+
+	// TemplateValues defines a cluster specific mapping of values to be sent to fleet.yaml values templating
+	TemplateValues *GenericMap `json:"templateValues,omitempty"`
 }
 
 type ClusterStatus struct {
diff --git a/pkg/apis/fleet.cattle.io/v1alpha1/zz_generated_deepcopy.go b/pkg/apis/fleet.cattle.io/v1alpha1/zz_generated_deepcopy.go
index c1b9d17e..1413aa0a 100644
--- a/pkg/apis/fleet.cattle.io/v1alpha1/zz_generated_deepcopy.go
+++ b/pkg/apis/fleet.cattle.io/v1alpha1/zz_generated_deepcopy.go
@@ -1007,6 +1007,10 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.TemplateValues != nil {
+		in, out := &in.TemplateValues, &out.TemplateValues
+		*out = (*in).DeepCopy()
+	}
 	return
 }
 
diff --git a/pkg/options/calculate.go b/pkg/options/calculate.go
index 0e4ef31b..6c72a496 100644
--- a/pkg/options/calculate.go
+++ b/pkg/options/calculate.go
@@ -81,6 +81,7 @@ func Merge(base, next fleet.BundleDeploymentOptions) fleet.BundleDeploymentOptio
 		result.Helm.Force = result.Helm.Force || next.Helm.Force
 		result.Helm.Atomic = result.Helm.Atomic || next.Helm.Atomic
 		result.Helm.TakeOwnership = result.Helm.TakeOwnership || next.Helm.TakeOwnership
+		result.Helm.DisablePreProcess = result.Helm.DisablePreProcess || next.Helm.DisablePreProcess
 	}
 	if next.Kustomize != nil {
 		if result.Kustomize == nil {
diff --git a/pkg/target/target.go b/pkg/target/target.go
index 4f247da4..89984370 100644
--- a/pkg/target/target.go
+++ b/pkg/target/target.go
@@ -6,10 +6,12 @@
 package target
 
 import (
+	"bytes"
 	"fmt"
 	"sort"
 	"strconv"
 	"strings"
+	"text/template"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -21,7 +23,6 @@ import (
 	"github.com/rancher/fleet/pkg/options"
 	"github.com/rancher/fleet/pkg/summary"
 
-	"github.com/rancher/wrangler/pkg/data"
 	corecontrollers "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	"github.com/rancher/wrangler/pkg/name"
 	"github.com/rancher/wrangler/pkg/yaml"
@@ -30,6 +31,8 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/sets"
+
+	"github.com/Masterminds/sprig/v3"
 )
 
 var (
@@ -39,6 +42,8 @@ var (
 	defMaxUnavailablePartitions = intstr.FromInt(0)
 )
 
+const maxTemplateRecursionDepth = 50
+
 type Manager struct {
 	clusters                    fleetcontrollers.ClusterCache
 	clusterGroups               fleetcontrollers.ClusterGroupCache
@@ -263,7 +268,7 @@ func (m *Manager) Targets(bundle *fleet.Bundle, manifest *manifest.Manifest) ([]
 			}
 
 			opts := options.Merge(bundle.Spec.BundleDeploymentOptions, target.BundleDeploymentOptions)
-			err = addClusterLabels(&opts, cluster.Labels)
+			err = preprocessHelmValues(&opts, cluster)
 			if err != nil {
 				return nil, err
 			}
@@ -290,9 +295,11 @@ func (m *Manager) Targets(bundle *fleet.Bundle, manifest *manifest.Manifest) ([]
 	return targets, m.foldInDeployments(bundle, targets)
 }
 
-func addClusterLabels(opts *fleet.BundleDeploymentOptions, labels map[string]string) (err error) {
-	clusterLabels := yaml.CleanAnnotationsForExport(labels)
-	for k, v := range labels {
+func preprocessHelmValues(opts *fleet.BundleDeploymentOptions, cluster *fleet.Cluster) (err error) {
+	clusterLabels := yaml.CleanAnnotationsForExport(cluster.Labels)
+	clusterAnnotations := yaml.CleanAnnotationsForExport(cluster.Annotations)
+
+	for k, v := range cluster.Labels {
 		if strings.HasPrefix(k, "fleet.cattle.io/") || strings.HasPrefix(k, "management.cattle.io/") {
 			clusterLabels[k] = v
 		}
@@ -301,27 +308,15 @@ func addClusterLabels(opts *fleet.BundleDeploymentOptions, labels map[string]str
 		return
 	}
 
-	newValues := map[string]interface{}{
-		"global": map[string]interface{}{
-			"fleet": map[string]interface{}{
-				"clusterLabels": clusterLabels,
-			},
-		},
-	}
-
 	if opts.Helm == nil {
-		opts.Helm = &fleet.HelmOptions{
-			Values: &fleet.GenericMap{
-				Data: newValues,
-			},
-		}
+		opts.Helm = &fleet.HelmOptions{}
 		return nil
 	}
 
 	opts.Helm = opts.Helm.DeepCopy()
 	if opts.Helm.Values == nil || opts.Helm.Values.Data == nil {
 		opts.Helm.Values = &fleet.GenericMap{
-			Data: newValues,
+			Data: map[string]interface{}{},
 		}
 		return nil
 	}
@@ -330,7 +325,28 @@ func addClusterLabels(opts *fleet.BundleDeploymentOptions, labels map[string]str
 		return err
 	}
 
-	opts.Helm.Values.Data = data.MergeMaps(opts.Helm.Values.Data, newValues)
+	if !opts.Helm.DisablePreProcess {
+
+		templateValues := map[string]interface{}{}
+		if cluster.Spec.TemplateValues != nil {
+			templateValues = cluster.Spec.TemplateValues.Data
+		}
+
+		values := map[string]interface{}{
+			"ClusterNamespace":   cluster.Namespace,
+			"ClusterName":        cluster.Name,
+			"ClusterLabels":      clusterLabels,
+			"ClusterAnnotations": clusterAnnotations,
+			"ClusterValues":      templateValues,
+		}
+
+		opts.Helm.Values.Data, err = processTemplateValues(opts.Helm.Values.Data, values)
+		if err != nil {
+			return err
+		}
+		logrus.Debugf("preProcess completed for %v", opts.Helm.ReleaseName)
+	}
+
 	return nil
 
 }
@@ -565,6 +581,86 @@ func Summary(targets []*Target) fleet.BundleSummary {
 	return bundleSummary
 }
 
+// tplFuncMap returns a mapping of all of the functions from sprig but removes potentially dangerous operations
+func tplFuncMap() template.FuncMap {
+	f := sprig.TxtFuncMap()
+	delete(f, "env")
+	delete(f, "expandenv")
+	delete(f, "include")
+	delete(f, "tpl")
+
+	return f
+}
+
+func processTemplateValues(valuesMap map[string]interface{}, templateContext map[string]interface{}) (map[string]interface{}, error) {
+	tplFn := template.New("values").Funcs(tplFuncMap()).Option("missingkey=error")
+	recursionDepth := 0
+	tplResult, err := templateSubstitutions(valuesMap, templateContext, tplFn, recursionDepth)
+	if err != nil {
+		return nil, err
+	}
+	compiledYaml, ok := tplResult.(map[string]interface{})
+	if !ok {
+		return nil, fmt.Errorf("templated result was expected to be map[string]interface{}, got %T", tplResult)
+	}
+
+	return compiledYaml, nil
+}
+
+func templateSubstitutions(src interface{}, templateContext map[string]interface{}, tplFn *template.Template, recursionDepth int) (interface{}, error) {
+	if recursionDepth > maxTemplateRecursionDepth {
+		return nil, fmt.Errorf("maximum recursion depth of %v exceeded for current templating operation, too many nested values", maxTemplateRecursionDepth)
+	}
+
+	switch tplVal := src.(type) {
+	case string:
+		tpl, err := tplFn.Parse(tplVal)
+		if err != nil {
+			return nil, err
+		}
+
+		var tplBytes bytes.Buffer
+		defer func() {
+			if r := recover(); r != nil {
+				err = fmt.Errorf("failed to process template substitution for string '%s': [%v]", tplVal, err)
+			}
+		}()
+		err = tpl.Execute(&tplBytes, templateContext)
+		if err != nil {
+			return nil, fmt.Errorf("failed to process template substitution for string '%s': [%v]", tplVal, err)
+		}
+		return tplBytes.String(), nil
+	case map[string]interface{}:
+		newMap := make(map[string]interface{})
+		for key, val := range tplVal {
+			processedKey, err := templateSubstitutions(key, templateContext, tplFn, recursionDepth+1)
+			if err != nil {
+				return nil, err
+			}
+			keyAsString, ok := processedKey.(string)
+			if !ok {
+				return nil, fmt.Errorf("expected a string to be returned, but instead got [%T]", processedKey)
+			}
+			if newMap[keyAsString], err = templateSubstitutions(val, templateContext, tplFn, recursionDepth+1); err != nil {
+				return nil, err
+			}
+		}
+		return newMap, nil
+	case []interface{}:
+		newSlice := make([]interface{}, len(tplVal))
+		for i, v := range tplVal {
+			newVal, err := templateSubstitutions(v, templateContext, tplFn, recursionDepth+1)
+			if err != nil {
+				return nil, err
+			}
+			newSlice[i] = newVal
+		}
+		return newSlice, nil
+	default:
+		return tplVal, nil
+	}
+}
+
 func processLabelValues(valuesMap map[string]interface{}, clusterLabels map[string]string) error {
 	prefix := "global.fleet.clusterLabels."
 	for key, val := range valuesMap {


Test patch:
diff --git a/pkg/target/target_test.go b/pkg/target/target_test.go
index 982ddf06..55b4c5ad 100644
--- a/pkg/target/target_test.go
+++ b/pkg/target/target_test.go
@@ -1,8 +1,11 @@
 package target
 
 import (
+	"fmt"
+	"strings"
 	"testing"
 
+	"github.com/pkg/errors"
 	"github.com/rancher/wrangler/pkg/yaml"
 
 	"github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
@@ -104,3 +107,375 @@ func TestProcessLabelValues(t *testing.T) {
 		t.Fatal("label replacement not performed in third element")
 	}
 }
+
+const bundleYamlWithTemplate = `namespace: default
+helm:
+  releaseName: labels
+  values:
+    clusterName: "{{ .ClusterLabels.name }}"
+    fromAnnotation: "{{ .ClusterAnnotations.testAnnotation }}"
+    clusterNamespace: "{{ .ClusterNamespace }}"
+    fleetClusterName: "{{ .ClusterName }}"
+    reallyLongClusterName: kubernets.io/cluster/{{ index .ClusterLabels "really-long-label-name-with-many-many-characters-in-it" }}
+    customStruct:
+      - name: "{{ .Values.topLevel }}"
+        key1: value1
+        key2: value2
+      - element2: "{{ .Values.nested.secondTier.thirdTier }}"
+      - "element3_{{ .ClusterLabels.envType }}": "{{ .ClusterLabels.name }}"
+    funcs:
+      upper: "{{ .Values.topLevel | upper }}_test"
+      join: '{{ .Values.list | join "," }}'
+diff:
+  comparePatches:
+  - apiVersion: networking.k8s.io/v1
+    kind: Ingress
+    name: labels-fleetlabelsdemo
+    namespace: default
+    operations:
+    - op: remove
+      path: /spec/rules/0/host
+`
+
+func TestProcessTemplateValues(t *testing.T) {
+
+	templateValues := map[string]interface{}{
+		"topLevel": "foo",
+		"nested": map[string]interface{}{
+			"secondTier": map[string]interface{}{
+				"thirdTier": "bar",
+			},
+		},
+		"list": []string{
+			"alpha",
+			"beta",
+			"omega",
+		},
+	}
+
+	clusterLabels := map[string]string{
+		"name":    "local",
+		"envType": "dev",
+		"really-long-label-name-with-many-many-characters-in-it": "foobar",
+	}
+
+	clusterAnnotations := map[string]string{
+		"testAnnotation": "test",
+	}
+
+	values := map[string]interface{}{
+		"ClusterNamespace":   "dev-clusters",
+		"ClusterName":        "my-cluster",
+		"ClusterLabels":      clusterLabels,
+		"ClusterAnnotations": clusterAnnotations,
+		"Values":             templateValues,
+	}
+
+	bundle := &v1alpha1.BundleSpec{}
+	err := yaml.Unmarshal([]byte(bundleYamlWithTemplate), bundle)
+	if err != nil {
+		t.Fatalf("error during yaml parsing %v", err)
+	}
+
+	templatedValues, err := processTemplateValues(bundle.Helm.Values.Data, values)
+	if err != nil {
+		t.Fatalf("error during label processing %v", err)
+	}
+
+	clusterName, ok := templatedValues["clusterName"]
+	if !ok {
+		t.Fatal("key clusterName not found")
+	}
+
+	if clusterName != "local" {
+		t.Fatal("unable to assert correct clusterName")
+	}
+
+	fromAnnotation, ok := templatedValues["fromAnnotation"]
+	if !ok {
+		t.Fatal("key fromAnnotation not found")
+	}
+
+	if fromAnnotation != "test" {
+		t.Fatal("unable to assert correct value for fromAnnotation")
+	}
+
+	clusterNamespace, ok := templatedValues["clusterNamespace"]
+	if !ok {
+		t.Fatal("key clusterNamespace not found")
+	}
+
+	if clusterNamespace != "dev-clusters" {
+		t.Fatal("unable to assert correct value for clusterNamespace")
+	}
+
+	fleetClusterName, ok := templatedValues["fleetClusterName"]
+	if !ok {
+		t.Fatal("key clusterName not found")
+	}
+
+	if fleetClusterName != "my-cluster" {
+		t.Fatal("unable to assert correct value fleetClusterName")
+	}
+
+	reallyLongClusterName, ok := templatedValues["reallyLongClusterName"]
+	if !ok {
+		t.Fatal("key reallyLongClusterName not found")
+	}
+
+	if reallyLongClusterName != "kubernets.io/cluster/foobar" {
+		t.Fatal("unable to assert correct value reallyLongClusterName")
+	}
+
+	customStruct, ok := templatedValues["customStruct"].([]interface{})
+	if !ok {
+		t.Fatal("key customStruct not found")
+	}
+
+	firstMap, ok := customStruct[0].(map[string]interface{})
+	if !ok {
+		t.Fatal("unable to assert first element to map[string]interface{}")
+	}
+
+	firstElemVal, ok := firstMap["name"]
+	if !ok {
+		t.Fatal("unable to find key name in the first element of customStruct")
+	}
+
+	if firstElemVal.(string) != "foo" {
+		t.Fatal("label replacement not performed in first element")
+	}
+
+	secondElement, ok := customStruct[1].(map[string]interface{})
+	if !ok {
+		t.Fatal("unable to assert second element of customStruct to map[string]interface{}")
+	}
+
+	secondElemVal, ok := secondElement["element2"]
+	if !ok {
+		t.Fatal("unable to find key element2")
+	}
+
+	if secondElemVal.(string) != "bar" {
+		t.Fatal("template replacement not performed in second element")
+	}
+
+	thirdElement, ok := customStruct[2].(map[string]interface{})
+	if !ok {
+		t.Fatal("unable to assert second element of customStruct to map[string]interface{}")
+	}
+
+	thirdElemVal, ok := thirdElement["element3_dev"]
+	if !ok {
+		t.Fatal("unable to find key element3_dev")
+	}
+
+	if thirdElemVal.(string) != "local" {
+		t.Fatal("template replacement not performed in third element")
+	}
+
+	funcs, ok := templatedValues["funcs"].(map[string]interface{})
+	if !ok {
+		t.Fatal("key funcs not found")
+	}
+
+	upper, ok := funcs["upper"]
+	if !ok {
+		t.Fatal("key upper not found")
+	}
+
+	if upper.(string) != "FOO_test" {
+		t.Fatal("upper func was not right")
+	}
+
+	join, ok := funcs["join"]
+	if !ok {
+		t.Fatal("key join not found")
+	}
+
+	if join.(string) != "alpha,beta,omega" {
+		t.Fatal("join func was not right")
+	}
+
+}
+
+const clusterYamlWithTemplateValues = `apiVersion: fleet.cattle.io/v1alpha1
+kind: Cluster
+metadata:
+  name: test-cluster
+  namespace: test-namespace
+  labels:
+    testLabel: test-label-value
+spec:
+  templateValues:
+    someKey: someValue
+`
+
+func getClusterAndBundle(bundleYaml string) (*v1alpha1.Cluster, *v1alpha1.BundleDeploymentOptions, error) {
+	cluster := &v1alpha1.Cluster{}
+	err := yaml.Unmarshal([]byte(clusterYamlWithTemplateValues), cluster)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "error during cluster yaml parsing")
+	}
+
+	bundle := &v1alpha1.BundleDeploymentOptions{}
+	err = yaml.Unmarshal([]byte(bundleYaml), bundle)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "error during bundle yaml parsing")
+	}
+
+	return cluster, bundle, nil
+}
+
+const bundleYamlWithDisablePreProcessEnabled = `namespace: default
+helm:
+  disablePreprocess: true
+  releaseName: labels
+  values:
+    clusterName: "{{ .ClusterName }}"
+    clusterContext: "{{ .Values.someKey }}"
+    templateFn: '{{ index .ClusterLabels "testLabel" }}'
+    syntaxError: "{{ non_existent_function }}"
+`
+
+func TestDisablePreProcessFlagEnabled(t *testing.T) {
+	cluster, bundle, err := getClusterAndBundle(bundleYamlWithDisablePreProcessEnabled)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	err = preprocessHelmValues(bundle, cluster)
+	if err != nil {
+		t.Fatalf("error during cluster processing %v", err)
+	}
+
+	valuesObj := bundle.Helm.Values.Data
+
+	for _, testCase := range []struct {
+		Key           string
+		ExpectedValue string
+	}{
+		{
+			Key:           "clusterName",
+			ExpectedValue: "{{ .ClusterName }}",
+		},
+		{
+			Key:           "clusterContext",
+			ExpectedValue: "{{ .Values.someKey }}",
+		},
+		{
+			Key:           "templateFn",
+			ExpectedValue: "{{ index .ClusterLabels \"testLabel\" }}",
+		},
+		{
+			Key:           "syntaxError",
+			ExpectedValue: "{{ non_existent_function }}",
+		},
+	} {
+		if field, ok := valuesObj[testCase.Key]; !ok {
+			t.Fatalf("key %s not found", testCase.Key)
+		} else {
+			if field != testCase.ExpectedValue {
+				t.Fatalf("key %s was not the expected value. Expected: '%s' Actual: '%s'", testCase.Key, field, testCase.ExpectedValue)
+			}
+		}
+
+	}
+
+}
+
+const bundleYamlWithDisablePreProcessDisabled = `namespace: default
+helm:
+  disablePreprocess: false
+  releaseName: labels
+  values:
+    clusterName: "{{ .ClusterName }}"
+`
+
+func TestDisablePreProcessFlagDisabled(t *testing.T) {
+	cluster, bundle, err := getClusterAndBundle(bundleYamlWithDisablePreProcessDisabled)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	err = preprocessHelmValues(bundle, cluster)
+	if err != nil {
+		t.Fatalf("error during cluster processing %v", err)
+	}
+
+	valuesObj := bundle.Helm.Values.Data
+
+	key := "clusterName"
+	expectedValue := "test-cluster"
+
+	if field, ok := valuesObj[key]; !ok {
+		t.Fatalf("key %s not found", key)
+	} else {
+		if field != expectedValue {
+			t.Fatalf("key %s was not the expected value. Expected: '%s' Actual: '%s'", key, field, expectedValue)
+		}
+	}
+
+}
+
+const bundleYamlWithDisablePreProcessMissing = `namespace: default
+helm:
+  releaseName: labels
+  values:
+    clusterName: "{{ .ClusterName }}"
+`
+
+func TestDisablePreProcessFlagMissing(t *testing.T) {
+	cluster, bundle, err := getClusterAndBundle(bundleYamlWithDisablePreProcessMissing)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	err = preprocessHelmValues(bundle, cluster)
+	if err != nil {
+		t.Fatalf("error during cluster processing %v", err)
+	}
+
+	valuesObj := bundle.Helm.Values.Data
+
+	key := "clusterName"
+	expectedValue := "test-cluster"
+
+	if field, ok := valuesObj[key]; !ok {
+		t.Fatalf("key %s not found", key)
+	} else {
+		if field != expectedValue {
+			t.Fatalf("key %s was not the expected value. Expected: '%s' Actual: '%s'", key, field, expectedValue)
+		}
+	}
+
+}
+
+func TestRecursionDepthForTemplating(t *testing.T) {
+	var bundleYaml = `namespace: default
+helm:
+  releaseName: labels
+  values:`
+	for i := 1; i <= maxTemplateRecursionDepth+1; i++ {
+		indent := " "
+		offset := strings.Repeat(indent, 2)
+		line := fmt.Sprintf("\n%s%s\"%d\":", offset, strings.Repeat(indent, i), i)
+		bundleYaml = bundleYaml + line
+	}
+	bundleYaml = bundleYaml + " final_value"
+
+	cluster, bundle, err := getClusterAndBundle(bundleYaml)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	err = preprocessHelmValues(bundle, cluster)
+	if err == nil {
+		t.Fatal("expected preprocessHelmValues to return an error, it did not.")
+	}
+
+	if !strings.HasPrefix(err.Error(), "maximum recursion depth") {
+		t.Fatalf("expected error to be about recursion, instead got: %v", err)
+	}
+
+}