Repository: grafana/tanka
Pull request #422 (2020-11-17 23:21:56)

feat: render kustomize into jsonnet
Similar to Helm, render Kustomize into jsonnet objects.

I didn't change `jsonnet_test.go`, these are the same tests we run against pkg/helm.

```
[I] ➜ cat kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- https://github.com/fluxcd/flux2/archive/v0.2.6.zip//flux2-0.2.6/manifests/bases/source-controller

[I] ➜ cat manifests.jsonnet
{
  source_controller:
    std.native('kustomizeBuild')('.', { calledFrom: std.thisFile }),
}
```

Duologic: To do after this:
* document Kustomize integration (including links from error messages)
* provide jsonnet-lib similar to helm-util

Patch:
diff --git a/docs/docs/env-vars.md b/docs/docs/env-vars.md
index 7999d543..27545ded 100644
--- a/docs/docs/env-vars.md
+++ b/docs/docs/env-vars.md
@@ -25,3 +25,8 @@ menu: "References"
 
 **Description**: Path to the `helm` executable  
 **Default**: `$PATH/helm`
+
+### TANKA_KUSTOMIZE_PATH
+
+**Description**: Path to the `kustomize` executable  
+**Default**: `$PATH/kustomize`
diff --git a/pkg/helm/jsonnet.go b/pkg/helm/jsonnet.go
index ced9a8cf..3a03596e 100644
--- a/pkg/helm/jsonnet.go
+++ b/pkg/helm/jsonnet.go
@@ -1,14 +1,11 @@
 package helm
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
-	"text/template"
 
-	"github.com/Masterminds/sprig/v3"
 	"github.com/google/go-jsonnet"
 	"github.com/google/go-jsonnet/ast"
 	"github.com/grafana/tanka/pkg/kubernetes/manifest"
@@ -71,7 +68,7 @@ func NativeFunc(h Helm) *jsonnet.NativeFunction {
 			}
 
 			// convert list to map
-			out, err := listAsMap(list, opts.NameFormat)
+			out, err := manifest.ListAsMap(list, opts.NameFormat)
 			if err != nil {
 				return nil, err
 			}
@@ -98,43 +95,3 @@ func parseOpts(data interface{}) (*JsonnetOpts, error) {
 
 	return &opts, nil
 }
-
-func listAsMap(list manifest.List, nameFormat string) (map[string]interface{}, error) {
-	if nameFormat == "" {
-		nameFormat = DefaultNameFormat
-	}
-
-	tmpl, err := template.New("").
-		Funcs(sprig.TxtFuncMap()).
-		Parse(nameFormat)
-	if err != nil {
-		return nil, fmt.Errorf("Parsing name format: %w", err)
-	}
-
-	out := make(map[string]interface{})
-	for _, m := range list {
-		var buf bytes.Buffer
-		if err := tmpl.Execute(&buf, m); err != nil {
-			return nil, err
-		}
-		name := buf.String()
-
-		if _, ok := out[name]; ok {
-			return nil, ErrorDuplicateName{name: name, format: nameFormat}
-		}
-		out[name] = map[string]interface{}(m)
-	}
-
-	return out, nil
-}
-
-// ErrorDuplicateName means two resources share the same name using the given
-// nameFormat.
-type ErrorDuplicateName struct {
-	name   string
-	format string
-}
-
-func (e ErrorDuplicateName) Error() string {
-	return fmt.Sprintf("Two resources share the same name '%s'. Please adapt the name template '%s'. See https://tanka.dev/helm#two-resources-share-the-same-name", e.name, e.format)
-}
diff --git a/pkg/jsonnet/native/funcs.go b/pkg/jsonnet/native/funcs.go
index 336cef54..e6ef0f06 100644
--- a/pkg/jsonnet/native/funcs.go
+++ b/pkg/jsonnet/native/funcs.go
@@ -10,6 +10,7 @@ import (
 	jsonnet "github.com/google/go-jsonnet"
 	"github.com/google/go-jsonnet/ast"
 	"github.com/grafana/tanka/pkg/helm"
+	"github.com/grafana/tanka/pkg/kustomize"
 	"github.com/pkg/errors"
 	yaml "gopkg.in/yaml.v3"
 )
@@ -32,6 +33,7 @@ func Funcs() []*jsonnet.NativeFunction {
 		regexSubst(),
 
 		helm.NativeFunc(helm.ExecHelm{}),
+		kustomize.NativeFunc(kustomize.ExecKustomize{}),
 	}
 }
 
diff --git a/pkg/kubernetes/manifest/errors.go b/pkg/kubernetes/manifest/errors.go
index 3afcf339..e1b6dbad 100644
--- a/pkg/kubernetes/manifest/errors.go
+++ b/pkg/kubernetes/manifest/errors.go
@@ -66,3 +66,14 @@ func (s SampleString) Indent(n int) string {
 	lines := strings.Split(s.String(), "\n")
 	return indent + strings.Join(lines, "\n"+indent)
 }
+
+// ErrorDuplicateName means two resources share the same name using the given
+// nameFormat.
+type ErrorDuplicateName struct {
+	name   string
+	format string
+}
+
+func (e ErrorDuplicateName) Error() string {
+	return fmt.Sprintf("Two resources share the same name '%s'. Please adapt the name template '%s'.", e.name, e.format)
+}
diff --git a/pkg/kubernetes/manifest/manifest.go b/pkg/kubernetes/manifest/manifest.go
index a62634c6..70a36aa4 100644
--- a/pkg/kubernetes/manifest/manifest.go
+++ b/pkg/kubernetes/manifest/manifest.go
@@ -4,7 +4,9 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"text/template"
 
+	"github.com/Masterminds/sprig/v3"
 	"github.com/pkg/errors"
 	"github.com/stretchr/objx"
 	yaml "gopkg.in/yaml.v2"
@@ -270,3 +272,35 @@ func m2o(m interface{}) objx.Map {
 	}
 	return nil
 }
+
+// DefaultNameFormat to use when no nameFormat is supplied
+const DefaultNameFormat = `{{ print .kind "_" .metadata.name | snakecase }}`
+
+func ListAsMap(list List, nameFormat string) (map[string]interface{}, error) {
+	if nameFormat == "" {
+		nameFormat = DefaultNameFormat
+	}
+
+	tmpl, err := template.New("").
+		Funcs(sprig.TxtFuncMap()).
+		Parse(nameFormat)
+	if err != nil {
+		return nil, fmt.Errorf("Parsing name format: %w", err)
+	}
+
+	out := make(map[string]interface{})
+	for _, m := range list {
+		var buf bytes.Buffer
+		if err := tmpl.Execute(&buf, m); err != nil {
+			return nil, err
+		}
+		name := buf.String()
+
+		if _, ok := out[name]; ok {
+			return nil, ErrorDuplicateName{name: name, format: nameFormat}
+		}
+		out[name] = map[string]interface{}(m)
+	}
+
+	return out, nil
+}
diff --git a/pkg/kustomize/build.go b/pkg/kustomize/build.go
new file mode 100644
index 00000000..ca53aea1
--- /dev/null
+++ b/pkg/kustomize/build.go
@@ -0,0 +1,40 @@
+package kustomize
+
+import (
+	"bytes"
+	"io"
+	"os"
+
+	"github.com/grafana/tanka/pkg/kubernetes/manifest"
+	"github.com/pkg/errors"
+	yaml "gopkg.in/yaml.v3"
+)
+
+// Build expands a Kustomize into a regular manifest.List using the `kustomize
+// build` command
+func (k ExecKustomize) Build(path string) (manifest.List, error) {
+	cmd := k.cmd("build", path)
+	var buf bytes.Buffer
+	cmd.Stdout = &buf
+	cmd.Stderr = os.Stderr
+
+	if err := cmd.Run(); err != nil {
+		return nil, errors.Wrap(err, "Expanding Kustomize")
+	}
+
+	var list manifest.List
+	d := yaml.NewDecoder(&buf)
+	for {
+		var m manifest.Manifest
+		if err := d.Decode(&m); err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, errors.Wrap(err, "Parsing Kustomize output")
+		}
+
+		list = append(list, m)
+	}
+
+	return list, nil
+}
diff --git a/pkg/kustomize/jsonnet.go b/pkg/kustomize/jsonnet.go
new file mode 100644
index 00000000..e2d8fa8f
--- /dev/null
+++ b/pkg/kustomize/jsonnet.go
@@ -0,0 +1,86 @@
+package kustomize
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/google/go-jsonnet"
+	"github.com/google/go-jsonnet/ast"
+	"github.com/grafana/tanka/pkg/kubernetes/manifest"
+)
+
+// JsonnetOpts are additional properties the consumer of the native func might
+// pass.
+type JsonnetOpts struct {
+	// CalledFrom is the file that calls kustomizeBuild. This is used to find the
+	// vendored Kustomize relative to this file
+	CalledFrom string `json:"calledFrom"`
+	// NameBuild is used to create the keys in the resulting map
+	NameFormat string `json:"nameFormat"`
+}
+
+// NativeFunc returns a jsonnet native function that provides the same
+// functionality as `Kustomize.Build` of this package. Kustomize yamls are required to be
+// present on the local filesystem, at a relative location to the file that
+// calls `kustomize.build()` / `std.native('kustomizeBuild')`. This guarantees
+// hermeticity
+func NativeFunc(k Kustomize) *jsonnet.NativeFunction {
+	return &jsonnet.NativeFunction{
+		Name: "kustomizeBuild",
+		// Similar to `kustomize build {path}` where {path} is a local path
+		Params: ast.Identifiers{"path", "opts"},
+		Func: func(data []interface{}) (interface{}, error) {
+			path, ok := data[0].(string)
+			if !ok {
+				return nil, fmt.Errorf("Argument 'path' must be of 'string' type, got '%T' instead", data[0])
+			}
+
+			// TODO: validate data[1] actually follows the struct scheme
+			opts, err := parseOpts(data[1])
+			if err != nil {
+				return "", err
+			}
+
+			// resolve the Kustomize path relative to the caller
+			callerDir := filepath.Dir(opts.CalledFrom)
+			actual_path := filepath.Join(callerDir, path)
+			if _, err := os.Stat(actual_path); err != nil {
+				return nil, fmt.Errorf("kustomizeBuild: Failed to find kustomize at '%s': %s.", actual_path, err)
+			}
+
+			// render resources
+			list, err := k.Build(actual_path)
+			if err != nil {
+				return nil, err
+			}
+
+			// convert list to map
+			out, err := manifest.ListAsMap(list, opts.NameFormat)
+			if err != nil {
+				return nil, err
+			}
+
+			return out, nil
+		},
+	}
+}
+
+func parseOpts(data interface{}) (*JsonnetOpts, error) {
+	c, err := json.Marshal(data)
+	if err != nil {
+		return nil, err
+	}
+	var opts JsonnetOpts
+	if err := json.Unmarshal(c, &opts); err != nil {
+		return nil, err
+	}
+
+	// Kustomize paths are only allowed at relative paths. Use conf.CalledFrom to find the callers directory
+	if opts.CalledFrom == "" {
+		return nil, fmt.Errorf("kustomizeBuild: 'opts.calledFrom' is unset or empty.\nTanka needs this to find your Kustomize.\n")
+	}
+
+	return &opts, nil
+}
diff --git a/pkg/kustomize/kustomize.go b/pkg/kustomize/kustomize.go
new file mode 100644
index 00000000..1d4c5bd5
--- /dev/null
+++ b/pkg/kustomize/kustomize.go
@@ -0,0 +1,39 @@
+package kustomize
+
+import (
+	"os"
+	"os/exec"
+
+	"github.com/grafana/tanka/pkg/kubernetes/manifest"
+)
+
+// Kustomize provides high level access to some Kustomize operations
+type Kustomize interface {
+	// Build returns the individual resources of a Kustomize
+	Build(path string) (manifest.List, error)
+}
+
+// ExecKustomize is a Kustomize implementation powered by the `kustomize`
+// command line utility
+type ExecKustomize struct{}
+
+// cmd returns a prepared exec.Cmd to use the `kustomize` binary
+func (e ExecKustomize) cmd(action string, args ...string) *exec.Cmd {
+	argv := []string{action}
+	argv = append(argv, args...)
+
+	cmd := kustomizeCmd(argv...)
+	cmd.Stderr = os.Stderr
+
+	return cmd
+}
+
+// kustomizeCmd returns a bare exec.Cmd pointed at the local kustomize binary
+func kustomizeCmd(args ...string) *exec.Cmd {
+	bin := "kustomize"
+	if env := os.Getenv("TANKA_KUSTOMIZE_PATH"); env != "" {
+		bin = env
+	}
+
+	return exec.Command(bin, args...)
+}


Test patch:
diff --git a/pkg/helm/jsonnet_test.go b/pkg/helm/jsonnet_test.go
deleted file mode 100644
index 4f74c25d..00000000
--- a/pkg/helm/jsonnet_test.go
+++ /dev/null
@@ -1,97 +0,0 @@
-package helm
-
-import (
-	"testing"
-
-	"github.com/google/go-cmp/cmp"
-	"github.com/grafana/tanka/pkg/kubernetes/manifest"
-)
-
-func TestListAsMap(t *testing.T) {
-	cases := []struct {
-		name       string
-		list       manifest.List
-		result     map[string]interface{}
-		nameFormat string
-		err        error
-	}{
-		{
-			name:       "simple",
-			nameFormat: "", // test it properly defaults to DefaultNameFormat
-			list: manifest.List{
-				configMap("foo"),
-				deployment("bar"),
-			},
-			result: map[string]interface{}{
-				"config_map_foo": configMap("foo"),
-				"deployment_bar": deployment("bar"),
-			},
-		},
-		{
-			name: "duplicate-default",
-			list: manifest.List{
-				secret("foo", map[string]interface{}{"id": 1}),
-				secret("foo", map[string]interface{}{"id": 2}),
-			},
-			err:    ErrorDuplicateName{name: "secret_foo", format: DefaultNameFormat},
-			result: nil, // expect no result
-		},
-		{
-			name:       "duplicate-custom",
-			nameFormat: `{{ print .metadata.name "_" .data.id }}`,
-			list: manifest.List{
-				secret("foo", map[string]interface{}{"id": 1}),
-				secret("foo", map[string]interface{}{"id": 2}),
-			},
-			result: map[string]interface{}{
-				"foo_1": secret("foo", map[string]interface{}{"id": 1}),
-				"foo_2": secret("foo", map[string]interface{}{"id": 2}),
-			},
-			err: nil, // expect no error
-		},
-	}
-
-	for _, c := range cases {
-		t.Run(c.name, func(t *testing.T) {
-			result, err := listAsMap(c.list, c.nameFormat)
-			if err != c.err {
-				t.Fatalf("err mismatch: want '%s' but got '%s'", c.err, err)
-			}
-
-			if diff := cmp.Diff(c.result, result); diff != "" {
-				t.Fatal(diff)
-			}
-		})
-	}
-}
-
-func configMap(name string) map[string]interface{} {
-	return map[string]interface{}{
-		"apiVersion": "v1",
-		"kind":       "ConfigMap",
-		"metadata":   map[string]interface{}{"name": name},
-		"data":       map[string]interface{}{},
-	}
-}
-
-func deployment(name string) map[string]interface{} {
-	return map[string]interface{}{
-		"apiVersion": "apps/v1",
-		"kind":       "Deployment",
-		"metadata":   map[string]interface{}{"name": name},
-		"spec":       map[string]interface{}{},
-	}
-}
-
-func secret(name string, data map[string]interface{}) map[string]interface{} {
-	if data == nil {
-		data = map[string]interface{}{}
-	}
-
-	return map[string]interface{}{
-		"apiVersion": "v1",
-		"kind":       "Secret",
-		"metadata":   map[string]interface{}{"name": name},
-		"data":       data,
-	}
-}
diff --git a/pkg/kubernetes/manifest/manifest_test.go b/pkg/kubernetes/manifest/manifest_test.go
index fabd524e..f821f3f5 100644
--- a/pkg/kubernetes/manifest/manifest_test.go
+++ b/pkg/kubernetes/manifest/manifest_test.go
@@ -90,3 +90,92 @@ spec:
 		t.Error(s)
 	}
 }
+
+func TestListAsMap(t *testing.T) {
+	cases := []struct {
+		name       string
+		list       List
+		result     map[string]interface{}
+		nameFormat string
+		err        error
+	}{
+		{
+			name:       "simple",
+			nameFormat: "", // test it properly defaults to DefaultNameFormat
+			list: List{
+				configMap("foo"),
+				deployment("bar"),
+			},
+			result: map[string]interface{}{
+				"config_map_foo": configMap("foo"),
+				"deployment_bar": deployment("bar"),
+			},
+		},
+		{
+			name: "duplicate-default",
+			list: List{
+				secret("foo", map[string]interface{}{"id": 1}),
+				secret("foo", map[string]interface{}{"id": 2}),
+			},
+			err:    ErrorDuplicateName{name: "secret_foo", format: DefaultNameFormat},
+			result: nil, // expect no result
+		},
+		{
+			name:       "duplicate-custom",
+			nameFormat: `{{ print .metadata.name "_" .data.id }}`,
+			list: List{
+				secret("foo", map[string]interface{}{"id": 1}),
+				secret("foo", map[string]interface{}{"id": 2}),
+			},
+			result: map[string]interface{}{
+				"foo_1": secret("foo", map[string]interface{}{"id": 1}),
+				"foo_2": secret("foo", map[string]interface{}{"id": 2}),
+			},
+			err: nil, // expect no error
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			result, err := ListAsMap(c.list, c.nameFormat)
+			if err != c.err {
+				t.Fatalf("err mismatch: want '%s' but got '%s'", c.err, err)
+			}
+
+			if diff := cmp.Diff(c.result, result); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}
+
+func configMap(name string) map[string]interface{} {
+	return map[string]interface{}{
+		"apiVersion": "v1",
+		"kind":       "ConfigMap",
+		"metadata":   map[string]interface{}{"name": name},
+		"data":       map[string]interface{}{},
+	}
+}
+
+func deployment(name string) map[string]interface{} {
+	return map[string]interface{}{
+		"apiVersion": "apps/v1",
+		"kind":       "Deployment",
+		"metadata":   map[string]interface{}{"name": name},
+		"spec":       map[string]interface{}{},
+	}
+}
+
+func secret(name string, data map[string]interface{}) map[string]interface{} {
+	if data == nil {
+		data = map[string]interface{}{}
+	}
+
+	return map[string]interface{}{
+		"apiVersion": "v1",
+		"kind":       "Secret",
+		"metadata":   map[string]interface{}{"name": name},
+		"data":       data,
+	}
+}