Repository: grafana/tanka
Pull request #436 (2020-12-10 17:12:33)

docs(jsonnet): add kustomize and update helm
Title, yes.


Patch:
diff --git a/Dockerfile b/Dockerfile
index bf58b8cc..d1ca901e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -13,11 +13,15 @@ RUN git clone https://github.com/jsonnet-bundler/jsonnet-bundler &&\
     make static &&\
     mv _output/jb /usr/local/bin/jb
 
+FROM alpine as helm
+RUN apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/testing helm
+
 # assemble final container
 FROM alpine
 RUN apk add --no-cache coreutils diffutils less git openssh-client
 COPY tk /usr/local/bin/tk
 COPY --from=kubectl /usr/local/bin/kubectl /usr/local/bin/kubectl
 COPY --from=jb /usr/local/bin/jb /usr/local/bin/jb
+COPY --from=helm /usr/bin/helm /usr/local/bin/helm
 WORKDIR /app
 ENTRYPOINT ["/usr/local/bin/tk"]
diff --git a/cmd/tk/tool.go b/cmd/tk/tool.go
index 6bf1bc44..84b1fc3f 100644
--- a/cmd/tk/tool.go
+++ b/cmd/tk/tool.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -30,29 +31,40 @@ func toolCmd() *cli.Command {
 
 func jpathCmd() *cli.Command {
 	cmd := &cli.Command{
-		Short: "print information about the jpath",
-		Use:   "jpath",
-		Run: func(cmd *cli.Command, args []string) error {
-			pwd, err := os.Getwd()
-			if err != nil {
-				return err
-			}
-			path, base, root, err := jpath.Resolve(pwd)
-			if err != nil {
-				return fmt.Errorf("Resolving JPATH: %s", err)
-			}
-			entrypoint, err := jpath.Entrypoint(base)
-			if err != nil {
-				return fmt.Errorf("Resolving JPATH: %s", err)
-			}
-			fmt.Println("main:", entrypoint)
-			fmt.Println("rootDir:", root)
-			fmt.Println("baseDir:", base)
-			fmt.Println("jpath:", path)
+		Short: "export JSONNET_PATH for use with other jsonnet tools",
+		Use:   "jpath [<file/dir>]",
+		Args:  workflowArgs,
+	}
+
+	debug := cmd.Flags().BoolP("debug", "d", false, "show debug info")
+
+	cmd.Run = func(cmd *cli.Command, args []string) error {
+		path := args[0]
 
-			return nil
-		},
+		entrypoint, err := jpath.Entrypoint(path)
+		if err != nil {
+			return fmt.Errorf("Resolving JPATH: %s", err)
+		}
+
+		jsonnetpath, base, root, err := jpath.Resolve(entrypoint)
+		if err != nil {
+			return fmt.Errorf("Resolving JPATH: %s", err)
+		}
+
+		if *debug {
+			// log to debug info to stderr
+			log.Println("main:", entrypoint)
+			log.Println("rootDir:", root)
+			log.Println("baseDir:", base)
+			log.Println("jpath:", jsonnetpath)
+		}
+
+		// print export JSONNET_PATH to stdout
+		fmt.Printf("%s", strings.Join(jsonnetpath, ":"))
+
+		return nil
 	}
+
 	return cmd
 }
 
diff --git a/docs/docs/helm.mdx b/docs/docs/helm.mdx
index 377f7b2e..2f615390 100644
--- a/docs/docs/helm.mdx
+++ b/docs/docs/helm.mdx
@@ -17,17 +17,18 @@ resource definition, you can still consume Helm resources, as described below.
 ## Consuming Helm Charts from Jsonnet
 
 Helm support is provided using the
-[`github.com/grafana/jsonnet-libs/helm-util`](https://github.com/grafana/jsonnet-libs/tree/master/helm-util)
+[`github.com/grafana/jsonnet-libs/tanka-util`](https://github.com/grafana/jsonnet-libs/tree/master/tanka-util)
 library.  Install it with:
 ```bash
-jb install github.com/grafana/jsonnet-libs/helm-util
+jb install github.com/grafana/jsonnet-libs/tanka-util
 ```
 
 The following example shows how to extract the individual resources of the
 [`grafana`](https://hub.helm.sh/charts/grafana/grafana) Helm Chart:
 
 ```jsonnet
-local helm = (import "github.com/grafana/jsonnet-libs/helm-util/helm.libsonnet").new(std.thisFile);
+local tanka = import "github.com/grafana/jsonnet-libs/tanka-util/main.libsonnet";
+local helm = tanka.helm.new(std.thisFile);
 
 {
   grafana: helm.template("grafana", "./charts/grafana", {
@@ -165,11 +166,12 @@ using [`TANKA_HELM_PATH`](/env-vars#tanka_helm_path)
 ### opts.calledFrom unset
 
 This occurs, when Tanka was not told where it `helm.template()` was invoked
-from. This most likely means you didn't call `new(std.thisFile)` when importing `helm-util`:
+from. This most likely means you didn't call `new(std.thisFile)` when importing `tanka-util`:
 
 ```jsonnet
-local helm = (import "github.com/grafana/jsonnet-libs/helm-util/helm.libsonnet").new(std.thisFile);
-                                                                                 ↑ This is important
+local tanka = import "github.com/grafana/jsonnet-libs/tanka-util/main.libsonnet";
+local helm = tanka.helm.new(std.thisFile);
+                       ↑ This is important
 ```
 
 ### Failed to find Chart
diff --git a/docs/docs/kustomize.mdx b/docs/docs/kustomize.mdx
new file mode 100644
index 00000000..92295ea5
--- /dev/null
+++ b/docs/docs/kustomize.mdx
@@ -0,0 +1,126 @@
+---
+name: "Kustomize support"
+route: "/kustomize"
+menu: Advanced features
+---
+
+# Kustomize Support
+
+[Kustomize](https://kustomize.io) provides a solution for customizing Kubernetes
+manifests in YAML.
+
+Even though Grafana Tanka uses the [Jsonnet language](/jsonnet/overview) for
+resource definition, you can still consume kustomizations, as described below.
+
+> **Warning:** Keep in mind this feature is considered EXPERIMENTAL
+
+## Consuming a Kustomization from Jsonnet
+
+Kustomize support is provided using the
+[`github.com/grafana/jsonnet-libs/tanka-util`](https://github.com/grafana/jsonnet-libs/tree/master/tanka-util)
+library.  Install it with:
+
+```bash
+jb install github.com/grafana/jsonnet-libs/tanka-util
+```
+
+The following example shows how to extract the individual resources of the
+[`flux2/source-controller`](https://github.com/fluxcd/flux2/tree/main/manifests/bases/source-controller)
+kustomization:
+
+```jsonnet
+local tanka = import 'github.com/grafana/jsonnet-libs/tanka-util/main.libsonnet';
+local kustomize = tanka.kustomize.new(std.thisFile);
+
+{
+  source_controller: kustomize.build(path='flux2')
+}
+```
+
+Kustomize takes a kustomization manifest as input. Go on an create this file
+`flux2/kustomization.yaml` relative to above jsonnet:
+
+```yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- https://github.com/fluxcd/flux2/archive/v0.4.3.zip//flux2-0.4.3/manifests/bases/source-controller
+```
+
+> **Important:** You MUST include the `.new(std.thisFile)` part in the import.
+> This is what tells Tanka where you actually call `kustomize.build()` from, so
+> it can find your kustomization manifest.
+
+<br />
+
+Once invoked, the `$.source_controller` key holds the individual resources of
+the kustomization as a regular Jsonnet object that looks roughly like so:
+
+```jsonnet
+{
+  'custom_resource_definition_buckets.source.toolkit.fluxcd.io': {/* ... */ },
+  'custom_resource_definition_gitrepositories.source.toolkit.fluxcd.io': {/* ... */ },
+  'custom_resource_definition_helmcharts.source.toolkit.fluxcd.io': {/* ... */ },
+  'custom_resource_definition_helmrepositories.source.toolkit.fluxcd.io': {/* ... */ },
+  deployment_source_controller: {/* ... */ },
+  service_source_controller: {/* ... */ },
+}
+```
+
+Above can be [manipulated](/tutorial/environments#patching) in the same way as
+any other Jsonnet data.
+
+## Working with Kustomize
+
+Tanka, like Jsonnet, is hermetic. It **always yields the same resources** when
+the project is strictly self-contained.
+
+Kustomize however has the ability to pull
+[resources](https://kubectl.docs.kubernetes.io/references/kustomize/resource/)
+from different sources at runtime, which violates above requirement. This is
+also apparent in the example above.
+
+> **Important:** Due to the nature of Kustomize, it is not feasible to ensure
+> hermetic and reproducible kustomize builds from within Tanka. Beware of that
+> when using Kustomize.
+
+## Troubleshooting
+
+### Kustomize executable missing
+
+Kustomize support in Tanka requires the `kustomize` binary installed on your
+system and available on the `$PATH`. If Kustomize is not installed, you will see
+this error message:
+
+```
+evaluating jsonnet: RUNTIME ERROR: Expanding Kustomize: exec: "kustomize": executable file not found in $PATH
+```
+
+To solve this, you need to 
+[install Kustomize](https://kubectl.docs.kubernetes.io/installation/kustomize/).
+If you cannot install it system-wide, you can point Tanka at your executable 
+using [`TANKA_KUSTOMIZE_PATH`](/env-vars#tanka_kustomize_path)
+
+### opts.calledFrom unset
+
+This occurs, when Tanka was not told where it `kustomize.build()` was invoked
+from. This most likely means you didn't call `new(std.thisFile)` when importing `tanka-util`:
+
+```jsonnet
+local tanka = import "github.com/grafana/jsonnet-libs/tanka-util/main.libsonnet";
+local helm = tanka.helm.new(std.thisFile);
+                       ↑ This is important
+```
+
+### Failed to find kustomization
+
+```
+helmTemplate: Failed to find a kustomization at 'flux2': No such file or directory.
+helmTemplate: Failed to find a kustomization at '/home/user/stuff/tanka/environments/default/flux2': No such file or directory.
+```
+
+Tanka failed to locate your kustomization on the filesystem. It looked at the
+relative path you provided in `kustomize.build()`, starting from the directory
+of the file you called `kustomize.build()` from.
+
+Please check there is actually a valid kustomization at this place.
diff --git a/docs/doczrc.js b/docs/doczrc.js
index 8014b222..37d36426 100644
--- a/docs/doczrc.js
+++ b/docs/doczrc.js
@@ -48,6 +48,7 @@ export default {
       menu: [
         "Garbage collection",
         "Helm support",
+        "Kustomize support",
         "Output filtering",
         "Exporting as YAML",
       ],
diff --git a/pkg/helm/charts.go b/pkg/helm/charts.go
index 0876349b..341e467f 100644
--- a/pkg/helm/charts.go
+++ b/pkg/helm/charts.go
@@ -68,6 +68,11 @@ type Charts struct {
 	Helm Helm
 }
 
+// chartManifest represents a Helm chart's Chart.yaml
+type chartManifest struct {
+	Version string `yaml:"version"`
+}
+
 // ChartDir returns the directory pulled charts are saved in
 func (c Charts) ChartDir() string {
 	return filepath.Join(c.projectRoot, c.Manifest.Directory)
@@ -95,12 +100,33 @@ func (c Charts) Vendor() error {
 	for _, r := range c.Manifest.Requires {
 		chartName := parseReqName(r.Chart)
 		chartPath := filepath.Join(dir, chartName)
-		if _, err := os.Stat(chartPath); err == nil {
-			log.Printf(" %s@%s exists", r.Chart, r.Version.String())
-			continue
+
+		_, err := os.Stat(chartPath)
+		if err == nil {
+			chartManifestPath := filepath.Join(chartPath, "Chart.yaml")
+			chartManifestBytes, err := ioutil.ReadFile(chartManifestPath)
+			if err != nil {
+				return fmt.Errorf("reading chart manifest: %w", err)
+			}
+			var chartYAML chartManifest
+			if err := yaml.Unmarshal(chartManifestBytes, &chartYAML); err != nil {
+				return fmt.Errorf("unmarshalling chart manifest: %w", err)
+			}
+
+			if chartYAML.Version == r.Version.String() {
+				log.Printf(" %s@%s exists", r.Chart, r.Version.String())
+				continue
+			} else {
+				log.Printf("Removing %s@%s", r.Chart, r.Version.String())
+				if err := os.RemoveAll(chartPath); err != nil {
+					return err
+				}
+			}
+		} else if !os.IsNotExist(err) {
+			return err
 		}
 
-		err := c.Helm.Pull(r.Chart, r.Version.String(), PullOpts{
+		err = c.Helm.Pull(r.Chart, r.Version.String(), PullOpts{
 			Destination: dir,
 			Opts:        Opts{Repositories: c.Manifest.Repositories},
 		})
diff --git a/pkg/kustomize/jsonnet.go b/pkg/kustomize/jsonnet.go
index e2d8fa8f..c857ec0b 100644
--- a/pkg/kustomize/jsonnet.go
+++ b/pkg/kustomize/jsonnet.go
@@ -47,7 +47,7 @@ func NativeFunc(k Kustomize) *jsonnet.NativeFunction {
 			callerDir := filepath.Dir(opts.CalledFrom)
 			actual_path := filepath.Join(callerDir, path)
 			if _, err := os.Stat(actual_path); err != nil {
-				return nil, fmt.Errorf("kustomizeBuild: Failed to find kustomize at '%s': %s.", actual_path, err)
+				return nil, fmt.Errorf("kustomizeBuild: Failed to find kustomization at '%s': %s. See https://tanka.dev/kustomize#failed-to-find-kustomization", actual_path, err)
 			}
 
 			// render resources
@@ -79,7 +79,7 @@ func parseOpts(data interface{}) (*JsonnetOpts, error) {
 
 	// Kustomize paths are only allowed at relative paths. Use conf.CalledFrom to find the callers directory
 	if opts.CalledFrom == "" {
-		return nil, fmt.Errorf("kustomizeBuild: 'opts.calledFrom' is unset or empty.\nTanka needs this to find your Kustomize.\n")
+		return nil, fmt.Errorf("kustomizeBuild: 'opts.calledFrom' is unset or empty.\nTanka needs this to find your Kustomize. See https://tanka.dev/kustomize#optscalledfrom-unset\n")
 	}
 
 	return &opts, nil
diff --git a/pkg/spec/spec.go b/pkg/spec/spec.go
index 93171e8b..2d0185f8 100644
--- a/pkg/spec/spec.go
+++ b/pkg/spec/spec.go
@@ -20,7 +20,7 @@ const Specfile = "spec.json"
 
 // ParseDir parses the given environments `spec.json` into a `v1alpha1.Environment`
 // object with the name set to the directories name
-func ParseDir(baseDir, name string) (*v1alpha1.Environment, error) {
+func ParseDir(baseDir, path string) (*v1alpha1.Environment, error) {
 	fi, err := os.Stat(baseDir)
 	if err != nil {
 		return nil, err
@@ -33,23 +33,24 @@ func ParseDir(baseDir, name string) (*v1alpha1.Environment, error) {
 	if err != nil {
 		if os.IsNotExist(err) {
 			c := v1alpha1.New()
-			c.Metadata.Name = name
-			return c, ErrNoSpec{name}
+			c.Metadata.Name = path // legacy behavior
+			c.Metadata.Namespace = path
+			return c, ErrNoSpec{path}
 		}
 		return nil, err
 	}
 
-	c, err := Parse(data)
+	c, err := Parse(data, path)
 	if c != nil {
 		// set the name field
-		c.Metadata.Name = name
+		c.Metadata.Name = path // legacy behavior
 	}
 
 	return c, err
 }
 
 // Parse parses the json `data` into a `v1alpha1.Environment` object.
-func Parse(data []byte) (*v1alpha1.Environment, error) {
+func Parse(data []byte, path string) (*v1alpha1.Environment, error) {
 	config := v1alpha1.New()
 	if err := json.Unmarshal(data, config); err != nil {
 		return nil, errors.Wrap(err, "parsing spec.json")
@@ -64,6 +65,8 @@ func Parse(data []byte) (*v1alpha1.Environment, error) {
 		config.Spec.APIServer = "https://" + config.Spec.APIServer
 	}
 
+	config.Metadata.Namespace = path
+
 	return config, nil
 }
 
diff --git a/pkg/spec/v1alpha1/environment.go b/pkg/spec/v1alpha1/environment.go
index e6031ed0..413676ed 100644
--- a/pkg/spec/v1alpha1/environment.go
+++ b/pkg/spec/v1alpha1/environment.go
@@ -29,8 +29,9 @@ type Environment struct {
 
 // Metadata is meant for humans and not parsed
 type Metadata struct {
-	Name   string            `json:"name,omitempty"`
-	Labels map[string]string `json:"labels,omitempty"`
+	Name      string            `json:"name,omitempty"`
+	Namespace string            `json:"namespace,omitempty"`
+	Labels    map[string]string `json:"labels,omitempty"`
 }
 
 // Has and Get make Metadata a simple wrapper for labels.Labels to use our map in their querier
diff --git a/pkg/tanka/parse.go b/pkg/tanka/parse.go
index c79a85ea..30808003 100644
--- a/pkg/tanka/parse.go
+++ b/pkg/tanka/parse.go
@@ -184,7 +184,11 @@ func ParseEnv(path string, opts ParseOpts) (interface{}, *v1alpha1.Environment,
 		if err != nil {
 			return nil, nil, err
 		}
-		env, err = spec.Parse(marshalled)
+		relpath, err := relativeEntrypoint(path)
+		if err != nil {
+			return nil, nil, err
+		}
+		env, err = spec.Parse(marshalled, relpath)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -244,3 +248,16 @@ func extractEnvironments(data interface{}) (manifest.List, error) {
 	// Extract only object of Kind: Environment
 	return process.Filter(out, process.MustStrExps("Environment/.*")), nil
 }
+
+func relativeEntrypoint(path string) (string, error) {
+	entrypoint, err := jpath.Entrypoint(path)
+	if err != nil {
+		return "", err
+	}
+	_, _, rootDir, err := jpath.Resolve(entrypoint)
+	if err != nil {
+		return "", errors.Wrap(err, "resolving jpath")
+	}
+
+	return filepath.Rel(rootDir, entrypoint)
+}


Test patch:
diff --git a/pkg/spec/depreciations_test.go b/pkg/spec/depreciations_test.go
index 2644a0ff..374afb60 100644
--- a/pkg/spec/depreciations_test.go
+++ b/pkg/spec/depreciations_test.go
@@ -26,7 +26,7 @@ func TestDeprecated(t *testing.T) {
 }
 `)
 
-	got, err := Parse(data)
+	got, err := Parse(data, "test")
 	require.Equal(t, ErrDeprecated{
 		{old: "server", new: "spec.apiServer"},
 		{old: "team", new: "metadata.labels.team"},
diff --git a/pkg/tanka/parse_test.go b/pkg/tanka/parse_test.go
index 520a484d..4145d298 100644
--- a/pkg/tanka/parse_test.go
+++ b/pkg/tanka/parse_test.go
@@ -43,8 +43,9 @@ func TestEvalJsonnet(t *testing.T) {
 				APIVersion: v1alpha1.New().APIVersion,
 				Kind:       v1alpha1.New().Kind,
 				Metadata: v1alpha1.Metadata{
-					Name:   "cases/withspecjson",
-					Labels: v1alpha1.New().Metadata.Labels,
+					Name:      "cases/withspecjson",
+					Namespace: "cases/withspecjson",
+					Labels:    v1alpha1.New().Metadata.Labels,
 				},
 				Spec: v1alpha1.Spec{
 					APIServer: "https://localhost",
@@ -64,8 +65,9 @@ func TestEvalJsonnet(t *testing.T) {
 				APIVersion: v1alpha1.New().APIVersion,
 				Kind:       v1alpha1.New().Kind,
 				Metadata: v1alpha1.Metadata{
-					Name:   "cases/withspecjson",
-					Labels: v1alpha1.New().Metadata.Labels,
+					Name:      "cases/withspecjson",
+					Namespace: "cases/withspecjson",
+					Labels:    v1alpha1.New().Metadata.Labels,
 				},
 				Spec: v1alpha1.Spec{
 					APIServer: "https://localhost",
@@ -96,8 +98,9 @@ func TestEvalJsonnet(t *testing.T) {
 				APIVersion: v1alpha1.New().APIVersion,
 				Kind:       v1alpha1.New().Kind,
 				Metadata: v1alpha1.Metadata{
-					Name:   "withenv",
-					Labels: v1alpha1.New().Metadata.Labels,
+					Name:      "withenv",
+					Namespace: "cases/withenv/main.jsonnet",
+					Labels:    v1alpha1.New().Metadata.Labels,
 				},
 				Spec: v1alpha1.Spec{
 					APIServer: "https://localhost",