Repository: grafana/tanka
Pull request #367 (2020-09-01 15:04:14)

feat(cli): Charttool
Adds a new `tk tool charts` command, which enables declarative
management of vendored Helm charts.

As proposed by
https://docs.google.com/document/d/171F0cm_VliMStmHe6oy5pAbuXihr-7Cb5yD-vmwqpP8,
Helm Charts should be vendored by the individual library that consumes
them.

To simplify this process, `tk tool charts` provides a declarative config
file to automate `helm pull`.

The rest of above document will be implemented in subsequent pull requests


Patch:
diff --git a/.gitignore b/.gitignore
index a5546f77..1521c8b7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,1 @@
 dist
-tk
-
diff --git a/cmd/tk/tool.go b/cmd/tk/tool.go
index 72f77e51..11d5fb3d 100644
--- a/cmd/tk/tool.go
+++ b/cmd/tk/tool.go
@@ -20,8 +20,11 @@ func toolCmd() *cli.Command {
 		Short: "handy utilities for working with jsonnet",
 		Use:   "tool [command]",
 	}
-	cmd.AddCommand(jpathCmd())
-	cmd.AddCommand(importsCmd())
+	cmd.AddCommand(
+		jpathCmd(),
+		importsCmd(),
+		chartsCmd(),
+	)
 	return cmd
 }
 
diff --git a/cmd/tk/toolCharts.go b/cmd/tk/toolCharts.go
new file mode 100644
index 00000000..217234c8
--- /dev/null
+++ b/cmd/tk/toolCharts.go
@@ -0,0 +1,126 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/go-clix/cli"
+	"github.com/grafana/tanka/pkg/helm"
+	"gopkg.in/yaml.v2"
+)
+
+func chartsCmd() *cli.Command {
+	cmd := &cli.Command{
+		Use:   "charts",
+		Short: "Declarative vendoring of Helm Charts",
+	}
+
+	cmd.AddCommand(
+		chartsInitCmd(),
+		chartsAddCmd(),
+		chartsVendorCmd(),
+		chartsConfigCmd(),
+	)
+
+	return cmd
+}
+
+func chartsVendorCmd() *cli.Command {
+	cmd := &cli.Command{
+		Use:   "vendor",
+		Short: "Download Charts to a local folder",
+	}
+
+	cmd.Run = func(cmd *cli.Command, args []string) error {
+		c, err := loadChartfile()
+		if err != nil {
+			return err
+		}
+
+		return c.Vendor()
+	}
+
+	return cmd
+}
+
+func chartsAddCmd() *cli.Command {
+	cmd := &cli.Command{
+		Use:   "add [chart@version] [...]",
+		Short: "Adds Charts to the chartfile",
+	}
+
+	cmd.Run = func(cmd *cli.Command, args []string) error {
+		c, err := loadChartfile()
+		if err != nil {
+			return err
+		}
+
+		return c.Add(args)
+	}
+
+	return cmd
+}
+
+func chartsConfigCmd() *cli.Command {
+	cmd := &cli.Command{
+		Use:   "config",
+		Short: "Displays the current manifest",
+	}
+
+	cmd.Run = func(cmd *cli.Command, args []string) error {
+		c, err := loadChartfile()
+		if err != nil {
+			return err
+		}
+
+		data, err := yaml.Marshal(c.Manifest)
+		if err != nil {
+			return err
+		}
+
+		fmt.Print(string(data))
+
+		return nil
+	}
+
+	return cmd
+}
+
+func chartsInitCmd() *cli.Command {
+	cmd := &cli.Command{
+		Use:   "init",
+		Short: "Create a new Chartfile",
+	}
+
+	cmd.Run = func(cmd *cli.Command, args []string) error {
+		wd, err := os.Getwd()
+		if err != nil {
+			return err
+		}
+
+		path := filepath.Join(wd, helm.Filename)
+		if _, err := os.Stat(path); err == nil {
+			return fmt.Errorf("Chartfile at '%s' already exists. Aborting", path)
+		}
+
+		if _, err := helm.InitChartfile(path); err != nil {
+			return err
+		}
+
+		log.Printf("Success! New Chartfile created at '%s'", path)
+		return nil
+	}
+
+	return cmd
+}
+
+func loadChartfile() (*helm.Charts, error) {
+	wd, err := os.Getwd()
+	if err != nil {
+		return nil, err
+	}
+
+	return helm.LoadChartfile(wd)
+}
diff --git a/go.mod b/go.mod
index 9db35c4c..8a06dbdc 100644
--- a/go.mod
+++ b/go.mod
@@ -21,4 +21,5 @@ require (
 	gopkg.in/yaml.v2 v2.2.8
 	gopkg.in/yaml.v3 v3.0.0-20191010095647-fc94e3f71652
 	k8s.io/apimachinery v0.18.3
+	sigs.k8s.io/yaml v1.2.0
 )
diff --git a/go.sum b/go.sum
index ae1d2b21..538a0c44 100644
--- a/go.sum
+++ b/go.sum
@@ -151,4 +151,5 @@ sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
diff --git a/pkg/helm/charts.go b/pkg/helm/charts.go
new file mode 100644
index 00000000..7a66c1e5
--- /dev/null
+++ b/pkg/helm/charts.go
@@ -0,0 +1,201 @@
+package helm
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/Masterminds/semver"
+	"sigs.k8s.io/yaml"
+)
+
+// LoadChartfile opens a Chartfile tree
+func LoadChartfile(projectRoot string) (*Charts, error) {
+	// make sure project root is valid
+	abs, err := filepath.Abs(projectRoot)
+	if err != nil {
+		return nil, err
+	}
+
+	// open chartfile
+	chartfile := filepath.Join(abs, Filename)
+	data, err := ioutil.ReadFile(chartfile)
+	if err != nil {
+		return nil, err
+	}
+
+	// parse it
+	c := Chartfile{
+		Version:   Version,
+		Directory: DefaultDir,
+	}
+	if err := yaml.UnmarshalStrict(data, &c); err != nil {
+		return nil, err
+	}
+
+	for i, r := range c.Requires {
+		if r.Chart == "" {
+			return nil, fmt.Errorf("requirements[%v]: 'chart' must be set", i)
+		}
+	}
+
+	// return Charts handle
+	charts := &Charts{
+		Manifest:    c,
+		projectRoot: abs,
+
+		// default to ExecHelm, but allow injecting from the outside
+		Helm: ExecHelm{},
+	}
+	return charts, nil
+}
+
+// Charts exposes the central Chartfile management functions
+type Charts struct {
+	// Manifest are the chartfile.yaml contents. It holds data about the developers intentions
+	Manifest Chartfile
+
+	// projectRoot is the enclosing directory of chartfile.yaml
+	projectRoot string
+
+	// Helm is the helm implementation underneath. ExecHelm is the default, but
+	// any implementation of the Helm interface may be used
+	Helm Helm
+}
+
+// ChartDir returns the directory pulled charts are saved in
+func (c Charts) ChartDir() string {
+	return filepath.Join(c.projectRoot, c.Manifest.Directory)
+}
+
+// ManifestFile returns the full path to the chartfile.yaml
+func (c Charts) ManifestFile() string {
+	return filepath.Join(c.projectRoot, Filename)
+}
+
+// Vendor pulls all Charts specified in the manifest into the local charts
+// directory. It fetches the repository index before doing so.
+func (c Charts) Vendor() error {
+	dir := c.ChartDir()
+	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
+		return err
+	}
+
+	log.Println("Syncing Repositories ...")
+	if err := c.Helm.RepoUpdate(Opts{Repositories: c.Manifest.Repositories}); err != nil {
+		return err
+	}
+
+	log.Println("Pulling Charts ...")
+	for _, r := range c.Manifest.Requires {
+		err := c.Helm.Pull(r.Chart, r.Version.String(), PullOpts{
+			Destination: dir,
+			Opts:        Opts{Repositories: c.Manifest.Repositories},
+		})
+		if err != nil {
+			return err
+		}
+
+		log.Printf(" %s@%s", r.Chart, r.Version.String())
+	}
+
+	return nil
+}
+
+// Add adds every Chart in reqs to the Manifest after validation, and runs
+// Vendor afterwards
+func (c Charts) Add(reqs []string) error {
+	log.Printf("Adding %v Charts ...", len(reqs))
+
+	skip := func(s string, err error) {
+		log.Printf(" Skipping %s: %s.", s, err)
+	}
+
+	// parse new charts, append in memory
+	added := 0
+	for _, s := range reqs {
+		r, err := parseReq(s)
+		if err != nil {
+			skip(s, err)
+			continue
+		}
+
+		if c.Manifest.Requires.Has(*r) {
+			skip(s, fmt.Errorf("already exists"))
+			continue
+		}
+
+		c.Manifest.Requires = append(c.Manifest.Requires, *r)
+		added++
+		log.Println(" OK:", s)
+	}
+
+	// write out
+	if err := write(c.Manifest, c.ManifestFile()); err != nil {
+		return err
+	}
+
+	// skipped some? fail then
+	if added != len(reqs) {
+		return fmt.Errorf("%v Charts were skipped. Please check above logs for details", len(reqs)-added)
+	}
+
+	// worked fine? vendor it
+	log.Printf("Added %v Charts to helmfile.yaml. Vendoring ...", added)
+	return c.Vendor()
+}
+
+func InitChartfile(path string) (*Charts, error) {
+	c := Chartfile{
+		Version: Version,
+		Repositories: []Repo{{
+			Name: "stable",
+			URL:  "https://kubernetes-charts.storage.googleapis.com",
+		}},
+		Requires: make(Requirements, 0),
+	}
+
+	if err := write(c, path); err != nil {
+		return nil, err
+	}
+
+	return LoadChartfile(filepath.Dir(path))
+}
+
+// write saves a Chartfile to dest
+func write(c Chartfile, dest string) error {
+	data, err := yaml.Marshal(c)
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(dest, data, 0644)
+}
+
+var chartExp = regexp.MustCompile(`\w+\/\w+@.+`)
+
+// parseReq parses a requirement from a string of the format `repo/name@version`
+func parseReq(s string) (*Requirement, error) {
+	if !chartExp.MatchString(s) {
+		return nil, fmt.Errorf("not of form 'repo/chart@version'")
+	}
+
+	elems := strings.Split(s, "@")
+	chart := elems[0]
+	ver, err := semver.NewVersion(elems[1])
+	if errors.Is(err, semver.ErrInvalidSemVer) {
+		return nil, fmt.Errorf("version is invalid")
+	} else if err != nil {
+		return nil, fmt.Errorf("version is invalid: %s", err)
+	}
+
+	return &Requirement{
+		Chart:   chart,
+		Version: *ver,
+	}, nil
+}
diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go
new file mode 100644
index 00000000..10354456
--- /dev/null
+++ b/pkg/helm/helm.go
@@ -0,0 +1,114 @@
+package helm
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+)
+
+// Helm provides high level access to some Helm operations
+type Helm interface {
+	// Pull downloads a Helm Chart from a remote
+	Pull(chart, version string, opts PullOpts) error
+
+	// RepoUpdate fetches the latest remote index
+	RepoUpdate(opts Opts) error
+}
+
+// PullOpts are additional, non-required options for Helm.Pull
+type PullOpts struct {
+	Opts
+
+	// Directory to put the resulting .tgz into
+	Destination string
+}
+
+// Opts are additional, non-required options that all Helm operations accept
+type Opts struct {
+	Repositories []Repo
+}
+
+// ExecHelm is a Helm implementation powered by the `helm` command line utility
+type ExecHelm struct{}
+
+// Pull implements Helm.Pull
+func (e ExecHelm) Pull(chart, version string, opts PullOpts) error {
+	repoFile, err := writeRepoTmpFile(opts.Repositories)
+	if err != nil {
+		return err
+	}
+	defer os.Remove(repoFile)
+
+	cmd := e.cmd("pull", chart,
+		"--version", version,
+		"--destination", opts.Destination,
+		"--repository-config", repoFile,
+	)
+
+	return cmd.Run()
+}
+
+// RepoUpdate implements Helm.RepoUpdate
+func (e ExecHelm) RepoUpdate(opts Opts) error {
+	repoFile, err := writeRepoTmpFile(opts.Repositories)
+	if err != nil {
+		return err
+	}
+	defer os.Remove(repoFile)
+
+	cmd := e.cmd("repo", "update",
+		"--repository-config", repoFile,
+	)
+	var errBuf bytes.Buffer
+	cmd.Stderr = &errBuf
+
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("%s\n%s", errBuf.String(), err)
+	}
+
+	return nil
+}
+
+// cmd returns a prepared exec.Cmd to use the `helm` binary
+func (e ExecHelm) cmd(action string, args ...string) *exec.Cmd {
+	argv := []string{action}
+	argv = append(argv, args...)
+
+	cmd := helmCmd(argv...)
+	cmd.Stderr = os.Stderr
+
+	return cmd
+}
+
+// helmCmd returns a bare exec.Cmd pointed at the local helm binary
+func helmCmd(args ...string) *exec.Cmd {
+	bin := "helm"
+	if env := os.Getenv("TANKA_HELM_PATH"); env != "" {
+		bin = env
+	}
+
+	return exec.Command(bin, args...)
+}
+
+// writeRepoTmpFile creates a temporary repositories.yaml from the passed Repo
+// slice to be used by the helm binary
+func writeRepoTmpFile(r []Repo) (string, error) {
+	m := map[string]interface{}{
+		"repositories": r,
+	}
+
+	f, err := ioutil.TempFile("", "charts-repos")
+	if err != nil {
+		return "", err
+	}
+
+	enc := json.NewEncoder(f)
+	if err := enc.Encode(m); err != nil {
+		return "", err
+	}
+
+	return f.Name(), nil
+}
diff --git a/pkg/helm/spec.go b/pkg/helm/spec.go
new file mode 100644
index 00000000..23b79923
--- /dev/null
+++ b/pkg/helm/spec.go
@@ -0,0 +1,65 @@
+package helm
+
+import (
+	"github.com/Masterminds/semver"
+)
+
+const (
+	// Version of the current Chartfile implementation
+	Version = 1
+
+	// Filename of the Chartfile
+	Filename = "chartfile.yaml"
+
+	// DefaultDir is the directory used for storing Charts if not specified
+	// otherwise
+	DefaultDir = "charts"
+)
+
+// Chartfile is the schema used to declaratively define locally required Helm
+// Charts
+type Chartfile struct {
+	// Version of the Chartfile schema (for future use)
+	Version uint `json:"version"`
+
+	// Repositories to source from
+	Repositories []Repo `json:"repositories"`
+
+	// Requires lists Charts expected to be present in the charts folder
+	Requires Requirements `json:"requires"`
+
+	// Folder to use for storing Charts. Defaults to 'charts'
+	Directory string `json:"directory,omitempty"`
+}
+
+// Repo describes a single Helm repository
+type Repo struct {
+	Name     string `json:"name,omitempty"`
+	URL      string `json:"url,omitempty"`
+	CAFile   string `json:"caFile,omitempty"`
+	CertFile string `json:"certFile,omitempty"`
+	KeyFile  string `json:"keyFile,omitempty"`
+	Username string `json:"username,omitempty"`
+	Password string `json:"password,omitempty"`
+}
+
+// Requirement describes a single required Helm Chart.
+// Both, Chart and Version are required
+type Requirement struct {
+	Chart   string         `json:"chart"`
+	Version semver.Version `json:"version"`
+}
+
+// Requirements is an aggregate of all required Charts
+type Requirements []Requirement
+
+// Has reports whether 'req' is already part of the requirements
+func (r Requirements) Has(req Requirement) bool {
+	for _, x := range r {
+		if x == req {
+			return true
+		}
+	}
+
+	return false
+}
diff --git a/pkg/helmraiser/helm.go b/pkg/helm/template.go
similarity index 87%
rename from pkg/helmraiser/helm.go
rename to pkg/helm/template.go
index 488b771c..2f817194 100644
--- a/pkg/helmraiser/helm.go
+++ b/pkg/helm/template.go
@@ -1,4 +1,4 @@
-package helmraiser
+package helm
 
 import (
 	"bytes"
@@ -7,7 +7,6 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"strings"
 
 	jsonnet "github.com/google/go-jsonnet"
@@ -17,24 +16,6 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-// Helm provides actions on Helm charts.
-type Helm struct{}
-
-func (h Helm) cmd(action string, args ...string) *exec.Cmd {
-	argv := []string{action}
-	argv = append(argv, args...)
-
-	return helmCmd(argv...)
-}
-
-func helmCmd(args ...string) *exec.Cmd {
-	binary := "helm"
-	if env := os.Getenv("TANKA_HELM_PATH"); env != "" {
-		binary = env
-	}
-	return exec.Command(binary, args...)
-}
-
 // TemplateOpts defines additional parameters that can be passed to the
 // Helm.Template action
 type TemplateOpts struct {
@@ -74,7 +55,7 @@ func confToArgs(conf TemplateOpts) ([]string, []string, error) {
 
 // Template expands a Helm Chart into a regular manifest.List using the `helm
 // template` command
-func (h Helm) Template(name, chart string, opts TemplateOpts) (manifest.List, error) {
+func (h ExecHelm) Template(name, chart string, opts TemplateOpts) (manifest.List, error) {
 	confArgs, tmpFiles, err := confToArgs(opts)
 	if err != nil {
 		return nil, err
@@ -139,7 +120,8 @@ func NativeFunc() *jsonnet.NativeFunction {
 				return "", err
 			}
 
-			var h Helm
+			// TODO: Define Template on the Helm interface instead
+			var h ExecHelm
 			list, err := h.Template(name, chart, conf)
 			if err != nil {
 				return nil, err
diff --git a/pkg/jsonnet/native/funcs.go b/pkg/jsonnet/native/funcs.go
index 8dbdb463..e5b250d8 100644
--- a/pkg/jsonnet/native/funcs.go
+++ b/pkg/jsonnet/native/funcs.go
@@ -9,7 +9,7 @@ import (
 
 	jsonnet "github.com/google/go-jsonnet"
 	"github.com/google/go-jsonnet/ast"
-	"github.com/grafana/tanka/pkg/helmraiser"
+	"github.com/grafana/tanka/pkg/helm"
 	"github.com/pkg/errors"
 	yaml "gopkg.in/yaml.v3"
 )
@@ -31,7 +31,7 @@ func Funcs() []*jsonnet.NativeFunction {
 		regexMatch(),
 		regexSubst(),
 
-		helmraiser.NativeFunc(),
+		helm.NativeFunc(),
 	}
 }
 


Test patch:
diff --git a/pkg/helmraiser/helm_test.go b/pkg/helm/template_test.go
similarity index 98%
rename from pkg/helmraiser/helm_test.go
rename to pkg/helm/template_test.go
index f316fc4b..056e69db 100644
--- a/pkg/helmraiser/helm_test.go
+++ b/pkg/helm/template_test.go
@@ -1,4 +1,4 @@
-package helmraiser
+package helm
 
 import (
 	"fmt"