Repository: grafana/tanka
Pull request #340 (2020-08-10 20:09:05)

feat(jsonnet): top level arguments
Exposes the TLA functionality of Jsonnet.

BREAKING: Changes the extVar flags to match those of the `jsonnet` CLI

/cc @sbarzowski, would be super cool if you could give the docs added by this PR a quick look if they are complete and correct


Patch:
diff --git a/cmd/tk/export.go b/cmd/tk/export.go
index 61d0e67e..d9324d83 100644
--- a/cmd/tk/export.go
+++ b/cmd/tk/export.go
@@ -33,7 +33,7 @@ func exportCmd() *cli.Command {
 	}
 
 	vars := workflowFlags(cmd.Flags())
-	getExtCode := extCodeParser(cmd.Flags())
+	getExtCode, getTLACode := cliCodeParser(cmd.Flags())
 	format := cmd.Flags().String("format", "{{.apiVersion}}.{{.kind}}-{{.metadata.name}}", "https://tanka.dev/exporting#filenames")
 	extension := cmd.Flags().String("extension", "yaml", "File extension")
 	merge := cmd.Flags().Bool("merge", false, "Allow merging with existing directory")
@@ -68,6 +68,7 @@ func exportCmd() *cli.Command {
 		// get the manifests
 		res, err := tanka.Show(args[0],
 			tanka.WithExtCode(getExtCode()),
+			tanka.WithTLACode(getTLACode()),
 			tanka.WithTargets(stringsToRegexps(vars.targets)),
 		)
 		if err != nil {
diff --git a/cmd/tk/jsonnet.go b/cmd/tk/jsonnet.go
index 01e39393..e42a0a25 100644
--- a/cmd/tk/jsonnet.go
+++ b/cmd/tk/jsonnet.go
@@ -6,9 +6,8 @@ import (
 	"log"
 	"strings"
 
-	"github.com/spf13/pflag"
-
 	"github.com/go-clix/cli"
+	"github.com/spf13/pflag"
 
 	"github.com/grafana/tanka/pkg/tanka"
 )
@@ -20,11 +19,12 @@ func evalCmd() *cli.Command {
 		Args:  workflowArgs,
 	}
 
-	getExtCode := extCodeParser(cmd.Flags())
+	getExtCode, getTLACode := cliCodeParser(cmd.Flags())
 
 	cmd.Run = func(cmd *cli.Command, args []string) error {
 		raw, err := tanka.Eval(args[0],
 			tanka.WithExtCode(getExtCode()),
+			tanka.WithTLACode(getTLACode()),
 		)
 
 		if err != nil {
@@ -43,29 +43,37 @@ func evalCmd() *cli.Command {
 	return cmd
 }
 
-func extCodeParser(fs *pflag.FlagSet) func() map[string]string {
+func cliCodeParser(fs *pflag.FlagSet) (func() map[string]string, func() map[string]string) {
 	// need to use StringArray instead of StringSlice, because pflag attempts to
 	// parse StringSlice using the csv parser, which breaks when passing objects
-	values := fs.StringArrayP("extCode", "e", nil, "Inject any Jsonnet from the outside (Format: key=<code>)")
-	strs := fs.StringArray("extVar", nil, "Inject a string from the outside (Format: key=value)")
+	extCode := fs.StringArray("ext-code", nil, "Set code value of extVar (Format: key=<code>)")
+	extStr := fs.StringArrayP("ext-str", "V", nil, "Set string value of extVar (Format: key=value)")
 
-	return func() map[string]string {
-		m := make(map[string]string)
-		for _, s := range *values {
-			split := strings.SplitN(s, "=", 2)
-			if len(split) != 2 {
-				log.Fatalf("extCode argument has wrong format: `%s`. Expected `key=<code>`", s)
+	tlaCode := fs.StringArray("tla-code", nil, "Set code value of top level function (Format: key=<code>)")
+	tlaStr := fs.StringArrayP("tla-str", "A", nil, "Set string value of top level function (Format: key=value)")
+
+	newParser := func(kind string, code, str *[]string) func() map[string]string {
+		return func() map[string]string {
+			m := make(map[string]string)
+			for _, s := range *code {
+				split := strings.SplitN(s, "=", 2)
+				if len(split) != 2 {
+					log.Fatalf(kind+"-code argument has wrong format: `%s`. Expected `key=<code>`", s)
+				}
+				m[split[0]] = split[1]
 			}
-			m[split[0]] = split[1]
-		}
 
-		for _, s := range *strs {
-			split := strings.SplitN(s, "=", 2)
-			if len(split) != 2 {
-				log.Fatalf("extCode argument has wrong format: `%s`. Expected `key=<value>`", s)
+			for _, s := range *str {
+				split := strings.SplitN(s, "=", 2)
+				if len(split) != 2 {
+					log.Fatalf(kind+"-str argument has wrong format: `%s`. Expected `key=<value>`", s)
+				}
+				m[split[0]] = fmt.Sprintf(`"%s"`, split[1])
 			}
-			m[split[0]] = fmt.Sprintf(`"%s"`, split[1])
+			return m
 		}
-		return m
 	}
+
+	return newParser("ext", extCode, extStr),
+		newParser("tla", tlaCode, tlaStr)
 }
diff --git a/cmd/tk/workflow.go b/cmd/tk/workflow.go
index 4c8f7a78..352a1b03 100644
--- a/cmd/tk/workflow.go
+++ b/cmd/tk/workflow.go
@@ -44,12 +44,14 @@ func applyCmd() *cli.Command {
 	force := cmd.Flags().Bool("force", false, "force applying (kubectl apply --force)")
 	validate := cmd.Flags().Bool("validate", true, "validation of resources (kubectl --validate=false)")
 	autoApprove := cmd.Flags().Bool("dangerous-auto-approve", false, "skip interactive approval. Only for automation!")
-	getExtCode := extCodeParser(cmd.Flags())
+
+	getExtCode, getTLACode := cliCodeParser(cmd.Flags())
 
 	cmd.Run = func(cmd *cli.Command, args []string) error {
 		err := tanka.Apply(args[0],
 			tanka.WithTargets(stringsToRegexps(vars.targets)),
 			tanka.WithExtCode(getExtCode()),
+			tanka.WithTLACode(getTLACode()),
 			tanka.WithApplyForce(*force),
 			tanka.WithApplyValidate(*validate),
 			tanka.WithApplyAutoApprove(*autoApprove),
@@ -69,13 +71,14 @@ func pruneCmd() *cli.Command {
 		Args:  workflowArgs,
 	}
 
-	getExtCode := extCodeParser(cmd.Flags())
+	getExtCode, getTLACode := cliCodeParser(cmd.Flags())
 	autoApprove := cmd.Flags().Bool("dangerous-auto-approve", false, "skip interactive approval. Only for automation!")
 	force := cmd.Flags().Bool("force", false, "force deleting (kubectl delete --force)")
 
 	cmd.Run = func(cmd *cli.Command, args []string) error {
 		return tanka.Prune(args[0],
 			tanka.WithExtCode(getExtCode()),
+			tanka.WithTLACode(getTLACode()),
 			tanka.WithApplyAutoApprove(*autoApprove),
 			tanka.WithApplyForce(*force),
 		)
@@ -95,12 +98,13 @@ func deleteCmd() *cli.Command {
 	force := cmd.Flags().Bool("force", false, "force deleting (kubectl delete --force)")
 	validate := cmd.Flags().Bool("validate", true, "validation of resources (kubectl --validate=false)")
 	autoApprove := cmd.Flags().Bool("dangerous-auto-approve", false, "skip interactive approval. Only for automation!")
-	getExtCode := extCodeParser(cmd.Flags())
+	getExtCode, getTLACode := cliCodeParser(cmd.Flags())
 
 	cmd.Run = func(cmd *cli.Command, args []string) error {
 		err := tanka.Delete(args[0],
 			tanka.WithTargets(stringsToRegexps(vars.targets)),
 			tanka.WithExtCode(getExtCode()),
+			tanka.WithTLACode(getTLACode()),
 			tanka.WithApplyForce(*force),
 			tanka.WithApplyValidate(*validate),
 			tanka.WithApplyAutoApprove(*autoApprove),
@@ -130,12 +134,13 @@ func diffCmd() *cli.Command {
 		summarize    = cmd.Flags().BoolP("summarize", "s", false, "quick summary of the differences, hides file contents")
 	)
 
-	getExtCode := extCodeParser(cmd.Flags())
+	getExtCode, getTLACode := cliCodeParser(cmd.Flags())
 
 	cmd.Run = func(cmd *cli.Command, args []string) error {
 		changes, err := tanka.Diff(args[0],
 			tanka.WithTargets(stringsToRegexps(vars.targets)),
 			tanka.WithExtCode(getExtCode()),
+			tanka.WithTLACode(getTLACode()),
 			tanka.WithDiffStrategy(*diffStrategy),
 			tanka.WithDiffSummarize(*summarize),
 		)
@@ -170,7 +175,7 @@ func showCmd() *cli.Command {
 	}
 	vars := workflowFlags(cmd.Flags())
 	allowRedirect := cmd.Flags().Bool("dangerous-allow-redirect", false, "allow redirecting output to a file or a pipe.")
-	getExtCode := extCodeParser(cmd.Flags())
+	getExtCode, getTLACode := cliCodeParser(cmd.Flags())
 	cmd.Run = func(cmd *cli.Command, args []string) error {
 		if !interactive && !*allowRedirect {
 			fmt.Fprintln(os.Stderr, `Redirection of the output of tk show is discouraged and disabled by default.
@@ -181,6 +186,7 @@ Otherwise run tk show --dangerous-allow-redirect to bypass this check.`)
 
 		pretty, err := tanka.Show(args[0],
 			tanka.WithExtCode(getExtCode()),
+			tanka.WithTLACode(getTLACode()),
 			tanka.WithTargets(stringsToRegexps(vars.targets)),
 		)
 		if err != nil {
diff --git a/docs/docs/jsonnet/injecting-values.md b/docs/docs/jsonnet/injecting-values.md
new file mode 100644
index 00000000..7d296cbe
--- /dev/null
+++ b/docs/docs/jsonnet/injecting-values.md
@@ -0,0 +1,94 @@
+---
+name: "Injecting Values"
+route: "/jsonnet/injecting-values"
+menu: "Writing Jsonnet"
+---
+
+# Injecting Values
+
+Sometimes it might be required to pass externally acquired data into Jsonnet.
+
+There are three ways of doing so:
+
+1. [JSON files](#json-files)
+2. [External variables](#external-variables)
+3. [Top level arguments](#top-level-arguments)
+
+Also check out the [official Jsonnet docs on this
+topic](https://jsonnet.org/ref/language.html#passing-data-to-jsonnet).
+
+## JSON files
+
+Jsonnet is a superset of JSON, it treats any JSON as valid Jsonnet. Because many
+systems can be told to output their data in JSON format, this provides a pretty
+good interface between those.
+
+For example, your build tooling like `make` could acquire secrets from systems such as
+[Vault](https://www.vaultproject.io/), etc. and write that into `secrets.json`.
+
+```jsonnet
+local secrets = import "secrets.json";
+
+{
+  foo: secrets.myPassword,
+}
+```
+
+> **Note**: Using `import` with JSON treats it as Jsonnet, so make sure to not
+> use it with untrusted code.  
+> A safer, but more verbose, alternative is `std.parseJson(importstr 'path_to_json.json')`
+
+## External variables
+
+Another way of passing values from the outside are external variables, which are specified like so:
+
+```bash
+# strings
+$ tk show . --ext-str hello=world
+
+# any Jsonnet snippet
+$ tk show . --ext-code foo=4 --ext-code bar='[ 1, 3 ]'
+```
+
+They can be accessed using `std.extVar` and the name given to them on the command line:
+
+```jsonnet
+{
+  foo: std.extVar('foo'), // 4, integer
+  bar: std.extVar('bar'), // [ 1, 3 ], array
+}
+```
+
+> **Warning**: External variables are directly accessible in all parts of the
+> configuration, which can make it difficult to track where they are used and
+> what effect they have on the final result.
+> Try to use [Top Level Arguments](#top-level-arguments) instead.
+
+## Top Level Arguments
+
+Usually with Tanka, your `main.jsonnet` holds an object at the top level (most
+outer type in the generated JSON):
+
+```jsonnet
+// main.jsonnet
+{
+  /* your resources */
+}
+```
+
+Another type of Jsonnet that naturally accepts parameters is the `function`.
+When the Jsonnet compiler finds a function at the top level, it invokes it and
+allows passing parameter values from the command line:
+
+```jsonnet
+// Actual output (object) returned by function, which is taking parameters and default values
+function(who, msg="Hello %s!") {
+  hello: msg % who
+}
+```
+
+Here, `who` needs a value while `msg` has a default. This can be invoked like so:
+
+```bash
+$ tk show . --tla-str who=John
+```
diff --git a/docs/docs/targets.md b/docs/docs/targets.md
index 6b8edb25..e8fdb270 100644
--- a/docs/docs/targets.md
+++ b/docs/docs/targets.md
@@ -78,3 +78,14 @@ apiVersion: apps/v1
 kind: Deployment
 # ...
 ```
+
+## Excluding
+
+Sometimes it may be desirably to exclude a single object, instead of including all others.
+
+To do so, prepend the regular expression with an exclamation mark (`!`), like so:
+
+```bash
+# filter out all Deployments
+$ tk show . -t '!deployment/.*'
+```
diff --git a/pkg/jsonnet/eval.go b/pkg/jsonnet/eval.go
index 34cf00e1..b2ae10fc 100644
--- a/pkg/jsonnet/eval.go
+++ b/pkg/jsonnet/eval.go
@@ -55,3 +55,11 @@ func WithExtCode(key, code string) Modifier {
 		return nil
 	}
 }
+
+// WithTLA allows to set the given code as a top level argument
+func WithTLA(key, code string) Modifier {
+	return func(vm *jsonnet.VM) error {
+		vm.TLACode(key, code)
+		return nil
+	}
+}
diff --git a/pkg/process/filter.go b/pkg/process/filter.go
index 8e8d7e79..6b2b8c5d 100644
--- a/pkg/process/filter.go
+++ b/pkg/process/filter.go
@@ -9,12 +9,16 @@ import (
 )
 
 // Filter returns all elements of the list that match at least one expression
+// and are not ignored
 func Filter(list manifest.List, exprs Matchers) manifest.List {
 	out := make(manifest.List, 0, len(list))
 	for _, m := range list {
 		if !exprs.MatchString(m.KindName()) {
 			continue
 		}
+		if exprs.IgnoreString(m.KindName()) {
+			continue
+		}
 		out = append(out, m)
 	}
 	return out
@@ -26,7 +30,13 @@ type Matcher interface {
 	MatchString(string) bool
 }
 
+// Ignorer is like matcher, but for explicitely ignoring resources
+type Ignorer interface {
+	IgnoreString(string) bool
+}
+
 // Matchers is a collection of multiple expressions.
+// A matcher may also implement Ignorer to explicitely ignore fields
 type Matchers []Matcher
 
 // MatchString returns whether at least one expression (OR) matches the string
@@ -38,6 +48,18 @@ func (e Matchers) MatchString(s string) bool {
 	return b
 }
 
+func (e Matchers) IgnoreString(s string) bool {
+	b := false
+	for _, exp := range e {
+		i, ok := exp.(Ignorer)
+		if !ok {
+			continue
+		}
+		b = b || i.IgnoreString(s)
+	}
+	return b
+}
+
 // RegExps is a helper to construct Matchers from regular expressions
 func RegExps(rs []*regexp.Regexp) Matchers {
 	xprs := make(Matchers, 0, len(rs))
@@ -50,11 +72,20 @@ func RegExps(rs []*regexp.Regexp) Matchers {
 func StrExps(strs ...string) (Matchers, error) {
 	exps := make(Matchers, 0, len(strs))
 	for _, raw := range strs {
-		s := fmt.Sprintf(`(?i)^%s$`, raw)
+		// trim exlamation mark, not supported by regex
+		s := fmt.Sprintf(`(?i)^%s$`, strings.TrimPrefix(raw, "!"))
+
+		// create regexp matcher
+		var exp Matcher
 		exp, err := regexp.Compile(s)
 		if err != nil {
 			return nil, ErrBadExpr{err}
 		}
+
+		// if negative (!), invert regex behaviour
+		if strings.HasPrefix(raw, "!") {
+			exp = NegMatcher{exp: exp}
+		}
 		exps = append(exps, exp)
 	}
 	return exps, nil
@@ -76,3 +107,16 @@ type ErrBadExpr struct {
 func (e ErrBadExpr) Error() string {
 	return fmt.Sprintf("%s.\nSee https://tanka.dev/output-filtering/#regular-expressions for details on regular expressions.", strings.Title(e.inner.Error()))
 }
+
+// NexMatcher is a matcher that inverts the original behaviour
+type NegMatcher struct {
+	exp Matcher
+}
+
+func (n NegMatcher) MatchString(s string) bool {
+	return true
+}
+
+func (n NegMatcher) IgnoreString(s string) bool {
+	return n.exp.MatchString(s)
+}
diff --git a/pkg/tanka/tanka.go b/pkg/tanka/tanka.go
index fe4f58fa..bc58e93e 100644
--- a/pkg/tanka/tanka.go
+++ b/pkg/tanka/tanka.go
@@ -21,6 +21,7 @@ func parseModifiers(mods []Modifier) *options {
 type options struct {
 	// `std.extVar`
 	extCode map[string]string
+	tlaCode map[string]string
 
 	// target regular expressions to limit the working set
 	targets process.Matchers
@@ -44,6 +45,12 @@ func WithExtCode(code map[string]string) Modifier {
 	}
 }
 
+func WithTLACode(code map[string]string) Modifier {
+	return func(opts *options) {
+		opts.extCode = code
+	}
+}
+
 // WithTargets allows to submit regular expressions to limit the working set of
 // objects (https://tanka.dev/output-filtering/).
 func WithTargets(t process.Matchers) Modifier {


Test patch:
diff --git a/pkg/process/process_test.go b/pkg/process/process_test.go
index 5014b380..cf046159 100644
--- a/pkg/process/process_test.go
+++ b/pkg/process/process_test.go
@@ -64,6 +64,15 @@ func TestProcess(t *testing.T) {
 				`DePlOyMeNt/GrAfAnA`,
 			),
 		},
+		{
+			name: "targets-negative",
+			deep: testDataDeep().Deep,
+			flat: manifest.List{
+				testDataDeep().Flat[".app.web.frontend.nodejs.express.service"],
+				testDataDeep().Flat[".app.namespace"],
+			},
+			targets: MustStrExps(`!deployment/.*`),
+		},
 		{
 			name: "unwrap-list",
 			deep: loadFixture("list").Deep,