Repository: grafana/tanka (go)

Issue:
After Upgrade to v0.11.0 tk apply shows diff on Ingress annotations
Hi,

v0.11.0 shows a diff in annotation of Ingress objects, even after I've deleted the ingress and recreate it with v0.11.0.
Going back to v0.10.0 solves the problem immediately.

Does anybody else see this kind of problem?

In fact tanka wants to remove ` kubectl.kubernetes.io/last-applied-configuration:` from the Ingress objects.

Best regards and thanks
Andreas

Pull request description:
fix(k8s): consistent types in manifest.Manifest
The manifest.Manifest is a `map[string]interface{}`, which sticks to what `json.Unmarshal` returns, meaning it only includes concrete values, `[]interface{}` and `map[string]interface{}`.

In a v0.10 change I accidentally broke that, without realizing it: https://github.com/grafana/tanka/blob/a6d5fde5aac69dc4d66bb57d5a501e52ba60fada/pkg/kubernetes/manifest/manifest.go#L248

Up to #312, this didn't create any issues. Now however, because we called `Annotations` on all objects, types weren't like they should have been.

This caused subsetdiff to miss annotations, not computing the subset on them at all.

Fixes #318

Patch:
diff --git a/cmd/tk/export.go b/cmd/tk/export.go
index 61d0e67e..d9324d83 100644
--- a/cmd/tk/export.go
+++ b/cmd/tk/export.go
@@ -33,7 +33,7 @@ func exportCmd() *cli.Command {
 	}
 
 	vars := workflowFlags(cmd.Flags())
-	getExtCode := extCodeParser(cmd.Flags())
+	getExtCode, getTLACode := cliCodeParser(cmd.Flags())
 	format := cmd.Flags().String("format", "{{.apiVersion}}.{{.kind}}-{{.metadata.name}}", "https://tanka.dev/exporting#filenames")
 	extension := cmd.Flags().String("extension", "yaml", "File extension")
 	merge := cmd.Flags().Bool("merge", false, "Allow merging with existing directory")
@@ -68,6 +68,7 @@ func exportCmd() *cli.Command {
 		// get the manifests
 		res, err := tanka.Show(args[0],
 			tanka.WithExtCode(getExtCode()),
+			tanka.WithTLACode(getTLACode()),
 			tanka.WithTargets(stringsToRegexps(vars.targets)),
 		)
 		if err != nil {
diff --git a/cmd/tk/jsonnet.go b/cmd/tk/jsonnet.go
index 01e39393..e42a0a25 100644
--- a/cmd/tk/jsonnet.go
+++ b/cmd/tk/jsonnet.go
@@ -6,9 +6,8 @@ import (
 	"log"
 	"strings"
 
-	"github.com/spf13/pflag"
-
 	"github.com/go-clix/cli"
+	"github.com/spf13/pflag"
 
 	"github.com/grafana/tanka/pkg/tanka"
 )
@@ -20,11 +19,12 @@ func evalCmd() *cli.Command {
 		Args:  workflowArgs,
 	}
 
-	getExtCode := extCodeParser(cmd.Flags())
+	getExtCode, getTLACode := cliCodeParser(cmd.Flags())
 
 	cmd.Run = func(cmd *cli.Command, args []string) error {
 		raw, err := tanka.Eval(args[0],
 			tanka.WithExtCode(getExtCode()),
+			tanka.WithTLACode(getTLACode()),
 		)
 
 		if err != nil {
@@ -43,29 +43,37 @@ func evalCmd() *cli.Command {
 	return cmd
 }
 
-func extCodeParser(fs *pflag.FlagSet) func() map[string]string {
+func cliCodeParser(fs *pflag.FlagSet) (func() map[string]string, func() map[string]string) {
 	// need to use StringArray instead of StringSlice, because pflag attempts to
 	// parse StringSlice using the csv parser, which breaks when passing objects
-	values := fs.StringArrayP("extCode", "e", nil, "Inject any Jsonnet from the outside (Format: key=<code>)")
-	strs := fs.StringArray("extVar", nil, "Inject a string from the outside (Format: key=value)")
+	extCode := fs.StringArray("ext-code", nil, "Set code value of extVar (Format: key=<code>)")
+	extStr := fs.StringArrayP("ext-str", "V", nil, "Set string value of extVar (Format: key=value)")
 
-	return func() map[string]string {
-		m := make(map[string]string)
-		for _, s := range *values {
-			split := strings.SplitN(s, "=", 2)
-			if len(split) != 2 {
-				log.Fatalf("extCode argument has wrong format: `%s`. Expected `key=<code>`", s)
+	tlaCode := fs.StringArray("tla-code", nil, "Set code value of top level function (Format: key=<code>)")
+	tlaStr := fs.StringArrayP("tla-str", "A", nil, "Set string value of top level function (Format: key=value)")
+
+	newParser := func(kind string, code, str *[]string) func() map[string]string {
+		return func() map[string]string {
+			m := make(map[string]string)
+			for _, s := range *code {
+				split := strings.SplitN(s, "=", 2)
+				if len(split) != 2 {
+					log.Fatalf(kind+"-code argument has wrong format: `%s`. Expected `key=<code>`", s)
+				}
+				m[split[0]] = split[1]
 			}
-			m[split[0]] = split[1]
-		}
 
-		for _, s := range *strs {
-			split := strings.SplitN(s, "=", 2)
-			if len(split) != 2 {
-				log.Fatalf("extCode argument has wrong format: `%s`. Expected `key=<value>`", s)
+			for _, s := range *str {
+				split := strings.SplitN(s, "=", 2)
+				if len(split) != 2 {
+					log.Fatalf(kind+"-str argument has wrong format: `%s`. Expected `key=<value>`", s)
+				}
+				m[split[0]] = fmt.Sprintf(`"%s"`, split[1])
 			}
-			m[split[0]] = fmt.Sprintf(`"%s"`, split[1])
+			return m
 		}
-		return m
 	}
+
+	return newParser("ext", extCode, extStr),
+		newParser("tla", tlaCode, tlaStr)
 }
diff --git a/cmd/tk/workflow.go b/cmd/tk/workflow.go
index 4c8f7a78..352a1b03 100644
--- a/cmd/tk/workflow.go
+++ b/cmd/tk/workflow.go
@@ -44,12 +44,14 @@ func applyCmd() *cli.Command {
 	force := cmd.Flags().Bool("force", false, "force applying (kubectl apply --force)")
 	validate := cmd.Flags().Bool("validate", true, "validation of resources (kubectl --validate=false)")
 	autoApprove := cmd.Flags().Bool("dangerous-auto-approve", false, "skip interactive approval. Only for automation!")
-	getExtCode := extCodeParser(cmd.Flags())
+
+	getExtCode, getTLACode := cliCodeParser(cmd.Flags())
 
 	cmd.Run = func(cmd *cli.Command, args []string) error {
 		err := tanka.Apply(args[0],
 			tanka.WithTargets(stringsToRegexps(vars.targets)),
 			tanka.WithExtCode(getExtCode()),
+			tanka.WithTLACode(getTLACode()),
 			tanka.WithApplyForce(*force),
 			tanka.WithApplyValidate(*validate),
 			tanka.WithApplyAutoApprove(*autoApprove),
@@ -69,13 +71,14 @@ func pruneCmd() *cli.Command {
 		Args:  workflowArgs,
 	}
 
-	getExtCode := extCodeParser(cmd.Flags())
+	getExtCode, getTLACode := cliCodeParser(cmd.Flags())
 	autoApprove := cmd.Flags().Bool("dangerous-auto-approve", false, "skip interactive approval. Only for automation!")
 	force := cmd.Flags().Bool("force", false, "force deleting (kubectl delete --force)")
 
 	cmd.Run = func(cmd *cli.Command, args []string) error {
 		return tanka.Prune(args[0],
 			tanka.WithExtCode(getExtCode()),
+			tanka.WithTLACode(getTLACode()),
 			tanka.WithApplyAutoApprove(*autoApprove),
 			tanka.WithApplyForce(*force),
 		)
@@ -95,12 +98,13 @@ func deleteCmd() *cli.Command {
 	force := cmd.Flags().Bool("force", false, "force deleting (kubectl delete --force)")
 	validate := cmd.Flags().Bool("validate", true, "validation of resources (kubectl --validate=false)")
 	autoApprove := cmd.Flags().Bool("dangerous-auto-approve", false, "skip interactive approval. Only for automation!")
-	getExtCode := extCodeParser(cmd.Flags())
+	getExtCode, getTLACode := cliCodeParser(cmd.Flags())
 
 	cmd.Run = func(cmd *cli.Command, args []string) error {
 		err := tanka.Delete(args[0],
 			tanka.WithTargets(stringsToRegexps(vars.targets)),
 			tanka.WithExtCode(getExtCode()),
+			tanka.WithTLACode(getTLACode()),
 			tanka.WithApplyForce(*force),
 			tanka.WithApplyValidate(*validate),
 			tanka.WithApplyAutoApprove(*autoApprove),
@@ -130,12 +134,13 @@ func diffCmd() *cli.Command {
 		summarize    = cmd.Flags().BoolP("summarize", "s", false, "quick summary of the differences, hides file contents")
 	)
 
-	getExtCode := extCodeParser(cmd.Flags())
+	getExtCode, getTLACode := cliCodeParser(cmd.Flags())
 
 	cmd.Run = func(cmd *cli.Command, args []string) error {
 		changes, err := tanka.Diff(args[0],
 			tanka.WithTargets(stringsToRegexps(vars.targets)),
 			tanka.WithExtCode(getExtCode()),
+			tanka.WithTLACode(getTLACode()),
 			tanka.WithDiffStrategy(*diffStrategy),
 			tanka.WithDiffSummarize(*summarize),
 		)
@@ -170,7 +175,7 @@ func showCmd() *cli.Command {
 	}
 	vars := workflowFlags(cmd.Flags())
 	allowRedirect := cmd.Flags().Bool("dangerous-allow-redirect", false, "allow redirecting output to a file or a pipe.")
-	getExtCode := extCodeParser(cmd.Flags())
+	getExtCode, getTLACode := cliCodeParser(cmd.Flags())
 	cmd.Run = func(cmd *cli.Command, args []string) error {
 		if !interactive && !*allowRedirect {
 			fmt.Fprintln(os.Stderr, `Redirection of the output of tk show is discouraged and disabled by default.
@@ -181,6 +186,7 @@ Otherwise run tk show --dangerous-allow-redirect to bypass this check.`)
 
 		pretty, err := tanka.Show(args[0],
 			tanka.WithExtCode(getExtCode()),
+			tanka.WithTLACode(getTLACode()),
 			tanka.WithTargets(stringsToRegexps(vars.targets)),
 		)
 		if err != nil {
diff --git a/docs/docs/jsonnet/injecting-values.md b/docs/docs/jsonnet/injecting-values.md
new file mode 100644
index 00000000..7d296cbe
--- /dev/null
+++ b/docs/docs/jsonnet/injecting-values.md
@@ -0,0 +1,94 @@
+---
+name: "Injecting Values"
+route: "/jsonnet/injecting-values"
+menu: "Writing Jsonnet"
+---
+
+# Injecting Values
+
+Sometimes it might be required to pass externally acquired data into Jsonnet.
+
+There are three ways of doing so:
+
+1. [JSON files](#json-files)
+2. [External variables](#external-variables)
+3. [Top level arguments](#top-level-arguments)
+
+Also check out the [official Jsonnet docs on this
+topic](https://jsonnet.org/ref/language.html#passing-data-to-jsonnet).
+
+## JSON files
+
+Jsonnet is a superset of JSON, it treats any JSON as valid Jsonnet. Because many
+systems can be told to output their data in JSON format, this provides a pretty
+good interface between those.
+
+For example, your build tooling like `make` could acquire secrets from systems such as
+[Vault](https://www.vaultproject.io/), etc. and write that into `secrets.json`.
+
+```jsonnet
+local secrets = import "secrets.json";
+
+{
+  foo: secrets.myPassword,
+}
+```
+
+> **Note**: Using `import` with JSON treats it as Jsonnet, so make sure to not
+> use it with untrusted code.  
+> A safer, but more verbose, alternative is `std.parseJson(importstr 'path_to_json.json')`
+
+## External variables
+
+Another way of passing values from the outside are external variables, which are specified like so:
+
+```bash
+# strings
+$ tk show . --ext-str hello=world
+
+# any Jsonnet snippet
+$ tk show . --ext-code foo=4 --ext-code bar='[ 1, 3 ]'
+```
+
+They can be accessed using `std.extVar` and the name given to them on the command line:
+
+```jsonnet
+{
+  foo: std.extVar('foo'), // 4, integer
+  bar: std.extVar('bar'), // [ 1, 3 ], array
+}
+```
+
+> **Warning**: External variables are directly accessible in all parts of the
+> configuration, which can make it difficult to track where they are used and
+> what effect they have on the final result.
+> Try to use [Top Level Arguments](#top-level-arguments) instead.
+
+## Top Level Arguments
+
+Usually with Tanka, your `main.jsonnet` holds an object at the top level (most
+outer type in the generated JSON):
+
+```jsonnet
+// main.jsonnet
+{
+  /* your resources */
+}
+```
+
+Another type of Jsonnet that naturally accepts parameters is the `function`.
+When the Jsonnet compiler finds a function at the top level, it invokes it and
+allows passing parameter values from the command line:
+
+```jsonnet
+// Actual output (object) returned by function, which is taking parameters and default values
+function(who, msg="Hello %s!") {
+  hello: msg % who
+}
+```
+
+Here, `who` needs a value while `msg` has a default. This can be invoked like so:
+
+```bash
+$ tk show . --tla-str who=John
+```
diff --git a/docs/docs/targets.md b/docs/docs/targets.md
index 6b8edb25..e8fdb270 100644
--- a/docs/docs/targets.md
+++ b/docs/docs/targets.md
@@ -78,3 +78,14 @@ apiVersion: apps/v1
 kind: Deployment
 # ...
 ```
+
+## Excluding
+
+Sometimes it may be desirably to exclude a single object, instead of including all others.
+
+To do so, prepend the regular expression with an exclamation mark (`!`), like so:
+
+```bash
+# filter out all Deployments
+$ tk show . -t '!deployment/.*'
+```
diff --git a/pkg/jsonnet/eval.go b/pkg/jsonnet/eval.go
index 34cf00e1..b2ae10fc 100644
--- a/pkg/jsonnet/eval.go
+++ b/pkg/jsonnet/eval.go
@@ -55,3 +55,11 @@ func WithExtCode(key, code string) Modifier {
 		return nil
 	}
 }
+
+// WithTLA allows to set the given code as a top level argument
+func WithTLA(key, code string) Modifier {
+	return func(vm *jsonnet.VM) error {
+		vm.TLACode(key, code)
+		return nil
+	}
+}
diff --git a/pkg/kubernetes/manifest/manifest.go b/pkg/kubernetes/manifest/manifest.go
index f2c1695f..688cb774 100644
--- a/pkg/kubernetes/manifest/manifest.go
+++ b/pkg/kubernetes/manifest/manifest.go
@@ -219,37 +219,22 @@ func (m Metadata) UID() string {
 }
 
 // Labels of the manifest
-func (m Metadata) Labels() map[string]string {
-	return safeStringMap(m, "labels")
+func (m Metadata) Labels() map[string]interface{} {
+	return safeMSI(m, "labels")
 }
 
 // Annotations of the manifest
-func (m Metadata) Annotations() map[string]string {
-	return safeStringMap(m, "annotations")
+func (m Metadata) Annotations() map[string]interface{} {
+	return safeMSI(m, "annotations")
 }
 
-// safeStringMap safely returns a string map:
-// - returns if map[string]string
-// - converts if map[string]interface{}
-// - zeroes if anything else
-func safeStringMap(m map[string]interface{}, key string) map[string]string {
+func safeMSI(m map[string]interface{}, key string) map[string]interface{} {
 	switch t := m[key].(type) {
-	case map[string]string:
-		return t
 	case map[string]interface{}:
-		mss := make(map[string]string)
-		for k, v := range t {
-			s, ok := v.(string)
-			if !ok {
-				continue
-			}
-			mss[k] = s
-		}
-		m[key] = mss
-		return m[key].(map[string]string)
+		return t
 	default:
-		m[key] = make(map[string]string)
-		return m[key].(map[string]string)
+		m[key] = make(map[string]interface{})
+		return m[key].(map[string]interface{})
 	}
 }
 
diff --git a/pkg/kubernetes/subsetdiff.go b/pkg/kubernetes/subsetdiff.go
index f3af4b2a..ce87dc45 100644
--- a/pkg/kubernetes/subsetdiff.go
+++ b/pkg/kubernetes/subsetdiff.go
@@ -4,7 +4,6 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
-	yaml "gopkg.in/yaml.v3"
 
 	"github.com/grafana/tanka/pkg/kubernetes/client"
 	"github.com/grafana/tanka/pkg/kubernetes/manifest"
@@ -93,60 +92,55 @@ func subsetDiff(c client.Client, m manifest.Manifest) (*difference, error) {
 		return nil, errors.Wrap(err, "getting state from cluster")
 	}
 
-	should, err := yaml.Marshal(m)
-	if err != nil {
-		return nil, err
-	}
+	should := m.String()
 
-	is, err := yaml.Marshal(subset(m, rawIs))
-	if err != nil {
-		return nil, err
-	}
-	if string(is) == "{}\n" {
-		is = []byte("")
+	sub := subset(m, rawIs)
+	is := manifest.Manifest(sub).String()
+	if is == "{}\n" {
+		is = ""
 	}
 
 	return &difference{
 		name:   name,
-		live:   string(is),
-		merged: string(should),
+		live:   is,
+		merged: should,
 	}, nil
 }
 
-// subset removes all keys from is, that are not present in should.
-// It makes is a subset of should.
+// subset removes all keys from big, that are not present in small.
+// It makes big a subset of small.
 // Kubernetes returns more keys than we can know about.
 // This means, we need to remove all keys from the kubectl output, that are not present locally.
-func subset(should, is map[string]interface{}) map[string]interface{} {
-	if should["namespace"] != nil {
-		is["namespace"] = should["namespace"]
+func subset(small, big map[string]interface{}) map[string]interface{} {
+	if small["namespace"] != nil {
+		big["namespace"] = small["namespace"]
 	}
 
 	// just ignore the apiVersion for now, too much bloat
-	if should["apiVersion"] != nil && is["apiVersion"] != nil {
-		is["apiVersion"] = should["apiVersion"]
+	if small["apiVersion"] != nil && big["apiVersion"] != nil {
+		big["apiVersion"] = small["apiVersion"]
 	}
 
-	for k, v := range is {
-		if should[k] == nil {
-			delete(is, k)
+	for k, v := range big {
+		if _, ok := small[k]; !ok {
+			delete(big, k)
 			continue
 		}
 
 		switch b := v.(type) {
 		case map[string]interface{}:
-			if a, ok := should[k].(map[string]interface{}); ok {
-				is[k] = subset(a, b)
+			if a, ok := small[k].(map[string]interface{}); ok {
+				big[k] = subset(a, b)
 			}
 		case []map[string]interface{}:
 			for i := range b {
-				if a, ok := should[k].([]map[string]interface{}); ok {
+				if a, ok := small[k].([]map[string]interface{}); ok {
 					b[i] = subset(a[i], b[i])
 				}
 			}
 		case []interface{}:
 			for i := range b {
-				if a, ok := should[k].([]interface{}); ok {
+				if a, ok := small[k].([]interface{}); ok {
 					if i >= len(a) {
 						// slice in config shorter than in live. Abort, as there are no entries to diff anymore
 						break
@@ -168,5 +162,5 @@ func subset(should, is map[string]interface{}) map[string]interface{} {
 			}
 		}
 	}
-	return is
+	return big
 }
diff --git a/pkg/process/filter.go b/pkg/process/filter.go
index 8e8d7e79..6b2b8c5d 100644
--- a/pkg/process/filter.go
+++ b/pkg/process/filter.go
@@ -9,12 +9,16 @@ import (
 )
 
 // Filter returns all elements of the list that match at least one expression
+// and are not ignored
 func Filter(list manifest.List, exprs Matchers) manifest.List {
 	out := make(manifest.List, 0, len(list))
 	for _, m := range list {
 		if !exprs.MatchString(m.KindName()) {
 			continue
 		}
+		if exprs.IgnoreString(m.KindName()) {
+			continue
+		}
 		out = append(out, m)
 	}
 	return out
@@ -26,7 +30,13 @@ type Matcher interface {
 	MatchString(string) bool
 }
 
+// Ignorer is like matcher, but for explicitely ignoring resources
+type Ignorer interface {
+	IgnoreString(string) bool
+}
+
 // Matchers is a collection of multiple expressions.
+// A matcher may also implement Ignorer to explicitely ignore fields
 type Matchers []Matcher
 
 // MatchString returns whether at least one expression (OR) matches the string
@@ -38,6 +48,18 @@ func (e Matchers) MatchString(s string) bool {
 	return b
 }
 
+func (e Matchers) IgnoreString(s string) bool {
+	b := false
+	for _, exp := range e {
+		i, ok := exp.(Ignorer)
+		if !ok {
+			continue
+		}
+		b = b || i.IgnoreString(s)
+	}
+	return b
+}
+
 // RegExps is a helper to construct Matchers from regular expressions
 func RegExps(rs []*regexp.Regexp) Matchers {
 	xprs := make(Matchers, 0, len(rs))
@@ -50,11 +72,20 @@ func RegExps(rs []*regexp.Regexp) Matchers {
 func StrExps(strs ...string) (Matchers, error) {
 	exps := make(Matchers, 0, len(strs))
 	for _, raw := range strs {
-		s := fmt.Sprintf(`(?i)^%s$`, raw)
+		// trim exlamation mark, not supported by regex
+		s := fmt.Sprintf(`(?i)^%s$`, strings.TrimPrefix(raw, "!"))
+
+		// create regexp matcher
+		var exp Matcher
 		exp, err := regexp.Compile(s)
 		if err != nil {
 			return nil, ErrBadExpr{err}
 		}
+
+		// if negative (!), invert regex behaviour
+		if strings.HasPrefix(raw, "!") {
+			exp = NegMatcher{exp: exp}
+		}
 		exps = append(exps, exp)
 	}
 	return exps, nil
@@ -76,3 +107,16 @@ type ErrBadExpr struct {
 func (e ErrBadExpr) Error() string {
 	return fmt.Sprintf("%s.\nSee https://tanka.dev/output-filtering/#regular-expressions for details on regular expressions.", strings.Title(e.inner.Error()))
 }
+
+// NexMatcher is a matcher that inverts the original behaviour
+type NegMatcher struct {
+	exp Matcher
+}
+
+func (n NegMatcher) MatchString(s string) bool {
+	return true
+}
+
+func (n NegMatcher) IgnoreString(s string) bool {
+	return n.exp.MatchString(s)
+}
diff --git a/pkg/tanka/tanka.go b/pkg/tanka/tanka.go
index fe4f58fa..bc58e93e 100644
--- a/pkg/tanka/tanka.go
+++ b/pkg/tanka/tanka.go
@@ -21,6 +21,7 @@ func parseModifiers(mods []Modifier) *options {
 type options struct {
 	// `std.extVar`
 	extCode map[string]string
+	tlaCode map[string]string
 
 	// target regular expressions to limit the working set
 	targets process.Matchers
@@ -44,6 +45,12 @@ func WithExtCode(code map[string]string) Modifier {
 	}
 }
 
+func WithTLACode(code map[string]string) Modifier {
+	return func(opts *options) {
+		opts.extCode = code
+	}
+}
+
 // WithTargets allows to submit regular expressions to limit the working set of
 // objects (https://tanka.dev/output-filtering/).
 func WithTargets(t process.Matchers) Modifier {


Test patch:
diff --git a/pkg/process/process_test.go b/pkg/process/process_test.go
index 5014b380..cf046159 100644
--- a/pkg/process/process_test.go
+++ b/pkg/process/process_test.go
@@ -64,6 +64,15 @@ func TestProcess(t *testing.T) {
 				`DePlOyMeNt/GrAfAnA`,
 			),
 		},
+		{
+			name: "targets-negative",
+			deep: testDataDeep().Deep,
+			flat: manifest.List{
+				testDataDeep().Flat[".app.web.frontend.nodejs.express.service"],
+				testDataDeep().Flat[".app.namespace"],
+			},
+			targets: MustStrExps(`!deployment/.*`),
+		},
 		{
 			name: "unwrap-list",
 			deep: loadFixture("list").Deep,