Repository: grafana/tanka
Pull request #700 (2022-05-04 22:10:47)

Support a diff-strategy of "none" for "tk apply" to suppress diffing.
This pull request adds a diff strategy of `none` for `tk apply` that completely suppresses diffing before the apply operation.

The reason I'm submitting this is that I'm using Tanka to create a very large set of Kubernetes resources and it takes 5 to 10 minutes to perform the diff operation before getting to the apply stage.  Since I often use a `tk diff` myself first, before the `tk apply`, I don't need the diff that's part of `apply` in some cases.  Also, when our automated job runs to do a `tk apply`, it uses the `--dangerous-auto-approve` option, with no human present observe the diff output.  The 5 to 10 minute delay in this step of the automation makes the automation job run longer that desired, delaying the deployment of the things that come _after_ this step.

I would like to change some documentation to describe the `none` option to `tk apply`, but I don't see any good place to put that.  Any suggestion would be welcome.


Patch:
diff --git a/cmd/tk/lint.go b/cmd/tk/lint.go
new file mode 100644
index 00000000..3c435f04
--- /dev/null
+++ b/cmd/tk/lint.go
@@ -0,0 +1,46 @@
+package main
+
+import (
+	"errors"
+
+	"github.com/go-clix/cli"
+	"github.com/gobwas/glob"
+	"github.com/posener/complete"
+
+	"github.com/grafana/tanka/pkg/jsonnet"
+)
+
+func lintCmd() *cli.Command {
+	cmd := &cli.Command{
+		Use:   "lint <FILES|DIRECTORIES>",
+		Short: "lint Jsonnet code",
+		Args: cli.Args{
+			Validator: cli.ValidateFunc(func(args []string) error {
+				if len(args) == 0 {
+					return errors.New("at least one file or directory is required")
+				}
+				return nil
+			}),
+			Predictor: complete.PredictFiles("*.*sonnet"),
+		},
+	}
+
+	exclude := cmd.Flags().StringSliceP("exclude", "e", []string{"**/.*", ".*", "**/vendor/**", "vendor/**"}, "globs to exclude")
+	parallelism := cmd.Flags().IntP("parallelism", "n", 4, "amount of workers")
+	verbose := cmd.Flags().BoolP("verbose", "v", false, "print each checked file")
+
+	cmd.Run = func(cmd *cli.Command, args []string) error {
+		globs := make([]glob.Glob, len(*exclude))
+		for i, e := range *exclude {
+			g, err := glob.Compile(e)
+			if err != nil {
+				return err
+			}
+			globs[i] = g
+		}
+
+		return jsonnet.Lint(args, &jsonnet.LintOpts{Excludes: globs, PrintNames: *verbose, Parallelism: *parallelism})
+	}
+
+	return cmd
+}
diff --git a/cmd/tk/main.go b/cmd/tk/main.go
index 52cf28ad..c8ff31e7 100644
--- a/cmd/tk/main.go
+++ b/cmd/tk/main.go
@@ -44,6 +44,7 @@ func main() {
 	// jsonnet commands
 	rootCmd.AddCommand(
 		fmtCmd(),
+		lintCmd(),
 		evalCmd(),
 		initCmd(),
 		toolCmd(),
diff --git a/cmd/tk/tool.go b/cmd/tk/tool.go
index a4043906..a5ab070f 100644
--- a/cmd/tk/tool.go
+++ b/cmd/tk/tool.go
@@ -46,7 +46,7 @@ func jpathCmd() *cli.Command {
 			return fmt.Errorf("Resolving JPATH: %s", err)
 		}
 
-		jsonnetpath, base, root, err := jpath.Resolve(entrypoint)
+		jsonnetpath, base, root, err := jpath.Resolve(entrypoint, false)
 		if err != nil {
 			return fmt.Errorf("Resolving JPATH: %s", err)
 		}
diff --git a/cmd/tk/workflow.go b/cmd/tk/workflow.go
index 07216492..125ec02b 100644
--- a/cmd/tk/workflow.go
+++ b/cmd/tk/workflow.go
@@ -35,7 +35,7 @@ func applyCmd() *cli.Command {
 		Short: "apply the configuration to the cluster",
 		Args:  workflowArgs,
 		Predictors: complete.Flags{
-			"diff-strategy":  cli.PredictSet("native", "subset", "validate", "server"),
+			"diff-strategy":  cli.PredictSet("native", "subset", "validate", "server", "none"),
 			"apply-strategy": cli.PredictSet("client", "server"),
 		},
 	}
diff --git a/docs/docs/tutorial/k-lib.mdx b/docs/docs/tutorial/k-lib.mdx
index 2f65e01b..5b1372bb 100644
--- a/docs/docs/tutorial/k-lib.mdx
+++ b/docs/docs/tutorial/k-lib.mdx
@@ -129,7 +129,7 @@ local k = import "github.com/grafana/jsonnet-libs/ksonnet-util/kausal.libsonnet"
 
     // instead of using a service constructor, our wrapper provides
     // a handy helper to automatically generate a service for a Deployment
-    service: $.util.serviceFor(self.deployment)
+    service: k.util.serv.util.serviceFor(self.deployment)
              + service.mixin.spec.withType("NodePort"),
   }
 }
diff --git a/pkg/jsonnet/eval.go b/pkg/jsonnet/eval.go
index 33e2eb0f..f532711d 100644
--- a/pkg/jsonnet/eval.go
+++ b/pkg/jsonnet/eval.go
@@ -132,7 +132,7 @@ func evaluateSnippet(evalFunc evalFunc, path, data string, opts Opts) (string, e
 	}
 
 	// Create VM
-	jpath, _, _, err := jpath.Resolve(path)
+	jpath, _, _, err := jpath.Resolve(path, false)
 	if err != nil {
 		return "", errors.Wrap(err, "resolving import paths")
 	}
diff --git a/pkg/jsonnet/files.go b/pkg/jsonnet/files.go
new file mode 100644
index 00000000..9b08cbb6
--- /dev/null
+++ b/pkg/jsonnet/files.go
@@ -0,0 +1,55 @@
+package jsonnet
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/gobwas/glob"
+	"github.com/karrick/godirwalk"
+)
+
+// FindFiles takes a file / directory and finds all Jsonnet files
+func FindFiles(target string, excludes []glob.Glob) ([]string, error) {
+	// if it's a file, don't try to find children
+	fi, err := os.Stat(target)
+	if err != nil {
+		return nil, err
+	}
+	if fi.Mode().IsRegular() {
+		return []string{target}, nil
+	}
+
+	var files []string
+
+	// godirwalk is faster than filepath.Walk, 'cause no os.Stat required
+	err = godirwalk.Walk(target, &godirwalk.Options{
+		Callback: func(rawPath string, de *godirwalk.Dirent) error {
+			// Normalize slashes for Windows
+			path := filepath.ToSlash(rawPath)
+
+			if de.IsDir() {
+				return nil
+			}
+
+			// excluded?
+			for _, g := range excludes {
+				if g.Match(path) {
+					return nil
+				}
+			}
+
+			// only .jsonnet or .libsonnet
+			if ext := filepath.Ext(path); ext == ".jsonnet" || ext == ".libsonnet" {
+				files = append(files, path)
+			}
+			return nil
+		},
+		// faster, no sort required
+		Unsorted: true,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return files, nil
+}
diff --git a/pkg/jsonnet/imports.go b/pkg/jsonnet/imports.go
index 543fb95c..ecc975b8 100644
--- a/pkg/jsonnet/imports.go
+++ b/pkg/jsonnet/imports.go
@@ -40,7 +40,7 @@ func TransitiveImports(dir string) ([]string, error) {
 		return nil, errors.Wrap(err, "opening file")
 	}
 
-	jpath, _, rootDir, err := jpath.Resolve(dir)
+	jpath, _, rootDir, err := jpath.Resolve(dir, false)
 	if err != nil {
 		return nil, errors.Wrap(err, "resolving JPATH")
 	}
diff --git a/pkg/jsonnet/jpath/jpath.go b/pkg/jsonnet/jpath/jpath.go
index d0372dc2..96ca678b 100644
--- a/pkg/jsonnet/jpath/jpath.go
+++ b/pkg/jsonnet/jpath/jpath.go
@@ -14,14 +14,19 @@ const DEFAULT_ENTRYPOINT = "main.jsonnet"
 // It then constructs a jPath with the base directory, vendor/ and lib/.
 // This results in predictable imports, as it doesn't matter whether the user called
 // called the command further down tree or not. A little bit like git.
-func Resolve(path string) (jpath []string, base, root string, err error) {
+func Resolve(path string, allowMissingBase bool) (jpath []string, base, root string, err error) {
 	root, err = FindRoot(path)
 	if err != nil {
 		return nil, "", "", err
 	}
 
 	base, err = FindBase(path, root)
-	if err != nil {
+	if err != nil && allowMissingBase {
+		base, err = FsDir(path)
+		if err != nil {
+			return nil, "", "", err
+		}
+	} else if err != nil {
 		return nil, "", "", err
 	}
 
diff --git a/pkg/jsonnet/lint.go b/pkg/jsonnet/lint.go
new file mode 100644
index 00000000..bd8e63fa
--- /dev/null
+++ b/pkg/jsonnet/lint.go
@@ -0,0 +1,100 @@
+package jsonnet
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/gobwas/glob"
+	"github.com/google/go-jsonnet/linter"
+	"github.com/grafana/tanka/pkg/jsonnet/jpath"
+	"github.com/pkg/errors"
+)
+
+// LintOpts modifies the behaviour of Lint
+type LintOpts struct {
+	// Excludes are a list of globs to exclude files while searching for Jsonnet
+	// files
+	Excludes []glob.Glob
+
+	// PrintNames causes all filenames to be printed
+	PrintNames bool
+
+	// Parallelism determines the number of workers that will process files
+	Parallelism int
+}
+
+// Lint takes a list of files and directories, processes them and prints
+// out to stderr if there are linting warnings
+func Lint(fds []string, opts *LintOpts) error {
+
+	var paths []string
+	for _, f := range fds {
+		fs, err := FindFiles(f, opts.Excludes)
+		if err != nil {
+			return errors.Wrap(err, "finding Jsonnet files")
+		}
+		paths = append(paths, fs...)
+	}
+
+	type result struct {
+		failed bool
+		output string
+	}
+	fileCh := make(chan string, len(paths))
+	resultCh := make(chan result, len(paths))
+	lintWorker := func(fileCh <-chan string, resultCh chan result) {
+		for file := range fileCh {
+			buf := &bytes.Buffer{}
+			var err error
+			file, err = filepath.Abs(file)
+			if err != nil {
+				fmt.Fprintf(buf, "got an error getting the absolute path for %s: %v\n\n", file, err)
+				resultCh <- result{failed: true, output: buf.String()}
+				continue
+			}
+
+			if opts.PrintNames {
+				fmt.Printf("Linting %s...\n", file)
+			}
+
+			vm := MakeVM(Opts{})
+			jpaths, _, _, err := jpath.Resolve(file, true)
+			if err != nil {
+				fmt.Fprintf(buf, "got an error getting JPATH for %s: %v\n\n", file, err)
+				resultCh <- result{failed: true, output: buf.String()}
+				continue
+			}
+
+			vm.Importer(NewExtendedImporter(jpaths))
+
+			content, _ := ioutil.ReadFile(file)
+			failed := linter.LintSnippet(vm, buf, []linter.Snippet{{FileName: file, Code: string(content)}})
+			resultCh <- result{failed: failed, output: buf.String()}
+		}
+	}
+
+	for i := 0; i < opts.Parallelism; i++ {
+		go lintWorker(fileCh, resultCh)
+	}
+
+	for _, file := range paths {
+		fileCh <- file
+	}
+	close(fileCh)
+
+	lintingFailed := false
+	for i := 0; i < len(paths); i++ {
+		result := <-resultCh
+		lintingFailed = lintingFailed || result.failed
+		if result.output != "" {
+			fmt.Print(result.output)
+		}
+	}
+
+	if lintingFailed {
+		return errors.New("Linting has failed for at least one file")
+	}
+	return nil
+}
diff --git a/pkg/tanka/format.go b/pkg/tanka/format.go
index 941c25aa..f71e8be6 100644
--- a/pkg/tanka/format.go
+++ b/pkg/tanka/format.go
@@ -3,11 +3,10 @@ package tanka
 import (
 	"fmt"
 	"os"
-	"path/filepath"
 
 	"github.com/gobwas/glob"
 	"github.com/google/go-jsonnet/formatter"
-	"github.com/karrick/godirwalk"
+	"github.com/grafana/tanka/pkg/jsonnet"
 	"github.com/pkg/errors"
 )
 
@@ -34,7 +33,7 @@ type OutFn func(name, content string) error
 func FormatFiles(fds []string, opts *FormatOpts) ([]string, error) {
 	var paths []string
 	for _, f := range fds {
-		fs, err := findFiles(f, opts.Excludes)
+		fs, err := jsonnet.FindFiles(f, opts.Excludes)
 		if err != nil {
 			return nil, errors.Wrap(err, "finding Jsonnet files")
 		}
@@ -87,49 +86,3 @@ func FormatFiles(fds []string, opts *FormatOpts) ([]string, error) {
 func Format(filename string, content string) (string, error) {
 	return formatter.Format(filename, content, formatter.DefaultOptions())
 }
-
-// findFiles takes a file / directory and finds all Jsonnet files
-func findFiles(target string, excludes []glob.Glob) ([]string, error) {
-	// if it's a file, don't try to find children
-	fi, err := os.Stat(target)
-	if err != nil {
-		return nil, err
-	}
-	if fi.Mode().IsRegular() {
-		return []string{target}, nil
-	}
-
-	var files []string
-
-	// godirwalk is faster than filepath.Walk, 'cause no os.Stat required
-	err = godirwalk.Walk(target, &godirwalk.Options{
-		Callback: func(rawPath string, de *godirwalk.Dirent) error {
-			// Normalize slashes for Windows
-			path := filepath.ToSlash(rawPath)
-
-			if de.IsDir() {
-				return nil
-			}
-
-			// excluded?
-			for _, g := range excludes {
-				if g.Match(path) {
-					return nil
-				}
-			}
-
-			// only .jsonnet or .libsonnet
-			if ext := filepath.Ext(path); ext == ".jsonnet" || ext == ".libsonnet" {
-				files = append(files, path)
-			}
-			return nil
-		},
-		// faster, no sort required
-		Unsorted: true,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return files, nil
-}
diff --git a/pkg/tanka/workflow.go b/pkg/tanka/workflow.go
index a31c0040..da3018a1 100644
--- a/pkg/tanka/workflow.go
+++ b/pkg/tanka/workflow.go
@@ -75,21 +75,23 @@ func Apply(baseDir string, opts ApplyOpts) error {
 	}
 	defer kube.Close()
 
-	// show diff
-	diff, err := kube.Diff(l.Resources, kubernetes.DiffOpts{Strategy: opts.DiffStrategy})
-	switch {
-	case err != nil:
-		// This is not fatal, the diff is not strictly required
-		log.Println("Error diffing:", err)
-	case diff == nil:
-		tmp := "Warning: There are no differences. Your apply may not do anything at all."
-		diff = &tmp
-	}
+	if opts.DiffStrategy != "none" {
+		// show diff
+		diff, err := kube.Diff(l.Resources, kubernetes.DiffOpts{Strategy: opts.DiffStrategy})
+		switch {
+		case err != nil:
+			// This is not fatal, the diff is not strictly required
+			log.Println("Error diffing:", err)
+		case diff == nil:
+			tmp := "Warning: There are no differences. Your apply may not do anything at all."
+			diff = &tmp
+		}
 
-	// in case of non-fatal error diff may be nil
-	if diff != nil {
-		b := term.Colordiff(*diff)
-		fmt.Print(b.String())
+		// in case of non-fatal error diff may be nil
+		if diff != nil {
+			b := term.Colordiff(*diff)
+			fmt.Print(b.String())
+		}
 	}
 
 	// prompt for confirmation


Test patch:
diff --git a/pkg/jsonnet/jpath/dirs_test.go b/pkg/jsonnet/jpath/dirs_test.go
index 5270d499..635e0c00 100644
--- a/pkg/jsonnet/jpath/dirs_test.go
+++ b/pkg/jsonnet/jpath/dirs_test.go
@@ -188,7 +188,7 @@ func TestFindRoot(t *testing.T) {
 			dir := makeTestdata(t, s.testdata)
 			defer os.RemoveAll(dir)
 
-			_, base, root, err := Resolve(filepath.Join(dir, s.environment))
+			_, base, root, err := Resolve(filepath.Join(dir, s.environment), false)
 			assert.Equal(t, s.err, err)
 
 			if err == nil {