Repository: grafana/tanka
Pull request #915 (2023-08-24 01:01:16)

Jsonnet: Add binary jsonnet implementation
_waiting for https://github.com/grafana/tanka/pull/917_

This allows users to specify `binary:<path>` as their jsonnet implementation.

When doing so, Tanka will evaluate jsonnet by invoking the given binary.

github-actions[bot]: [PR Preview Action](https://github.com/rossjrw/pr-preview-action) v1.4.4
:---:
:rocket: Deployed preview to https://grafana.github.io/tanka/pr-preview/pr-915/
on branch [`gh-pages`](https://github.com/grafana/tanka/tree/gh-pages) at 2023-08-24 01:04 UTC
<!-- Sticky Pull Request Commentpr-preview -->

Patch:
diff --git a/.drone/drone.jsonnet b/.drone/drone.jsonnet
index a2ecf75e..a7a8382a 100644
--- a/.drone/drone.jsonnet
+++ b/.drone/drone.jsonnet
@@ -32,6 +32,7 @@ local make(target) = go(target, [
   '  rm -f helm-v3.9.0-linux-amd64.tar.gz',
   'fi',
   'cp linux-amd64/helm /usr/local/bin/helm',
+  'go install github.com/google/go-jsonnet/cmd/jsonnet@v0.20.0',
   'make ' + target,
 ]);
 
@@ -83,6 +84,7 @@ local docker(arch, depends_on=[]) =
     },
     steps: [
       go('benchmark', [
+        'go install github.com/google/go-jsonnet/cmd/jsonnet@v0.20.0',
         'go test -bench=. -benchmem -count=6 -run=^$ ./... | tee bench-pr.txt',
         'git fetch origin main',
         'git reset --hard origin/main',
diff --git a/.drone/drone.yml b/.drone/drone.yml
index 16ba2dda..143da179 100644
--- a/.drone/drone.yml
+++ b/.drone/drone.yml
@@ -16,6 +16,7 @@ steps:
   - '  rm -f helm-v3.9.0-linux-amd64.tar.gz'
   - fi
   - cp linux-amd64/helm /usr/local/bin/helm
+  - go install github.com/google/go-jsonnet/cmd/jsonnet@v0.20.0
   - make lint
   image: golang:1.20
   name: lint
@@ -29,6 +30,7 @@ steps:
   - '  rm -f helm-v3.9.0-linux-amd64.tar.gz'
   - fi
   - cp linux-amd64/helm /usr/local/bin/helm
+  - go install github.com/google/go-jsonnet/cmd/jsonnet@v0.20.0
   - make test
   image: golang:1.20
   name: test
@@ -42,6 +44,7 @@ steps:
   - '  rm -f helm-v3.9.0-linux-amd64.tar.gz'
   - fi
   - cp linux-amd64/helm /usr/local/bin/helm
+  - go install github.com/google/go-jsonnet/cmd/jsonnet@v0.20.0
   - make cross
   image: golang:1.20
   name: build
@@ -62,6 +65,7 @@ node:
   type: no-parallel
 steps:
 - commands:
+  - go install github.com/google/go-jsonnet/cmd/jsonnet@v0.20.0
   - go test -bench=. -benchmem -count=6 -run=^$ ./... | tee bench-pr.txt
   - git fetch origin main
   - git reset --hard origin/main
@@ -99,6 +103,7 @@ steps:
   - '  rm -f helm-v3.9.0-linux-amd64.tar.gz'
   - fi
   - cp linux-amd64/helm /usr/local/bin/helm
+  - go install github.com/google/go-jsonnet/cmd/jsonnet@v0.20.0
   - make cross
   image: golang:1.20
   name: cross
@@ -163,6 +168,7 @@ steps:
   - '  rm -f helm-v3.9.0-linux-amd64.tar.gz'
   - fi
   - cp linux-amd64/helm /usr/local/bin/helm
+  - go install github.com/google/go-jsonnet/cmd/jsonnet@v0.20.0
   - make static
   image: golang:1.20
   name: static
@@ -209,6 +215,7 @@ steps:
   - '  rm -f helm-v3.9.0-linux-amd64.tar.gz'
   - fi
   - cp linux-amd64/helm /usr/local/bin/helm
+  - go install github.com/google/go-jsonnet/cmd/jsonnet@v0.20.0
   - make static
   image: golang:1.20
   name: static
@@ -306,6 +313,6 @@ kind: secret
 name: dockerhub_password
 ---
 kind: signature
-hmac: f0dfa2a5202af05dbfca14e09d1c66222bf6d670d77b5f41b66d696985e89fb7
+hmac: d6e20fdf35f6a2177b563a0db363b55872b276a880165349685c1f6aa1641495
 
 ...
diff --git a/cmd/tk/flags.go b/cmd/tk/flags.go
index ad2715ab..79e43627 100644
--- a/cmd/tk/flags.go
+++ b/cmd/tk/flags.go
@@ -22,7 +22,7 @@ func workflowFlags(fs *pflag.FlagSet) *workflowFlagVars {
 	v := workflowFlagVars{}
 	fs.StringVar(&v.name, "name", "", "string that only a single inline environment contains in its name")
 	fs.StringSliceVarP(&v.targets, "target", "t", nil, "Regex filter on '<kind>/<name>'. See https://tanka.dev/output-filtering")
-	fs.StringVar(&v.jsonnetImplementation, "jsonnet-implementation", "go", "Use `go` to use native go-jsonnet implementation and `binary:<path>` to delegate evaluation to a binary (with the same API as the regular `jsonnet` binary)")
+	fs.StringVar(&v.jsonnetImplementation, "jsonnet-implementation", "go", "Use `go` to use native go-jsonnet implementation and `binary:<path>` to delegate evaluation to a binary (with the same API as the regular `jsonnet` binary, see the BinaryImplementation docstrings for more details)")
 	return &v
 }
 
diff --git a/pkg/jsonnet/implementations/binary/impl.go b/pkg/jsonnet/implementations/binary/impl.go
new file mode 100644
index 00000000..30f37fd4
--- /dev/null
+++ b/pkg/jsonnet/implementations/binary/impl.go
@@ -0,0 +1,70 @@
+package binary
+
+import (
+	"fmt"
+	"os/exec"
+	"strconv"
+
+	"github.com/grafana/tanka/pkg/jsonnet/implementations/types"
+)
+
+type JsonnetBinaryRunner struct {
+	binPath string
+	args    []string
+}
+
+func (r *JsonnetBinaryRunner) EvaluateAnonymousSnippet(snippet string) (string, error) {
+	cmd := exec.Command(r.binPath, append(r.args, "-e", snippet)...)
+
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("error running anonymous snippet: %w\n%s", err, string(out))
+	}
+
+	return string(out), nil
+}
+
+func (r *JsonnetBinaryRunner) EvaluateFile(filename string) (string, error) {
+	cmd := exec.Command(r.binPath, append(r.args, filename)...)
+
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("error running file %s: %w\n%s", filename, err, string(out))
+	}
+
+	return string(out), nil
+}
+
+// JsonnetBinaryImplementation runs Jsonnet in a subprocess. It doesn't support native functions.
+// The interface of the binary has to compatible with the official Jsonnet CLI.
+// It has to support the following flags:
+// -J <path> for specifying import paths
+// --ext-code <name>=<value> for specifying external variables
+// --tla-code <name>=<value> for specifying top-level arguments
+// --max-stack <value> for specifying the maximum stack size
+// -e <code> for evaluating code snippets
+// <filename> positional arg for evaluating files
+type JsonnetBinaryImplementation struct {
+	BinPath string
+}
+
+func (i *JsonnetBinaryImplementation) MakeEvaluator(importPaths []string, extCode map[string]string, tlaCode map[string]string, maxStack int) types.JsonnetEvaluator {
+	args := []string{}
+	for _, p := range importPaths {
+		args = append(args, "-J", p)
+	}
+	if maxStack > 0 {
+		args = append(args, "--max-stack", strconv.Itoa(maxStack))
+	}
+	for k, v := range extCode {
+		args = append(args, "--ext-code", k+"="+v)
+	}
+	for k, v := range tlaCode {
+		args = append(args, "--tla-code", k+"="+v)
+	}
+
+	return &JsonnetBinaryRunner{
+		binPath: i.BinPath,
+		args:    args,
+	}
+}
diff --git a/pkg/tanka/load.go b/pkg/tanka/load.go
index 5e30efd3..0721c6f7 100644
--- a/pkg/tanka/load.go
+++ b/pkg/tanka/load.go
@@ -4,7 +4,9 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
+	"github.com/grafana/tanka/pkg/jsonnet/implementations/binary"
 	"github.com/grafana/tanka/pkg/jsonnet/implementations/goimpl"
 	"github.com/grafana/tanka/pkg/jsonnet/implementations/types"
 	"github.com/grafana/tanka/pkg/jsonnet/jpath"
@@ -103,6 +105,23 @@ func List(path string, opts Opts) ([]*v1alpha1.Environment, error) {
 }
 
 func getJsonnetImplementation(path string, opts Opts) (types.JsonnetImplementation, error) {
+	if strings.HasPrefix(opts.JsonnetImplementation, "binary:") {
+		binPath := strings.TrimPrefix(opts.JsonnetImplementation, "binary:")
+
+		// check if binary exists and is executable
+		stat, err := os.Stat(binPath)
+		if err != nil {
+			return nil, fmt.Errorf("binary %q does not exist", binPath)
+		}
+		if stat.Mode()&0111 == 0 {
+			return nil, fmt.Errorf("binary %q is not executable", binPath)
+		}
+
+		return &binary.JsonnetBinaryImplementation{
+			BinPath: binPath,
+		}, nil
+	}
+
 	switch opts.JsonnetImplementation {
 	case "go", "":
 		return &goimpl.JsonnetGoImplementation{


Test patch:
diff --git a/pkg/jsonnet/eval_test.go b/pkg/jsonnet/eval_test.go
index db15b3de..a1051502 100644
--- a/pkg/jsonnet/eval_test.go
+++ b/pkg/jsonnet/eval_test.go
@@ -2,9 +2,11 @@ package jsonnet
 
 import (
 	"os"
+	"os/exec"
 	"path/filepath"
 	"testing"
 
+	"github.com/grafana/tanka/pkg/jsonnet/implementations/binary"
 	"github.com/grafana/tanka/pkg/jsonnet/implementations/goimpl"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -59,6 +61,21 @@ func TestEvaluateFile(t *testing.T) {
 	assert.Equal(t, thisFileResult, result)
 }
 
+func TestEvaluateFileWithInvalidBinary(t *testing.T) {
+	binaryImpl := &binary.JsonnetBinaryImplementation{BinPath: "this-file-doesnt-exist"}
+	result, err := EvaluateFile(binaryImpl, "testdata/thisFile/main.jsonnet", Opts{})
+	assert.Equal(t, result, "")
+	assert.ErrorIs(t, err, exec.ErrNotFound)
+}
+
+// This test requires jsonnet to be installed and available in the PATH
+func TestEvaluateFileWithJsonnetBinary(t *testing.T) {
+	binaryImpl := &binary.JsonnetBinaryImplementation{BinPath: "jsonnet"}
+	result, err := EvaluateFile(binaryImpl, "testdata/thisFile/main.jsonnet", Opts{})
+	assert.NoError(t, err)
+	assert.Equal(t, thisFileResult, result)
+}
+
 func TestEvaluateFileDoesntExist(t *testing.T) {
 	result, err := EvaluateFile(jsonnetImpl, "testdata/doesnt-exist/main.jsonnet", Opts{})
 	assert.EqualError(t, err, "open testdata/doesnt-exist/main.jsonnet: no such file or directory")
diff --git a/pkg/tanka/load_test.go b/pkg/tanka/load_test.go
index f82774b2..b2f44d6f 100644
--- a/pkg/tanka/load_test.go
+++ b/pkg/tanka/load_test.go
@@ -2,10 +2,15 @@ package tanka
 
 import (
 	"os"
+	"path/filepath"
 	"testing"
 
+	"github.com/grafana/tanka/pkg/jsonnet/implementations/binary"
+	"github.com/grafana/tanka/pkg/jsonnet/implementations/goimpl"
+	"github.com/grafana/tanka/pkg/jsonnet/implementations/types"
 	"github.com/grafana/tanka/pkg/kubernetes/manifest"
 	"github.com/grafana/tanka/pkg/spec/v1alpha1"
+	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -211,3 +216,68 @@ func TestLoadFailsWhenBothSpecAndInline(t *testing.T) {
 	_, err := Load("./testdata/cases/static-and-inline", Opts{Name: "inline"})
 	assert.EqualError(t, err, "found a tanka Environment resource. Check that you aren't using a spec.json and inline environments simultaneously")
 }
+
+func TestGetJsonnetImplementation(t *testing.T) {
+	tempDir := t.TempDir()
+
+	// Write file that is not executable
+	notExecutablePath := filepath.Join(tempDir, "not-executable")
+	require.NoError(t, os.WriteFile(notExecutablePath, []byte("not executable"), 0644))
+
+	// Write file that is executable
+	executablePath := filepath.Join(tempDir, "executable")
+	require.NoError(t, os.WriteFile(executablePath, []byte("executable"), 0755))
+
+	cases := []struct {
+		implementationName string
+		path               string
+		expected           types.JsonnetImplementation
+		expectedErr        error
+	}{
+		{
+			implementationName: "",
+			path:               "my-dir",
+			expected: &goimpl.JsonnetGoImplementation{
+				Path: "my-dir",
+			},
+		},
+		{
+			implementationName: "go",
+			path:               "my-dir",
+			expected: &goimpl.JsonnetGoImplementation{
+				Path: "my-dir",
+			},
+		},
+		{
+			implementationName: "binary:does-not-exist",
+			expectedErr:        errors.New(`binary "does-not-exist" does not exist`),
+		},
+		{
+			implementationName: "binary:" + notExecutablePath,
+			expectedErr:        errors.New(`binary "` + notExecutablePath + `" is not executable`),
+		},
+		{
+			implementationName: "binary:" + executablePath,
+			expected: &binary.JsonnetBinaryImplementation{
+				BinPath: executablePath,
+			},
+		},
+		{
+			implementationName: "invalid",
+			expectedErr:        errors.New("unknown jsonnet implementation: invalid"),
+		},
+	}
+
+	for _, tt := range cases {
+		t.Run(tt.implementationName, func(t *testing.T) {
+			implementation, err := getJsonnetImplementation(tt.path, Opts{JsonnetImplementation: tt.implementationName})
+
+			if tt.expectedErr != nil {
+				assert.EqualError(t, err, tt.expectedErr.Error())
+				return
+			}
+
+			assert.Equal(t, tt.expected, implementation)
+		})
+	}
+}