Repository: score-spec/score-compose
Pull request #84 (2024-03-15 16:37:50)

feat: added support for cmd:// provisioners
This PR add's a new kind of provisioner: `cmd`. This allows you to encode the provisioner inputs as json and pass them via stdin to a new process, this process will execute and pass json back out to the resource system to persist.

This supports:

- `cmd://my-binary` - a binary that exists on the `$PATH`
- `cmd:///an/absolute/path` - an absolute path
- `cmd://./a/relative/path` - a relative path
- `cmd://~/bin/thing` - something in the home directory

Additional arguments can be passed by providing them in the `args` configuration:

```
- uri: cmd://python3
  type: my-resource-type
  args:
  - -c
  - |
    import sys, json
    inputs = json.load(sys.stdin)
    print(json.dumps({"resource_outputs": {"key": "value"}})
```

And stderr content will be printed in the same shell as the `score-compose generate` invocation.

This will support much more advanced scripting mechanism for resource provisioning. The executed binary can still take advantage of all the state storage, and volume/file management that the provisioning system does in the .score-compose directory and doesn't need to manage state itself.



Patch:
diff --git a/README.md b/README.md
index 5015c94..2f04e50 100644
--- a/README.md
+++ b/README.md
@@ -52,7 +52,7 @@ See the [examples](./examples) for more examples of using Score and provisioning
 - [03-files](examples/03-files) - mounting local files into the running Workload
 - [04-multiple-workloads](examples/04-multiple-workloads) - examples of multiple containers and workloads together
 - [05-volume-mounts](examples/05-volume-mounts) - an example of an "empty-dir" volume resource with `type: volume`
-- [06-resource-provisioning](examples/06-resource-provisioning) - detailed example and information about resource provisioning and the operation of the `template://` provisioner
+- [06-resource-provisioning](examples/06-resource-provisioning) - detailed example and information about resource provisioning and the operation of the `template://` and `cmd://` provisioners
 - [07-overrides](examples/07-overrides) - details of how to override aspects of the input Score file and output Docker compose files
 
 If you're getting started, you can use `score-compose init` to create a basic `score.yaml` file in the current directory along with a `.score-compose/` working directory.
diff --git a/examples/06-resource-provisioning/README.md b/examples/06-resource-provisioning/README.md
index 3624d6b..87f8ece 100644
--- a/examples/06-resource-provisioning/README.md
+++ b/examples/06-resource-provisioning/README.md
@@ -84,7 +84,7 @@ Each entry in the file has the following common fields, other fields may also ex
   id: <optional resource id>
 ```
 
-More provisioner types will be added in the future.
+The uri of each provisioner is a combination of it's implementation (either `template://` or `cmd://`) and a unique identifier.
 
 ### The `template://` provisioner
 
@@ -124,3 +124,23 @@ type Data struct {
 ```
 
 Browse the default provisioners for inspiration or more clues to how these work!
+
+### The `cmd://` provisioner
+
+The command provisioner implementation can be used to execute an external binary or script to provision the resource. The provision IO structures are serialised to json and send on standard-input to the new process, any stdout content is decoded as json and is used as the outputs of the provisioner.
+
+The uri of the provisioner encodes the binary to be executed:
+
+- `cmd://python` will execute the `python` binary on the PATH
+- `cmd://../my-script` will execute `../my-script`
+- `cmd://./my-script` will execute `my-script` in the current directory
+- and `cmd://~/my-script` will execute the `my-script` binary in the home directory
+
+Additional arguments can be provided via the `args` configuration key, for example a basic provisioner can be created using python inline scripts:
+
+```
+- uri: "cmd://python"
+  args: ["-c", "print({\"resource_outputs\":{}})"]
+```
+
+The JSON structures are the `Input` and `ProvisionOutput` structures in [internal/provisioners/core.go](https://github.com/score-spec/score-compose/blob/3cee56c624a70821a55af44a15513ebf8b594f9a/internal/provisioners/core.go#L35).
diff --git a/internal/provisioners/cmdprov/commandprov.go b/internal/provisioners/cmdprov/commandprov.go
new file mode 100644
index 0000000..de37e41
--- /dev/null
+++ b/internal/provisioners/cmdprov/commandprov.go
@@ -0,0 +1,154 @@
+// Copyright 2024 Humanitec
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmdprov
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/score-spec/score-compose/internal/project"
+	"github.com/score-spec/score-compose/internal/provisioners"
+)
+
+type Provisioner struct {
+	ProvisionerUri string   `yaml:"uri"`
+	ResType        string   `yaml:"type"`
+	ResClass       *string  `yaml:"class,omitempty"`
+	ResId          *string  `yaml:"id,omitempty"`
+	Args           []string `yaml:"args"`
+}
+
+func (p *Provisioner) Uri() string {
+	return p.ProvisionerUri
+}
+
+func (p *Provisioner) Match(resUid project.ResourceUid) bool {
+	if resUid.Type() != p.ResType {
+		return false
+	} else if p.ResClass != nil && resUid.Class() != *p.ResClass {
+		return false
+	} else if p.ResId != nil && resUid.Id() != *p.ResId {
+		return false
+	}
+	return true
+}
+
+func decodeBinary(uri string) (string, error) {
+	parts, _ := url.Parse(uri)
+	pathParts := strings.Split(parts.EscapedPath(), "/")
+	switch parts.Hostname() {
+	case "":
+		return string(filepath.Separator) + filepath.Join(pathParts...), nil
+	case "~":
+		hd, err := os.UserHomeDir()
+		if err != nil {
+			return "", fmt.Errorf("failed to resolve user home directory: %w", err)
+		}
+		pathParts = slices.Insert(pathParts, 0, hd)
+	case ".":
+		pwd, err := os.Getwd()
+		if err != nil {
+			return "", fmt.Errorf("failed to resolve current working directory: %w", err)
+		}
+		pathParts = slices.Insert(pathParts, 0, pwd)
+	case "..":
+		pwd, err := os.Getwd()
+		if err != nil {
+			return "", fmt.Errorf("failed to resolve current working directory: %w", err)
+		}
+		pathParts = slices.Insert(pathParts, 0, filepath.Dir(pwd))
+	default:
+		if len(pathParts) > 1 {
+			return "", fmt.Errorf("direct command reference cannot contain additional path parts")
+		}
+		b, err := exec.LookPath(parts.Hostname())
+		if err != nil {
+			return "", fmt.Errorf("failed to find '%s' on path: %w", parts.Hostname(), err)
+		}
+		pathParts = slices.Insert(pathParts, 0, b)
+	}
+	return filepath.Join(pathParts...), nil
+}
+
+func (p *Provisioner) Provision(ctx context.Context, input *provisioners.Input) (*provisioners.ProvisionOutput, error) {
+	bin, err := decodeBinary(p.Uri())
+	if err != nil {
+		return nil, err
+	}
+
+	rawInput, err := json.Marshal(input)
+	if err != nil {
+		return nil, fmt.Errorf("failed to encode json input: %w", err)
+	}
+	outputBuffer := new(bytes.Buffer)
+
+	cmd := exec.CommandContext(ctx, bin, p.Args...)
+	slog.Debug(fmt.Sprintf("Executing '%s %v' for command provisioner", bin, p.Args))
+	cmd.Stdin = bytes.NewReader(rawInput)
+	cmd.Stdout = outputBuffer
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return nil, fmt.Errorf("failed to execute cmd provisioner: %w", err)
+	}
+
+	var output provisioners.ProvisionOutput
+	dec := json.NewDecoder(bytes.NewReader(outputBuffer.Bytes()))
+	dec.DisallowUnknownFields()
+	if err := dec.Decode(&output); err != nil {
+		slog.Debug("Output from command provisioner:\n" + outputBuffer.String())
+		return nil, fmt.Errorf("failed to decode output from cmd provisioner: %w", err)
+	}
+
+	return &output, nil
+}
+
+func Parse(raw map[string]interface{}) (*Provisioner, error) {
+	p := new(Provisioner)
+	intermediate, _ := yaml.Marshal(raw)
+	dec := yaml.NewDecoder(bytes.NewReader(intermediate))
+	dec.KnownFields(true)
+	if err := dec.Decode(&p); err != nil {
+		return nil, err
+	}
+	if p.ProvisionerUri == "" {
+		return nil, fmt.Errorf("uri not set")
+	} else if p.ResType == "" {
+		return nil, fmt.Errorf("type not set")
+	}
+
+	parts, err := url.Parse(p.ProvisionerUri)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse url: %w", err)
+	} else if parts.User != nil {
+		return nil, fmt.Errorf("cmd provisioner uri cannot contain user info")
+	} else if len(parts.Query()) != 0 {
+		return nil, fmt.Errorf("cmd provisioner uri cannot contain query params")
+	} else if parts.Port() != "" {
+		return nil, fmt.Errorf("cmd provisioner uri cannot contain a port")
+	}
+
+	return p, nil
+}
diff --git a/internal/provisioners/loader/load.go b/internal/provisioners/loader/load.go
index a77658b..232fa4f 100644
--- a/internal/provisioners/loader/load.go
+++ b/internal/provisioners/loader/load.go
@@ -26,6 +26,7 @@ import (
 	"gopkg.in/yaml.v3"
 
 	"github.com/score-spec/score-compose/internal/provisioners"
+	"github.com/score-spec/score-compose/internal/provisioners/cmdprov"
 	"github.com/score-spec/score-compose/internal/provisioners/templateprov"
 )
 
@@ -54,6 +55,13 @@ func LoadProvisioners(raw []byte) ([]provisioners.Provisioner, error) {
 				slog.Debug(fmt.Sprintf("Loaded provisioner %s", p.Uri()))
 				out = append(out, p)
 			}
+		case "cmd":
+			if p, err := cmdprov.Parse(m); err != nil {
+				return nil, fmt.Errorf("%d: %s: failed to parse: %w", i, uri, err)
+			} else {
+				slog.Debug(fmt.Sprintf("Loaded provisioner %s", p.Uri()))
+				out = append(out, p)
+			}
 		default:
 			return nil, fmt.Errorf("%d: unsupported provisioner type '%s'", i, u.Scheme)
 		}


Test patch:
diff --git a/internal/provisioners/cmdprov/commandprov_test.go b/internal/provisioners/cmdprov/commandprov_test.go
new file mode 100644
index 0000000..58364ba
--- /dev/null
+++ b/internal/provisioners/cmdprov/commandprov_test.go
@@ -0,0 +1,131 @@
+// Copyright 2024 Humanitec
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmdprov
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/score-spec/score-compose/internal/provisioners"
+)
+
+func TestParseUri_success(t *testing.T) {
+	for _, k := range []string{
+		"cmd://python",
+		"cmd:///absolute/path/here",
+		"cmd://./relative/path",
+		"cmd://../parent/path",
+		"cmd://~/path",
+	} {
+		t.Run(k, func(t *testing.T) {
+			out, err := Parse(map[string]interface{}{"uri": k, "type": "foo"})
+			if assert.NoError(t, err) {
+				assert.Equal(t, k, out.Uri())
+			}
+		})
+	}
+}
+
+func TestParseUri_fail(t *testing.T) {
+	for k, v := range map[string]string{
+		"":                      "uri not set",
+		"cmd://x:x":             "failed to parse url: parse \"cmd://x:x\": invalid port \":x\" after host",
+		"cmd://something@foo":   "cmd provisioner uri cannot contain user info",
+		"cmd://something:80":    "cmd provisioner uri cannot contain a port",
+		"cmd://something?foo=x": "cmd provisioner uri cannot contain query params",
+	} {
+		t.Run(k, func(t *testing.T) {
+			_, err := Parse(map[string]interface{}{"uri": k, "type": "foo"})
+			assert.EqualError(t, err, v)
+		})
+	}
+}
+
+func TestDecodeBinary_success(t *testing.T) {
+	dir, _ := os.Getwd()
+	gogo, _ := exec.LookPath("go")
+	for k, v := range map[string]string{
+		"cmd://./thing":        dir + "/thing",
+		"cmd://../thing/foo":   filepath.Dir(dir) + "/thing/foo",
+		"cmd://~/path":         os.Getenv("HOME") + "/path",
+		"cmd:///absolute/path": "/absolute/path",
+		"cmd://go":             gogo,
+	} {
+		t.Run(k, func(t *testing.T) {
+			out, err := decodeBinary(k)
+			if assert.NoError(t, err) {
+				assert.Equal(t, v, out)
+			}
+		})
+	}
+}
+
+func TestDecodeBinary_fail(t *testing.T) {
+	for k, v := range map[string]string{
+		"cmd://absolutely-unknown-score-compose-provisioner": "failed to find 'absolutely-unknown-score-compose-provisioner' on path: exec: \"absolutely-unknown-score-compose-provisioner\": executable file not found in $PATH",
+		"cmd://something/foo":                                "direct command reference cannot contain additional path parts",
+	} {
+		t.Run(k, func(t *testing.T) {
+			_, err := decodeBinary(k)
+			assert.EqualError(t, err, v)
+		})
+	}
+}
+
+func TestProvision_success(t *testing.T) {
+	p, err := Parse(map[string]interface{}{
+		"uri":  "cmd://sh",
+		"type": "thing",
+		"args": []string{"-c", "echo '{\"resource_outputs\":' `cat` '}'"},
+	})
+	require.NoError(t, err)
+	po, err := p.Provision(context.Background(), &provisioners.Input{
+		ResourceUid: "thing.default#w.r",
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "thing.default#w.r", po.ResourceOutputs["resource_uid"])
+}
+
+func TestProvision_fail_command(t *testing.T) {
+	p, err := Parse(map[string]interface{}{
+		"uri":  "cmd://sh",
+		"type": "thing",
+		"args": []string{"-c", "false"},
+	})
+	require.NoError(t, err)
+	_, err = p.Provision(context.Background(), &provisioners.Input{
+		ResourceUid: "thing.default#w.r",
+	})
+	require.EqualError(t, err, "failed to execute cmd provisioner: exit status 1")
+}
+
+func TestProvision_fail_decode(t *testing.T) {
+	p, err := Parse(map[string]interface{}{
+		"uri":  "cmd://sh",
+		"type": "thing",
+		"args": []string{"-c", "echo bananas"},
+	})
+	require.NoError(t, err)
+	_, err = p.Provision(context.Background(), &provisioners.Input{
+		ResourceUid: "thing.default#w.r",
+	})
+	require.EqualError(t, err, "failed to decode output from cmd provisioner: invalid character 'b' looking for beginning of value")
+}