Repository: score-spec/score-compose
Pull request #72 (2024-03-12 15:25:26)

feat: added minio-based s3 provisioner
See demo video here in Loom https://www.loom.com/share/f51c30a60e38477fa7dbf1b83e316c42?sid=19960a22-0197-4e5a-8370-3700d1d95f97.

This gives an S3 compatible resource that can be used to build applications talking to object stores. Since the S3 API is generally a universal standard now, this is a great API use when building apps that needs some object storage.

The outputs are `endpoint`, `bucket`, `access_key_id` and `secret_key` which are the standard outputs.

This will be documented in the combined docs push for project Melody.


Patch:
diff --git a/README.md b/README.md
index 2f04e50..3c492df 100644
--- a/README.md
+++ b/README.md
@@ -27,13 +27,14 @@ The `score.yaml` specification file can be executed against a _Score Implementat
 
 `score-compose` comes with out-of-the-box support for:
 
-| Type        | Class   | Params | Output                                                                                      |
-|-------------|---------|--------|---------------------------------------------------------------------------------------------|
-| environment | default | (none) | `${KEY}`                                                                                    |
-| service     | !       | !      | Not supported yet, tracked in [#87](https://github.com/score-spec/score-compose/issues/87). |
-| volume      | *       | (none) | `source`                                                                                    |
-| redis       | *       | (none) | `host`, `port`, `username`, `password`                                                      | 
-| postgres    | *       | (none) | `host`, `port`, `name` (aka `database`), `username`, `password`                             |
+| Type        | Class   | Params | Output                                                                                                                                                          |
+|-------------|---------|--------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| environment | default | (none) | `${KEY}`                                                                                                                                                        |
+| service     | !       | !      | Not supported yet, tracked in [#87](https://github.com/score-spec/score-compose/issues/87).                                                                     |
+| volume      | *       | (none) | `source`                                                                                                                                                        |
+| redis       | *       | (none) | `host`, `port`, `username`, `password`                                                                                                                          | 
+| postgres    | *       | (none) | `host`, `port`, `name` (aka `database`), `username`, `password`                                                                                                 |
+| s3          | *       | (none) | `endpoint`, `access_key_id`, `secret_key`, `bucket`, with `region=""`, `aws_access_key_id=<access_key_id>`, and `aws_secret_key=<secret_key>` for compatibility |
 
 ## ![Installation](docs/images/install.svg) Installation
 
diff --git a/go.mod b/go.mod
index 5ef902a..6831360 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	github.com/compose-spec/compose-go/v2 v2.0.0-rc.8
 	github.com/imdario/mergo v0.3.13
 	github.com/mitchellh/mapstructure v1.5.0
-	github.com/score-spec/score-go v1.2.0
+	github.com/score-spec/score-go v1.3.0
 	github.com/spf13/cobra v1.6.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.4
diff --git a/go.sum b/go.sum
index 5a92275..7c746a7 100644
--- a/go.sum
+++ b/go.sum
@@ -49,8 +49,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
-github.com/score-spec/score-go v1.2.0 h1:1fEJXoK3YtL8Ko7RrY5lSfx80/Qs19zM/NJMuG9jlYA=
-github.com/score-spec/score-go v1.2.0/go.mod h1:nt6TOq2Ld9SiH3Fd9NF8tiJ9L7S17OE3FNgCrSet5GQ=
+github.com/score-spec/score-go v1.3.0 h1:3RM2tkeyA4HDHBtxxj+2l/t98gpRmMNyXuFe9bGbs+A=
+github.com/score-spec/score-go v1.3.0/go.mod h1:nt6TOq2Ld9SiH3Fd9NF8tiJ9L7S17OE3FNgCrSet5GQ=
 github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
diff --git a/internal/command/default.provisioners.yaml b/internal/command/default.provisioners.yaml
index e9a2216..40669c0 100644
--- a/internal/command/default.provisioners.yaml
+++ b/internal/command/default.provisioners.yaml
@@ -181,3 +181,97 @@
     {{ if ne .Init.publishPort "0" }}
     - "Or connect your postgres client to \"postgres://{{ .State.username }}:{{ .State.password }}@localhost:{{ .Init.publishPort }}/{{ .State.database }}\""
     {{ end }}
+
+# This resource provides a minio based S3 bucket with AWS-style credentials.
+# This provides some common and well known outputs that can be used with any generic AWS s3 client.
+# If the provider has a publish port annotation, it can expose a management port on the local network for debugging and
+# connectivity.
+- uri: template://default-provisioners/s3
+  type: s3
+  # The init template contains some initial seed data that can be used it needed.
+  init: |
+    randomServiceName: minio-{{ randAlphaNum 6 }}
+    randomUsername: user-{{ randAlpha 8 }}
+    randomPassword: {{ randAlphaNum 16 | quote }}
+    randomBucket: bucket-{{ randAlpha 8 | lower }}-{{ .Id | lower | trunc 47 }}
+    randomAccessKeyId: {{ randAlphaNum 20 | quote }}
+    randomSecretKey: {{ randAlphaNum 40 | quote }}
+    sk: default-provisioners-minio-instance
+    publishPort: {{ dig "annotations" "compose.score.dev/publish-port" "0" .Metadata | atoi }}
+  # The only instance state is the bucket name, for now we provision a single aws key across s3 resources.
+  state: |
+    bucket: {{ dig "bucket" .Init.randomBucket .State | quote }}
+  # The shared state contains the chosen service name and credentials
+  shared: |
+    {{ .Init.sk }}:
+      instanceServiceName: {{ dig .Init.sk "instanceServiceName" .Init.randomServiceName .Shared | quote }}
+      instanceUsername: {{ dig .Init.sk "instanceUsername" .Init.randomUsername .Shared | quote }}
+      instancePassword: {{ dig .Init.sk "instancePassword" .Init.randomPassword .Shared | quote }}
+      instanceAccessKeyId: {{ dig .Init.sk "instanceAccessKeyId" .Init.randomAccessKeyId .Shared | quote }}
+      instanceSecretKey: {{ dig .Init.sk "instanceSecretKey" .Init.randomSecretKey .Shared | quote }}
+      publishPort: {{ with (dig .Init.sk "publishPort" 0 .Shared) }}{{ if ne . 0 }}{{ . }}{{ else }}{{ $.Init.publishPort }}{{ end }}{{ end }}
+  # the outputs that we can expose
+  outputs: |
+    bucket: {{ .State.bucket }}
+    access_key_id: {{ dig .Init.sk "instanceAccessKeyId" "" .Shared | quote }}
+    secret_key: {{ dig .Init.sk "instanceSecretKey" "" .Shared | quote }}
+    endpoint: http://{{ dig .Init.sk "instanceServiceName" "" .Shared }}:9000
+    # for compatibility with Humanitec's existing s3 resource
+    region: ""
+    aws_access_key_id: {{ dig .Init.sk "instanceAccessKeyId" "" .Shared | quote }}
+    aws_secret_key: {{ dig .Init.sk "instanceSecretKey" "" .Shared | quote }}
+  # we store 2 files, 1 is always the same and overridden, the other is per bucket
+  files: |
+    {{ dig .Init.sk "instanceServiceName" "" .Shared }}-setup-scripts/00-svcacct.sh: |
+      mc alias set myminio http://{{ dig .Init.sk "instanceServiceName" "" .Shared }}:9000 {{ dig .Init.sk "instanceUsername" "" .Shared }} {{ dig .Init.sk "instancePassword" "" .Shared }}
+      mc admin user svcacct info myminio {{ dig .Init.sk "instanceAccessKeyId" "" .Shared | quote }} || mc admin user svcacct add myminio {{ dig .Init.sk "instanceUsername" "" .Shared | quote }} --access-key {{ dig .Init.sk "instanceAccessKeyId" "" .Shared | quote }} --secret-key {{ dig .Init.sk "instanceSecretKey" "" .Shared | quote }}
+    {{ dig .Init.sk "instanceServiceName" "" .Shared }}-setup-scripts/10-bucket-{{ .State.bucket }}.sh: |
+      mc alias set myminio http://{{ dig .Init.sk "instanceServiceName" "" .Shared }}:9000 {{ dig .Init.sk "instanceUsername" "" .Shared }} {{ dig .Init.sk "instancePassword" "" .Shared }}
+      mc mb -p myminio/{{ .State.bucket }}
+  volumes: |
+    {{ dig .Init.sk "instanceServiceName" "" .Shared }}-data:
+      driver: local
+  # 2 services, the minio one, and the init container which ensures the service account and buckets exist
+  services: |
+    {{ dig .Init.sk "instanceServiceName" "" .Shared }}:
+      image: quay.io/minio/minio
+      command: ["server", "/data", "--console-address", ":9001"]
+      restart: always
+      {{ if ne .Init.publishPort 0 }}
+      ports:
+      - target: 9001
+        published: {{ .Init.publishPort }}
+      {{ end }}
+      healthcheck:
+        test: ["CMD-SHELL", "mc alias set myminio http://localhost:9000 {{ dig .Init.sk "instanceUsername" "" .Shared }} {{ dig .Init.sk "instancePassword" "" .Shared }}"]
+        interval: 2s
+        timeout: 2s
+        retries: 10
+      environment:
+        MINIO_ROOT_USER: {{ dig .Init.sk "instanceUsername" "" .Shared | quote }}
+        MINIO_ROOT_PASSWORD: {{ dig .Init.sk "instancePassword" "" .Shared | quote }}
+      volumes:
+      - type: volume
+        source: {{ dig .Init.sk "instanceServiceName" "" .Shared }}-data
+        target: /data
+    {{ dig .Init.sk "instanceServiceName" "" .Shared }}-init:
+      image: quay.io/minio/minio
+      entrypoint: ["/bin/sh"]
+      command:
+      - "-c"
+      - "for s in $$(ls /setup-scripts -1); do sh /setup-scripts/$$s; done"
+      labels:
+        dev.score.compose.labels.is-init-container: "true"
+      depends_on:
+        {{ dig .Init.sk "instanceServiceName" "" .Shared }}:
+          condition: service_healthy
+          restart: true
+      volumes:
+      - type: bind
+        source: {{ .MountsDirectory }}/{{ dig .Init.sk "instanceServiceName" "" .Shared }}-setup-scripts
+        target: /setup-scripts
+  info_logs: |
+    - "To connect with a minio client: use the myminio alias at \"docker run -it --network {{ .ComposeProjectName }}_default --rm --entrypoint /bin/bash quay.io/minio/minio -c 'mc alias set myminio http://{{ dig .Init.sk "instanceServiceName" "" .Shared }}:9000 {{ dig .Init.sk "instanceAccessKeyId" "" .Shared }} {{ dig .Init.sk "instanceSecretKey" "" .Shared }}; bash'\""
+    {{ if ne .Init.publishPort 0 }}
+    - "Or enter {{ dig .Init.sk "instanceUsername" "" .Shared }} / {{ dig .Init.sk "instancePassword" "" .Shared }} at https://localhost:{{ .Init.publishPort }}"
+    {{ end }}


Test patch:
diff --git a/internal/command/generate_test.go b/internal/command/generate_test.go
index 147c16d..5399fef 100644
--- a/internal/command/generate_test.go
+++ b/internal/command/generate_test.go
@@ -404,6 +404,59 @@ resources:
 	})
 }
 
+func TestGenerateS3Resource(t *testing.T) {
+	td := changeToTempDir(t)
+	stdout, _, err := executeAndResetCommand(context.Background(), rootCmd, []string{"init"})
+	assert.NoError(t, err)
+	assert.Equal(t, "", stdout)
+	assert.NoError(t, os.WriteFile(filepath.Join(td, "score.yaml"), []byte(`
+apiVersion: score.dev/v1b1
+metadata:
+  name: example
+containers:
+  example:
+    image: foo
+    variables:
+      output: ${resources.bucket1.endpoint} ${resources.bucket1.region} ${resources.bucket1.bucket} ${resources.bucket1.access_key_id} ${resources.bucket1.secret_key}
+resources:
+  bucket1:
+    metadata:
+      annotations:
+        compose.score.dev/publish-port: "9001"
+    type: s3
+  bucket2:
+    type: s3
+`), 0644))
+	stdout, _, err = executeAndResetCommand(context.Background(), rootCmd, []string{"generate", "score.yaml"})
+	assert.NoError(t, err)
+	assert.Equal(t, "", stdout)
+
+	// check that state was persisted
+	sd, ok, err := project.LoadStateDirectory(td)
+	assert.NoError(t, err)
+	assert.True(t, ok)
+	assert.Len(t, sd.State.Workloads, 1)
+	assert.Len(t, sd.State.Resources, 2)
+	assert.Contains(t, sd.State.Resources["s3.default#example.bucket1"].State, "bucket")
+	assert.Contains(t, sd.State.Resources["s3.default#example.bucket2"].State, "bucket")
+	assert.NotEqual(t, sd.State.Resources["s3.default#example.bucket1"].State, sd.State.Resources["postgres.default#example.bucket2"].State)
+	assert.Contains(t, sd.State.SharedState, "default-provisioners-minio-instance")
+
+	t.Run("validate compose spec", func(t *testing.T) {
+		if os.Getenv("NO_DOCKER") != "" {
+			t.Skip("NO_DOCKER is set")
+			return
+		}
+		dockerCmd, err := exec.LookPath("docker")
+		require.NoError(t, err)
+		cmd := exec.Command(dockerCmd, "compose", "-f", "compose.yaml", "convert", "--quiet", "--dry-run")
+		cmd.Dir = td
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		assert.NoError(t, cmd.Run())
+	})
+}
+
 func TestInitAndGenerate_with_depends_on(t *testing.T) {
 	td := changeToTempDir(t)
 	stdout, _, err := executeAndResetCommand(context.Background(), rootCmd, []string{"init"})