Repository: buchgr/bazel-remote
Pull request #492 (2021-10-27 22:15:01)

Add support for retrieving s3 credentials from AWS credentials file
This pull request adds support for retrieving credentials from AWS credentials files (often used by [aws-cli](https://aws.amazon.com/cli/)).

As a regular user of the aws-cli, this change allows bazel-remote to leverage credentials I've already got (access key, secret key, session tokens, etc) without having to goof around with anything else. Note, this is something that minio-go already supports, I am just wiring things up.

I took the opportunity move the credentials parsing out to a `GetCredentials` function. I felt uncomfortable adding 3 more arguments to `s3proxy.New()`, which was already very long.



Patch:
diff --git a/README.md b/README.md
index f63084d..4a84b94 100644
--- a/README.md
+++ b/README.md
@@ -232,11 +232,28 @@ OPTIONS:
    --s3.prefix value The S3/minio object prefix to use when using S3 proxy
       backend. [$BAZEL_REMOTE_S3_PREFIX]
 
+   --s3.auth_method value The S3/minio authentication method. This argument
+      is required when an s3 proxy backend is used. Allowed values: iam_role,
+      access_key, aws_credentials_file. [$BAZEL_REMOTE_S3_AUTH_METHOD]
+
    --s3.access_key_id value The S3/minio access key to use when using S3
-      proxy backend. [$BAZEL_REMOTE_S3_ACCESS_KEY_ID]
+      proxy backend. Applies to s3 auth method(s): access_key.
+      [$BAZEL_REMOTE_S3_ACCESS_KEY_ID]
 
    --s3.secret_access_key value The S3/minio secret access key to use when
-      using S3 proxy backend. [$BAZEL_REMOTE_S3_SECRET_ACCESS_KEY]
+      using S3 proxy backend. Applies to s3 auth method(s): access_key.
+      [$BAZEL_REMOTE_S3_SECRET_ACCESS_KEY]
+
+   --s3.aws_shared_credentials_file value Path to the AWS credentials file.
+      If not specified, the minio client will default to '~/.aws/credentials'.
+      Applies to s3 auth method(s): aws_credentials_file.
+      [$BAZEL_REMOTE_S3_AWS_SHARED_CREDENTIALS_FILE,
+      $AWS_SHARED_CREDENTIALS_FILE]
+
+   --s3.aws_profile value The aws credentials profile to use from within
+      s3.aws_shared_credentials_file. Applies to s3 auth method(s):
+      aws_credentials_file. (default: "default") [$BAZEL_REMOTE_S3_AWS_PROFILE,
+      $AWS_PROFILE]
 
    --s3.disable_ssl Whether to disable TLS/SSL when using the S3 proxy
       backend. (default: false, ie enable TLS/SSL)
@@ -244,7 +261,8 @@ OPTIONS:
 
    --s3.iam_role_endpoint value Endpoint for using IAM security credentials.
       By default it will look for credentials in the standard locations for the
-      AWS platform. [$BAZEL_REMOTE_S3_IAM_ROLE_ENDPOINT]
+      AWS platform. Applies to s3 auth method(s): iam_role.
+      [$BAZEL_REMOTE_S3_IAM_ROLE_ENDPOINT]
 
    --s3.region value The AWS region. Required when not specifying S3/minio
       access keys. [$BAZEL_REMOTE_S3_REGION]
@@ -356,14 +374,24 @@ host: localhost
 #  endpoint: minio.example.com:9000
 #  bucket: test-bucket
 #  prefix: test-prefix
+#  disable_ssl: true
+#
+# Provide exactly one auth_method (access_key, iam_role, or credentials_file) and accompanying configuration.
+#
+# Access key authenticaiton:
+#  auth_method: access_key
 #  access_key_id: EXAMPLE_ACCESS_KEY
 #  secret_access_key: EXAMPLE_SECRET_KEY
-#  disable_ssl: true
 #
-# Provide either access_key_id/secret_access_key, or iam_role_endpoint/region.
-# iam_role_endpoint can also be left empty, and figured out automatically.
+# IAM Role authentication.
+#  auth_method: iam_role
 #  iam_role_endpoint: http://169.254.169.254
 #  region: us-east-1
+# 
+# AWS credentials file.
+#  auth_method: credentials_file
+#  aws_shared_credentials_file: path/to/aws/credentials
+#  aws_profile: my-profile
 #
 #http_proxy:
 #  url: https://remote-cache.com:8080/cache
diff --git a/cache/s3proxy/BUILD.bazel b/cache/s3proxy/BUILD.bazel
index 7635826..e56d204 100644
--- a/cache/s3proxy/BUILD.bazel
+++ b/cache/s3proxy/BUILD.bazel
@@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
-    srcs = ["s3proxy.go"],
+    srcs = [
+        "auth_methods.go",
+        "s3proxy.go",
+    ],
     importpath = "github.com/buchgr/bazel-remote/cache/s3proxy",
     visibility = ["//visibility:public"],
     deps = [
diff --git a/cache/s3proxy/auth_methods.go b/cache/s3proxy/auth_methods.go
new file mode 100644
index 0000000..ed95582
--- /dev/null
+++ b/cache/s3proxy/auth_methods.go
@@ -0,0 +1,24 @@
+package s3proxy
+
+const (
+	AuthMethodIAMRole            = "iam_role"
+	AuthMethodAccessKey          = "access_key"
+	AuthMethodAWSCredentialsFile = "aws_credentials_file"
+)
+
+func GetAuthMethods() []string {
+	return []string{
+		AuthMethodIAMRole,
+		AuthMethodAccessKey,
+		AuthMethodAWSCredentialsFile,
+	}
+}
+
+func IsValidAuthMethod(authMethod string) bool {
+	for _, b := range GetAuthMethods() {
+		if authMethod == b {
+			return true
+		}
+	}
+	return false
+}
diff --git a/cache/s3proxy/s3proxy.go b/cache/s3proxy/s3proxy.go
index 440b8eb..c3d2850 100644
--- a/cache/s3proxy/s3proxy.go
+++ b/cache/s3proxy/s3proxy.go
@@ -54,10 +54,8 @@ func New(
 	Endpoint string,
 	Bucket string,
 	Prefix string,
-	AccessKeyID string,
-	SecretAccessKey string,
+	Credentials *credentials.Credentials,
 	DisableSSL bool,
-	IAMRoleEndpoint string,
 	Region string,
 
 	storageMode string, accessLogger cache.Logger,
@@ -68,38 +66,20 @@ func New(
 	var minioCore *minio.Core
 	var err error
 
-	if AccessKeyID != "" && SecretAccessKey != "" {
-		// Initialize minio client object.
-		opts := &minio.Options{
-			Creds:  credentials.NewStaticV4(AccessKeyID, SecretAccessKey, ""),
-			Secure: !DisableSSL,
-			Region: Region,
-		}
-		minioCore, err = minio.NewCore(Endpoint, opts)
-		if err != nil {
-			log.Fatalln(err)
-		}
-	} else {
-		// Initialize minio client object with IAM credentials
-		opts := &minio.Options{
-			// This config value may be empty.
-			Creds: credentials.NewIAM(IAMRoleEndpoint),
-
-			Region: Region,
-			Secure: !DisableSSL,
-		}
+	if Credentials == nil {
+		log.Fatalf("Failed to determine s3proxy credentials")
+	}
 
-		minioClient, err := minio.New(
-			Endpoint,
-			opts,
-		)
-		if err != nil {
-			log.Fatalln(err)
-		}
+	// Initialize minio client with credentials
+	opts := &minio.Options{
+		Creds: Credentials,
 
-		minioCore = &minio.Core{
-			Client: minioClient,
-		}
+		Region: Region,
+		Secure: !DisableSSL,
+	}
+	minioCore, err = minio.NewCore(Endpoint, opts)
+	if err != nil {
+		log.Fatalln(err)
 	}
 
 	if storageMode != "zstd" && storageMode != "uncompressed" {
diff --git a/config/BUILD.bazel b/config/BUILD.bazel
index ff3f13b..6e50b2d 100644
--- a/config/BUILD.bazel
+++ b/config/BUILD.bazel
@@ -6,6 +6,7 @@ go_library(
         "config.go",
         "logger.go",
         "proxy.go",
+        "s3.go",
         "tls.go",
     ],
     importpath = "github.com/buchgr/bazel-remote/config",
@@ -15,6 +16,7 @@ go_library(
         "//cache/gcsproxy:go_default_library",
         "//cache/httpproxy:go_default_library",
         "//cache/s3proxy:go_default_library",
+        "@com_github_minio_minio_go_v7//pkg/credentials:go_default_library",
         "@com_github_urfave_cli_v2//:go_default_library",
         "@in_gopkg_yaml_v2//:go_default_library",
     ],
diff --git a/config/config.go b/config/config.go
index 298172e..17b8638 100644
--- a/config/config.go
+++ b/config/config.go
@@ -12,24 +12,12 @@ import (
 	"time"
 
 	"github.com/buchgr/bazel-remote/cache"
+	"github.com/buchgr/bazel-remote/cache/s3proxy"
 
 	"github.com/urfave/cli/v2"
 	yaml "gopkg.in/yaml.v2"
 )
 
-// S3CloudStorageConfig stores the configuration of an S3 API proxy backend.
-type S3CloudStorageConfig struct {
-	Endpoint        string `yaml:"endpoint"`
-	Bucket          string `yaml:"bucket"`
-	Prefix          string `yaml:"prefix"`
-	AccessKeyID     string `yaml:"access_key_id"`
-	SecretAccessKey string `yaml:"secret_access_key"`
-	DisableSSL      bool   `yaml:"disable_ssl"`
-	IAMRoleEndpoint string `yaml:"iam_role_endpoint"`
-	Region          string `yaml:"region"`
-	KeyVersion      *int   `yaml:"key_version"`
-}
-
 // GoogleCloudStorageConfig stores the configuration of a GCS proxy backend.
 type GoogleCloudStorageConfig struct {
 	Bucket                string `yaml:"bucket"`
@@ -269,8 +257,8 @@ func validateConfig(c *Config) error {
 	}
 
 	if c.S3CloudStorage != nil {
-		if c.S3CloudStorage.AccessKeyID != "" && c.S3CloudStorage.IAMRoleEndpoint != "" {
-			return errors.New("Expected either 's3.access_key_id' or 's3.iam_role_endpoint', found both")
+		if !s3proxy.IsValidAuthMethod(c.S3CloudStorage.AuthMethod) {
+			return fmt.Errorf("invalid s3.auth_method: %s", c.S3CloudStorage.AuthMethod)
 		}
 
 		if c.S3CloudStorage.KeyVersion != nil && *c.S3CloudStorage.KeyVersion != 2 {
@@ -335,14 +323,17 @@ func get(ctx *cli.Context) (*Config, error) {
 	var s3 *S3CloudStorageConfig
 	if ctx.String("s3.bucket") != "" {
 		s3 = &S3CloudStorageConfig{
-			Endpoint:        ctx.String("s3.endpoint"),
-			Bucket:          ctx.String("s3.bucket"),
-			Prefix:          ctx.String("s3.prefix"),
-			AccessKeyID:     ctx.String("s3.access_key_id"),
-			SecretAccessKey: ctx.String("s3.secret_access_key"),
-			DisableSSL:      ctx.Bool("s3.disable_ssl"),
-			IAMRoleEndpoint: ctx.String("s3.iam_role_endpoint"),
-			Region:          ctx.String("s3.region"),
+			Endpoint:                 ctx.String("s3.endpoint"),
+			Bucket:                   ctx.String("s3.bucket"),
+			Prefix:                   ctx.String("s3.prefix"),
+			AuthMethod:               ctx.String("s3.auth_method"),
+			AccessKeyID:              ctx.String("s3.access_key_id"),
+			SecretAccessKey:          ctx.String("s3.secret_access_key"),
+			DisableSSL:               ctx.Bool("s3.disable_ssl"),
+			IAMRoleEndpoint:          ctx.String("s3.iam_role_endpoint"),
+			Region:                   ctx.String("s3.region"),
+			AWSProfile:               ctx.String("s3.aws_profile"),
+			AWSSharedCredentialsFile: ctx.String("s3.aws_shared_credentials_file"),
 		}
 	}
 
diff --git a/config/proxy.go b/config/proxy.go
index 361f59f..7d3de2f 100644
--- a/config/proxy.go
+++ b/config/proxy.go
@@ -40,14 +40,17 @@ func (c *Config) setProxy() error {
 	}
 
 	if c.S3CloudStorage != nil {
+		creds, err := c.S3CloudStorage.GetCredentials()
+		if err != nil {
+			return err
+		}
+
 		c.ProxyBackend = s3proxy.New(
 			c.S3CloudStorage.Endpoint,
 			c.S3CloudStorage.Bucket,
 			c.S3CloudStorage.Prefix,
-			c.S3CloudStorage.AccessKeyID,
-			c.S3CloudStorage.SecretAccessKey,
+			creds,
 			c.S3CloudStorage.DisableSSL,
-			c.S3CloudStorage.IAMRoleEndpoint,
 			c.S3CloudStorage.Region,
 			c.StorageMode, c.AccessLogger, c.ErrorLogger, c.NumUploaders, c.MaxQueuedUploads)
 		return nil
diff --git a/config/s3.go b/config/s3.go
new file mode 100644
index 0000000..323a516
--- /dev/null
+++ b/config/s3.go
@@ -0,0 +1,47 @@
+package config
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/buchgr/bazel-remote/cache/s3proxy"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+// S3CloudStorageConfig stores the configuration of an S3 API proxy backend.
+type S3CloudStorageConfig struct {
+	Endpoint                 string `yaml:"endpoint"`
+	Bucket                   string `yaml:"bucket"`
+	Prefix                   string `yaml:"prefix"`
+	AuthMethod               string `yaml:"auth_method"`
+	AccessKeyID              string `yaml:"access_key_id"`
+	SecretAccessKey          string `yaml:"secret_access_key"`
+	DisableSSL               bool   `yaml:"disable_ssl"`
+	IAMRoleEndpoint          string `yaml:"iam_role_endpoint"`
+	Region                   string `yaml:"region"`
+	KeyVersion               *int   `yaml:"key_version"`
+	AWSProfile               string `yaml:"aws_profile"`
+	AWSSharedCredentialsFile string `yaml:"aws_shared_credentials_file"`
+}
+
+func (s3c S3CloudStorageConfig) GetCredentials() (*credentials.Credentials, error) {
+	if s3c.AuthMethod == s3proxy.AuthMethodAWSCredentialsFile {
+		log.Println("S3 Credentials: using AWS credentials file.")
+		return credentials.NewFileAWSCredentials(s3c.AWSSharedCredentialsFile, s3c.AWSProfile), nil
+	} else if s3c.AuthMethod == s3proxy.AuthMethodAccessKey {
+		if s3c.AccessKeyID == "" {
+			return nil, fmt.Errorf("missing s3.access_key_id for s3.auth_method = '%s'", s3proxy.AuthMethodAccessKey)
+		}
+		if s3c.SecretAccessKey == "" {
+			return nil, fmt.Errorf("missing s3.secret_access_key for s3.auth_method = '%s'", s3proxy.AuthMethodAccessKey)
+		}
+		log.Println("S3 Credentials: using access/secret access key.")
+		return credentials.NewStaticV4(s3c.AccessKeyID, s3c.SecretAccessKey, ""), nil
+	} else if s3c.AuthMethod == s3proxy.AuthMethodIAMRole {
+		// Fall back to getting credentials from IAM
+		log.Println("S3 Credentials: using IAM.")
+		return credentials.NewIAM(s3c.IAMRoleEndpoint), nil
+	}
+
+	return nil, fmt.Errorf("invalid s3.auth_method: %s", s3c.AuthMethod)
+}
diff --git a/utils/flags/BUILD.bazel b/utils/flags/BUILD.bazel
index eb2a15c..9f2edd2 100644
--- a/utils/flags/BUILD.bazel
+++ b/utils/flags/BUILD.bazel
@@ -8,7 +8,10 @@ go_library(
     ],
     importpath = "github.com/buchgr/bazel-remote/utils/flags",
     visibility = ["//visibility:public"],
-    deps = ["@com_github_urfave_cli_v2//:go_default_library"],
+    deps = [
+        "//cache/s3proxy:go_default_library",
+        "@com_github_urfave_cli_v2//:go_default_library",
+    ],
 )
 
 go_test(
diff --git a/utils/flags/flags.go b/utils/flags/flags.go
index 1896545..09161ee 100644
--- a/utils/flags/flags.go
+++ b/utils/flags/flags.go
@@ -1,11 +1,19 @@
 package flags
 
 import (
-	"github.com/urfave/cli/v2"
+	"fmt"
 	"math"
 	"strconv"
+	"strings"
+
+	"github.com/buchgr/bazel-remote/cache/s3proxy"
+	"github.com/urfave/cli/v2"
 )
 
+func s3AuthMsg(authMethods ...string) string {
+	return fmt.Sprintf("Applies to s3 auth method(s): %s.", strings.Join(authMethods, ", "))
+}
+
 // GetCliFlags returns a slice of cli.Flag's that bazel-remote accepts.
 func GetCliFlags() []cli.Flag {
 	return []cli.Flag{
@@ -178,18 +186,36 @@ func GetCliFlags() []cli.Flag {
 			Usage:   "The S3/minio object prefix to use when using S3 proxy backend.",
 			EnvVars: []string{"BAZEL_REMOTE_S3_PREFIX"},
 		},
+		&cli.StringFlag{
+			Name:    "s3.auth_method",
+			Value:   "",
+			Usage:   fmt.Sprintf("The S3/minio authentication method. This argument is required when an s3 proxy backend is used. Allowed values: %s.", strings.Join(s3proxy.GetAuthMethods(), ", ")),
+			EnvVars: []string{"BAZEL_REMOTE_S3_AUTH_METHOD"},
+		},
 		&cli.StringFlag{
 			Name:    "s3.access_key_id",
 			Value:   "",
-			Usage:   "The S3/minio access key to use when using S3 proxy backend.",
+			Usage:   "The S3/minio access key to use when using S3 proxy backend. " + s3AuthMsg(s3proxy.AuthMethodAccessKey),
 			EnvVars: []string{"BAZEL_REMOTE_S3_ACCESS_KEY_ID"},
 		},
 		&cli.StringFlag{
 			Name:    "s3.secret_access_key",
 			Value:   "",
-			Usage:   "The S3/minio secret access key to use when using S3 proxy backend.",
+			Usage:   "The S3/minio secret access key to use when using S3 proxy backend. " + s3AuthMsg(s3proxy.AuthMethodAccessKey),
 			EnvVars: []string{"BAZEL_REMOTE_S3_SECRET_ACCESS_KEY"},
 		},
+		&cli.StringFlag{
+			Name:    "s3.aws_shared_credentials_file",
+			Value:   "",
+			Usage:   "Path to the AWS credentials file. If not specified, the minio client will default to '~/.aws/credentials'. " + s3AuthMsg(s3proxy.AuthMethodAWSCredentialsFile),
+			EnvVars: []string{"BAZEL_REMOTE_S3_AWS_SHARED_CREDENTIALS_FILE", "AWS_SHARED_CREDENTIALS_FILE"},
+		},
+		&cli.StringFlag{
+			Name:    "s3.aws_profile",
+			Value:   "default",
+			Usage:   "The aws credentials profile to use from within s3.aws_shared_credentials_file. " + s3AuthMsg(s3proxy.AuthMethodAWSCredentialsFile),
+			EnvVars: []string{"BAZEL_REMOTE_S3_AWS_PROFILE", "AWS_PROFILE"},
+		},
 		&cli.BoolFlag{
 			Name:        "s3.disable_ssl",
 			Usage:       "Whether to disable TLS/SSL when using the S3 proxy backend.",
@@ -199,7 +225,7 @@ func GetCliFlags() []cli.Flag {
 		&cli.StringFlag{
 			Name:    "s3.iam_role_endpoint",
 			Value:   "",
-			Usage:   "Endpoint for using IAM security credentials. By default it will look for credentials in the standard locations for the AWS platform.",
+			Usage:   "Endpoint for using IAM security credentials. By default it will look for credentials in the standard locations for the AWS platform. " + s3AuthMsg(s3proxy.AuthMethodIAMRole),
 			EnvVars: []string{"BAZEL_REMOTE_S3_IAM_ROLE_ENDPOINT"},
 		},
 		&cli.StringFlag{


Test patch:
diff --git a/.bazelci/system-test.sh b/.bazelci/system-test.sh
index 2788aee..222cfc1 100755
--- a/.bazelci/system-test.sh
+++ b/.bazelci/system-test.sh
@@ -78,6 +78,7 @@ rm -rf $test_cache_dir
 	--s3.endpoint 127.0.0.1:9000 \
 	--s3.bucket bazel-remote \
 	--s3.prefix files \
+	--s3.auth_method access_key \
 	--s3.access_key_id minioadmin \
 	--s3.secret_access_key minioadmin \
 	--s3.disable_ssl \
@@ -134,6 +135,7 @@ rm -rf $test_cache_dir
 	--s3.endpoint 127.0.0.1:9000 \
 	--s3.bucket bazel-remote \
 	--s3.prefix files \
+	--s3.auth_method access_key \
 	--s3.access_key_id minioadmin \
 	--s3.secret_access_key minioadmin \
 	--s3.disable_ssl \
@@ -211,6 +213,7 @@ rm -rf $test_cache_dir
 	--s3.endpoint 127.0.0.1:9000 \
 	--s3.bucket bazel-remote \
 	--s3.prefix files \
+	--s3.auth_method access_key \
 	--s3.access_key_id minioadmin \
 	--s3.secret_access_key minioadmin \
 	--s3.disable_ssl \
diff --git a/cache/s3proxy/minio-test-setup.md b/cache/s3proxy/minio-test-setup.md
index c323bff..e2ca1c4 100644
--- a/cache/s3proxy/minio-test-setup.md
+++ b/cache/s3proxy/minio-test-setup.md
@@ -26,6 +26,7 @@ production use.
     	--s3.endpoint 127.0.0.1:9000 \
     	--s3.bucket bazel-remote \
     	--s3.prefix files \
+    	--s3.auth_method access_key \
     	--s3.access_key_id minioadmin \
     	--s3.secret_access_key minioadmin \
     	--s3.disable_ssl
diff --git a/config/config_test.go b/config/config_test.go
index 727fc34..eb44fbd 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -179,6 +179,7 @@ s3_proxy:
   endpoint: minio.example.com:9000
   bucket: test-bucket
   prefix: test-prefix
+  auth_method: access_key
   access_key_id: EXAMPLE_ACCESS_KEY
   secret_access_key: EXAMPLE_SECRET_KEY
 `
@@ -197,6 +198,7 @@ s3_proxy:
 			Endpoint:        "minio.example.com:9000",
 			Bucket:          "test-bucket",
 			Prefix:          "test-prefix",
+			AuthMethod:      "access_key",
 			AccessKeyID:     "EXAMPLE_ACCESS_KEY",
 			SecretAccessKey: "EXAMPLE_SECRET_KEY",
 		},