Repository: docker/compose-cli
Pull request #795 (2020-10-16 08:58:25)

Store external cluster as metadata
**What I did**
1. add a sanity check to sdk.DescribeService
2. store external cluster as template metadata so we can get this information back

DescribeService API, even we pass a service ARN, require cluster ID (?)
If cluster was set by user as `x-aws-cluster`, we can't get this ID from CloudFormation stack resources
Proposed change uses CloudFormation template Metadata to track the user-provided cluster so we can easily get this ID back on `compose ps`

**Related issue**
https://github.com/docker/compose-cli/issues/790

/test-ecs

**(not mandatory) A picture of a cute animal, if possible in relation with what you did**
![image](https://user-images.githubusercontent.com/132757/96238237-8d9c1d00-0f9e-11eb-8317-a52977e71881.png)





Patch:
diff --git a/aci/convert/convert.go b/aci/convert/convert.go
index 9afabb10..1119ae8b 100644
--- a/aci/convert/convert.go
+++ b/aci/convert/convert.go
@@ -23,7 +23,6 @@ import (
 	"io/ioutil"
 	"math"
 	"os"
-	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -51,8 +50,7 @@ const (
 	volumeDriveroptsAccountNameKey = "storage_account_name"
 	volumeReadOnly                 = "read_only"
 
-	defaultSecretsPath         = "/run/secrets"
-	serviceSecretAbsPathPrefix = "aci-service-secret-path-"
+	serviceSecretPrefix = "aci-service-secret-"
 )
 
 // ToContainerGroup converts a compose project into a ACI container group
@@ -190,15 +188,13 @@ func getDNSSidecar(containers []containerinstance.Container) containerinstance.C
 
 type projectAciHelper types.Project
 
-func getServiceSecretKey(serviceName, targetDir string) string {
-	return fmt.Sprintf("%s-%s--%s",
-		serviceSecretAbsPathPrefix, serviceName, strings.ReplaceAll(targetDir, "/", "-"))
-}
-
 func (p projectAciHelper) getAciSecretVolumes() ([]containerinstance.Volume, error) {
 	var secretVolumes []containerinstance.Volume
 	for _, svc := range p.Services {
-		squashedTargetVolumes := make(map[string]containerinstance.Volume)
+		secretServiceVolume := containerinstance.Volume{
+			Name:   to.StringPtr(serviceSecretPrefix + svc.Name),
+			Secret: make(map[string]*string),
+		}
 		for _, scr := range svc.Secrets {
 			data, err := ioutil.ReadFile(p.Secrets[scr.Source].File)
 			if err != nil {
@@ -211,31 +207,14 @@ func (p projectAciHelper) getAciSecretVolumes() ([]containerinstance.Volume, err
 			if scr.Target == "" {
 				scr.Target = scr.Source
 			}
-
-			if !filepath.IsAbs(scr.Target) && strings.ContainsAny(scr.Target, "\\/") {
+			if strings.ContainsAny(scr.Target, "\\/") {
 				return []containerinstance.Volume{},
-					errors.Errorf("in service %q, secret with source %q cannot have a relative path as target. "+
-						"Only absolute paths are allowed. Found %q",
-						svc.Name, scr.Source, scr.Target)
-			}
-
-			if !filepath.IsAbs(scr.Target) {
-				scr.Target = filepath.Join(defaultSecretsPath, scr.Target)
-			}
-
-			targetDir := filepath.Dir(scr.Target)
-			targetDirKey := getServiceSecretKey(svc.Name, targetDir)
-			if _, ok := squashedTargetVolumes[targetDir]; !ok {
-				squashedTargetVolumes[targetDir] = containerinstance.Volume{
-					Name:   to.StringPtr(targetDirKey),
-					Secret: make(map[string]*string),
-				}
+					errors.Errorf("in service %q, secret with source %q cannot have a path as target. Found %q", svc.Name, scr.Source, scr.Target)
 			}
-
-			squashedTargetVolumes[targetDir].Secret[filepath.Base(scr.Target)] = &dataStr
+			secretServiceVolume.Secret[scr.Target] = &dataStr
 		}
-		for _, v := range squashedTargetVolumes {
-			secretVolumes = append(secretVolumes, v)
+		if len(secretServiceVolume.Secret) > 0 {
+			secretVolumes = append(secretVolumes, secretServiceVolume)
 		}
 	}
 
@@ -347,62 +326,15 @@ func (s serviceConfigAciHelper) getAciFileVolumeMounts(volumesCache map[string]b
 	return aciServiceVolumes, nil
 }
 
-func (s serviceConfigAciHelper) getAciSecretsVolumeMounts() ([]containerinstance.VolumeMount, error) {
-	vms := []containerinstance.VolumeMount{}
-	presenceSet := make(map[string]bool)
-	for _, scr := range s.Secrets {
-		if scr.Target == "" {
-			scr.Target = scr.Source
-		}
-		if !filepath.IsAbs(scr.Target) {
-			scr.Target = filepath.Join(defaultSecretsPath, scr.Target)
-		}
-
-		presenceKey := filepath.Dir(scr.Target)
-		if !presenceSet[presenceKey] {
-			vms = append(vms, containerinstance.VolumeMount{
-				Name:      to.StringPtr(getServiceSecretKey(s.Name, filepath.Dir(scr.Target))),
-				MountPath: to.StringPtr(filepath.Dir(scr.Target)),
-				ReadOnly:  to.BoolPtr(true),
-			})
-			presenceSet[presenceKey] = true
-		}
+func (s serviceConfigAciHelper) getAciSecretsVolumeMount() *containerinstance.VolumeMount {
+	if len(s.Secrets) == 0 {
+		return nil
 	}
-	err := validateMountPathCollisions(vms)
-	if err != nil {
-		return []containerinstance.VolumeMount{}, err
+	return &containerinstance.VolumeMount{
+		Name:      to.StringPtr(serviceSecretPrefix + s.Name),
+		MountPath: to.StringPtr("/run/secrets"),
+		ReadOnly:  to.BoolPtr(true),
 	}
-	return vms, nil
-}
-
-func validateMountPathCollisions(vms []containerinstance.VolumeMount) error {
-	for i, vm1 := range vms {
-		for j, vm2 := range vms {
-			if i == j {
-				continue
-			}
-			var (
-				biggerVMPath  = strings.Split(*vm1.MountPath, string(filepath.Separator))
-				smallerVMPath = strings.Split(*vm2.MountPath, string(filepath.Separator))
-			)
-			if len(smallerVMPath) > len(biggerVMPath) {
-				tmp := biggerVMPath
-				biggerVMPath = smallerVMPath
-				smallerVMPath = tmp
-			}
-			isPrefixed := true
-			for i := 0; i < len(smallerVMPath); i++ {
-				if smallerVMPath[i] != biggerVMPath[i] {
-					isPrefixed = false
-					break
-				}
-			}
-			if isPrefixed {
-				return errors.Errorf("mount paths %q and %q collide. A volume mount cannot include another one.", *vm1.MountPath, *vm2.MountPath)
-			}
-		}
-	}
-	return nil
 }
 
 func (s serviceConfigAciHelper) getAciContainer(volumesCache map[string]bool) (containerinstance.Container, error) {
@@ -410,11 +342,11 @@ func (s serviceConfigAciHelper) getAciContainer(volumesCache map[string]bool) (c
 	if err != nil {
 		return containerinstance.Container{}, err
 	}
-	serviceSecretVolumes, err := s.getAciSecretsVolumeMounts()
-	if err != nil {
-		return containerinstance.Container{}, err
+	allVolumes := aciServiceVolumes
+	secretVolumeMount := s.getAciSecretsVolumeMount()
+	if secretVolumeMount != nil {
+		allVolumes = append(allVolumes, *secretVolumeMount)
 	}
-	allVolumes := append(aciServiceVolumes, serviceSecretVolumes...)
 	var volumes *[]containerinstance.VolumeMount
 	if len(allVolumes) > 0 {
 		volumes = &allVolumes
diff --git a/cli/cmd/context/create_ecs.go b/cli/cmd/context/create_ecs.go
index 8dde7b04..48c7b680 100644
--- a/cli/cmd/context/create_ecs.go
+++ b/cli/cmd/context/create_ecs.go
@@ -56,6 +56,8 @@ func createEcsCommand() *cobra.Command {
 	cmd.Flags().BoolVar(&localSimulation, "local-simulation", false, "Create context for ECS local simulation endpoints")
 	cmd.Flags().StringVar(&opts.Profile, "profile", "", "Profile")
 	cmd.Flags().StringVar(&opts.Region, "region", "", "Region")
+	cmd.Flags().StringVar(&opts.AwsID, "key-id", "", "AWS Access Key ID")
+	cmd.Flags().StringVar(&opts.AwsSecret, "secret-key", "", "AWS Secret Access Key")
 	return cmd
 }
 
diff --git a/docs/examples.md b/docs/examples.md
deleted file mode 100644
index 840f48b4..00000000
--- a/docs/examples.md
+++ /dev/null
@@ -1,274 +0,0 @@
-# Compose file samples - ECS specific
-
-
-
-## Service 
-
-A service mapping may define a Docker image and runtime constraints and container requirements.
-
-```yaml
-services:
-  test:
-    image: "image"
-    command: "command"
-    entrypoint: "entrypoint"
-    environment:
-      - "FOO=BAR"
-    cap_add:
-      - SYS_PTRACE
-    cap_drop:
-      - SYSLOG
-    init: true
-    user: "user"
-    working_dir: "working_dir"
-```
-
-
-###### Task size
-
-Set resource limits that will get translated to Fargate task size values:
-
-```yaml
-services:
-  test:
-    image: nginx
-    deploy:
-      resources:
-        limits:
-          cpus: '0.5'
-          memory: 2048M
-```
-
-###### Logging
-Pass options to awslogs driver
-```yaml
-services:
-  foo:
-    image: nginx
-    logging:
-      options:
-        awslogs-datetime-pattern: "FOO"
-
-x-aws-logs_retention: 10
-```
-
-
-###### Autoscaling
-
-Set a CPU percent target
-```yaml
-services:
-  foo:
-    image: nginx
-    deploy:
-      x-aws-autoscaling: 75
-```
-
-
-###### GPU
-Set `generic_resources` for services that require accelerators as GPUs.
-```yaml
-services:
-  learning:
-    image: tensorflow/tensorflow:latest-gpus
-    deploy:
-      resources:
-        reservations:
-          memory: 32Gb
-          cpus: "32"
-          generic_resources:
-          - discrete_resource_spec:
-            kind: gpus
-            value: 2
-```
-
-
-
-
-#### Load Balancers
-
-When a service in the compose file exposes a port, a load balancer is being created and configured to distribute the traffic between all containers.
-
-There are 2 types of Load Balancers that can be created. For a service exposing a non-http port/protocol, a __Network Load Balancer (NLB)__ is created. Services with http/https ports/protocols get an __Application Load Balancer (ALB)__.
-
- There is only one load balancer created/configured for a Compose stack. If there are both http/non-http ports configured for services in a compose stack, an NLB is created.
-
-The compose file below configured only the http port,therefore, on deployment it gets an ALB created.
-
-```yaml
-services:
-  app:
-    image: nginx
-    ports:
-      - 80:80
-```
-NLB is created for non-http port
-```yaml
-services:
-  app:
-    image: nginx
-    ports:
-      - 8080:8080
-```
-
-To use the http protocol with custom ports and get an ALB, use the `x-aws-protocol` port property.
-```yaml
-services:
-  test:
-    image: nginx
-    ports:
-      - target: 8080
-        x-aws-protocol: http
-```
-
-To re-use an external load balancer and avoid creating a dedicated one, set the top-level property `x-aws-loadbalancer` as below:
-```yaml
-x-aws-loadbalancer: "LoadBalancerName"
-services:
-  app:
-    image: nginx
-    ports:
-      - 80:80
-```
-
-Similarly, an external `VPC` and `Cluster` can be reused:
-
-```yaml
-x-aws-vpc: "vpc-25435e"
-x-aws-cluster: "ClusterName"
-
-services:
-  app:
-    image: nginx
-    ports:
-      - 80:80
-```
-
-Keep in mind, that external resources are not managed as part of the compose stack's lifecycle.
-
-
-## Volumes
-
-```yaml
-services:
-  app:
-    image: nginx
-    volumes:
-      - data:/test
-volumes:
-  data:
-```
-To use of an external volume that has been previously created, set its id/ARN as the name:
-
-```yaml
-services:
-  app:
-    image: nginx
-    volumes:
-      - data:/test
-
-volumes:
-  data:
-    external: true
-    name: "fs-f534645"
-```
-
-Customize volume configuration via `driver_opts`
-
-```yaml
-services:
-  test:
-    image: nginx
-volumes:
-  db-data: 
-    driver_opts:
-        backup_policy: ENABLED
-        lifecycle_policy: AFTER_30_DAYS
-        performance_mode: maxIO
-        throughput_mode: provisioned
-        provisioned_throughput: 1024
-```
-
-
-## Secrets
-Secrets are stored in __AWS SecretsManager__ as strings and are mounted to containers  under `/run/secrets/`.
-```yaml
-services:
-  app:
-    image: nginx
-    ports:
-      - 80:80
-    secrets:
-      - mysecret
-
-secrets:
-  mysecret: 
-    file: ./secrets/mysecret.txt
-```
-
-When using external secrets, set a valid secret `ARN` under the `name` property:
-
-```yaml
-services:
-  app:
-    image: nginx
-    secrets:
-      - foo_bar
-
-secrets:
-  foo_bar:
-    name: "arn:aws:secretsmanager:eu-west-3:xxx:secret:foo_bar"
-    external: true
-```
-
-## Access private images
-When a service is configured with an image from a private repository on Docker Hub, make sure you have configured pull credentials correctly before deploying the Compose stack.
-
-To create a pull credential, create a file with the following content:
-```sh
-$ cat creds.json
-{
-  "username":"DockerHubID",
-  "password":"GeneratedHubTokenOrPassword"
-}
-```
-To create the pull credential and retrieve the `ARN/ID` to use in the compose file run:
-```sh
-$ docker secret create pullcred /path/to/creds.json
-arn:aws:secretsmanager:eu-west-3:xxx:secret:pullcred
-```
-
-Use the `ARN` in the output to set the `x-aws-pull_credentials` service property as below:
-```yaml
-services:
-  app:
-    image: DockerHubID/privateimage
-    x-aws-pull_credentials: arn:aws:secretsmanager:eu-west-3:xxx:secret:pullcred
-    ports:
-      - 80:80
-```
-
-
-
-
-#### Networks
-
-Networks are mapped to security groups.
-```yaml
-services:
-  test:
-    image: nginx
-networks:
-  default:
-```
-Using an external network/security group:
-```yaml
-services:
-  test:
-    image: nginx
-networks:
-  default:
-    external: true
-    name: sg-123abc
-```
-
diff --git a/ecs/awsResources.go b/ecs/awsResources.go
index 566754eb..980329d8 100644
--- a/ecs/awsResources.go
+++ b/ecs/awsResources.go
@@ -64,10 +64,10 @@ func (r *awsResources) allSecurityGroups() []string {
 }
 
 // parse look into compose project for configured resource to use, and check they are valid
-func (b *ecsAPIService) parse(ctx context.Context, project *types.Project) (awsResources, error) {
+func (b *ecsAPIService) parse(ctx context.Context, project *types.Project, template *cloudformation.Template) (awsResources, error) {
 	r := awsResources{}
 	var err error
-	r.cluster, err = b.parseClusterExtension(ctx, project)
+	r.cluster, err = b.parseClusterExtension(ctx, project, template)
 	if err != nil {
 		return r, err
 	}
@@ -90,7 +90,7 @@ func (b *ecsAPIService) parse(ctx context.Context, project *types.Project) (awsR
 	return r, nil
 }
 
-func (b *ecsAPIService) parseClusterExtension(ctx context.Context, project *types.Project) (string, error) {
+func (b *ecsAPIService) parseClusterExtension(ctx context.Context, project *types.Project, template *cloudformation.Template) (string, error) {
 	if x, ok := project.Extensions[extensionCluster]; ok {
 		cluster := x.(string)
 		ok, err := b.aws.ClusterExists(ctx, cluster)
@@ -100,6 +100,8 @@ func (b *ecsAPIService) parseClusterExtension(ctx context.Context, project *type
 		if !ok {
 			return "", errors.Wrapf(errdefs.ErrNotFound, "cluster %q does not exist", cluster)
 		}
+
+		template.Metadata["Cluster"] = cluster
 		return cluster, nil
 	}
 	return "", nil
diff --git a/ecs/backend.go b/ecs/backend.go
index b7218191..ab5b3dde 100644
--- a/ecs/backend.go
+++ b/ecs/backend.go
@@ -41,6 +41,9 @@ type ContextParams struct {
 	Description string
 	Region      string
 	Profile     string
+
+	AwsID     string
+	AwsSecret string
 }
 
 func init() {
diff --git a/ecs/cloudformation.go b/ecs/cloudformation.go
index 5c116fb4..e284dd51 100644
--- a/ecs/cloudformation.go
+++ b/ecs/cloudformation.go
@@ -52,12 +52,12 @@ func (b *ecsAPIService) convert(ctx context.Context, project *types.Project) (*c
 		return nil, err
 	}
 
-	resources, err := b.parse(ctx, project)
+	template := cloudformation.NewTemplate()
+	resources, err := b.parse(ctx, project, template)
 	if err != nil {
 		return nil, err
 	}
 
-	template := cloudformation.NewTemplate()
 	err = b.ensureResources(&resources, project, template)
 	if err != nil {
 		return nil, err
diff --git a/ecs/context.go b/ecs/context.go
index ce91a853..65096ea8 100644
--- a/ecs/context.go
+++ b/ecs/context.go
@@ -20,13 +20,13 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"reflect"
 	"strings"
 
 	"github.com/AlecAivazis/survey/v2/terminal"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/defaults"
-	"github.com/pkg/errors"
 	"gopkg.in/ini.v1"
 
 	"github.com/docker/compose-cli/context/store"
@@ -44,57 +44,60 @@ func newContextCreateHelper() contextCreateAWSHelper {
 	}
 }
 
-func (h contextCreateAWSHelper) createProfile(name string) error {
-	accessKey, secretKey, err := h.askCredentials()
-	if err != nil {
-		return err
-	}
-	if accessKey != "" && secretKey != "" {
-		return h.saveCredentials(name, accessKey, secretKey)
-	}
-	return nil
-}
-
-func (h contextCreateAWSHelper) createContext(profile, region, description string) (interface{}, string) {
-	if profile == "default" {
-		profile = ""
-	}
-	description = strings.TrimSpace(
-		fmt.Sprintf("%s (%s)", description, region))
-	return store.EcsContext{
-		Profile: profile,
-		Region:  region,
-	}, description
-}
-
 func (h contextCreateAWSHelper) createContextData(_ context.Context, opts ContextParams) (interface{}, string, error) {
-	profile := opts.Profile
-	region := opts.Region
+	accessKey := opts.AwsID
+	secretKey := opts.AwsSecret
 
-	profilesList, err := h.getProfiles()
-	if err != nil {
-		return nil, "", err
+	ecsCtx := store.EcsContext{
+		Profile: opts.Profile,
+		Region:  opts.Region,
 	}
-	if profile != "" {
-		// validate profile
-		if profile != "default" && !contains(profilesList, profile) {
-			return nil, "", errors.Wrapf(errdefs.ErrNotFound, "profile %q", profile)
+
+	if h.missingRequiredFlags(ecsCtx) {
+		profilesList, err := h.getProfiles()
+		if err != nil {
+			return nil, "", err
+		}
+		// get profile
+		_, ok := profilesList[ecsCtx.Profile]
+		if !ok {
+			profile, err := h.chooseProfile(profilesList)
+			if err != nil {
+				return nil, "", err
+			}
+			ecsCtx.Profile = profile
 		}
-	} else {
-		// choose profile
-		profile, err = h.chooseProfile(profilesList)
+		// set region
+		region, err := h.chooseRegion(ecsCtx.Region, profilesList[ecsCtx.Profile])
 		if err != nil {
 			return nil, "", err
 		}
-	}
-	if region == "" {
-		region, err = h.chooseRegion(region, profile)
+		ecsCtx.Region = region
+
+		accessKey, secretKey, err = h.askCredentials()
 		if err != nil {
 			return nil, "", err
 		}
 	}
-	ecsCtx, descr := h.createContext(profile, region, opts.Description)
-	return ecsCtx, descr, nil
+	if accessKey != "" && secretKey != "" {
+		if err := h.saveCredentials(ecsCtx.Profile, accessKey, secretKey); err != nil {
+			return nil, "", err
+		}
+	}
+
+	description := ecsCtx.Region
+	if opts.Description != "" {
+		description = fmt.Sprintf("%s (%s)", opts.Description, description)
+	}
+
+	return ecsCtx, description, nil
+}
+
+func (h contextCreateAWSHelper) missingRequiredFlags(ctx store.EcsContext) bool {
+	if ctx.Profile == "" || ctx.Region == "" {
+		return true
+	}
+	return false
 }
 
 func (h contextCreateAWSHelper) saveCredentials(profile string, accessKeyID string, secretAccessKey string) error {
@@ -129,151 +132,75 @@ func (h contextCreateAWSHelper) saveCredentials(profile string, accessKeyID stri
 	return credIni.SaveTo(p.Filename)
 }
 
-func (h contextCreateAWSHelper) getProfiles() ([]string, error) {
-	profiles := []string{}
-	// parse both .aws/credentials and .aws/config for profiles
-	configFiles := map[string]bool{
-		defaults.SharedCredentialsFilename(): false,
-		defaults.SharedConfigFilename():      true,
+func (h contextCreateAWSHelper) getProfiles() (map[string]ini.Section, error) {
+	profiles := map[string]ini.Section{"new profile": {}}
+	credIni, err := ini.Load(defaults.SharedConfigFilename())
+	if err != nil {
+		return nil, err
 	}
-	for f, prefix := range configFiles {
-		sections, err := loadIniFile(f, prefix)
-		if err != nil {
-			if os.IsNotExist(err) {
-				continue
-			}
-			return nil, err
-		}
-		for key := range sections {
-			name := strings.ToLower(key)
-			if !contains(profiles, name) {
-				profiles = append(profiles, name)
-			}
+	for _, section := range credIni.Sections() {
+		if strings.HasPrefix(section.Name(), "profile") {
+			profiles[section.Name()[len("profile "):]] = *section
 		}
 	}
 	return profiles, nil
 }
 
-func (h contextCreateAWSHelper) chooseProfile(profiles []string) (string, error) {
-	options := []string{"new profile"}
-	options = append(options, profiles...)
+func (h contextCreateAWSHelper) chooseProfile(section map[string]ini.Section) (string, error) {
+	keys := reflect.ValueOf(section).MapKeys()
+	profiles := make([]string, len(keys))
+	for i := 0; i < len(keys); i++ {
+		profiles[i] = keys[i].String()
+	}
 
-	selected, err := h.user.Select("Select AWS Profile", options)
+	selected, err := h.user.Select("Select AWS Profile", profiles)
 	if err != nil {
 		if err == terminal.InterruptErr {
 			return "", errdefs.ErrCanceled
 		}
 		return "", err
 	}
-	profile := options[selected]
-	if options[selected] == "new profile" {
-		suggestion := ""
-		if !contains(profiles, "default") {
-			suggestion = "default"
-		}
-		name, err := h.user.Input("profile name", suggestion)
-		if err != nil {
-			return "", err
-		}
-		if name == "" {
-			return "", fmt.Errorf("profile name cannot be empty")
-		}
-		return name, h.createProfile(name)
+	profile := profiles[selected]
+	if profiles[selected] == "new profile" {
+		return h.user.Input("profile name", "")
 	}
 	return profile, nil
 }
 
-func (h contextCreateAWSHelper) chooseRegion(region string, profile string) (string, error) {
-	suggestion := region
-
-	// only load ~/.aws/config
-	awsConfig := defaults.SharedConfigFilename()
-	configIni, err := ini.Load(awsConfig)
-
-	if err != nil {
-		if !os.IsNotExist(err) {
-			return "", err
-		}
-		configIni = ini.Empty()
-	}
-	if profile != "default" {
-		profile = fmt.Sprintf("profile %s", profile)
-	}
-	section, err := configIni.GetSection(profile)
-	if err != nil {
-		if !strings.Contains(err.Error(), "does not exist") {
-			return "", err
-		}
-		section, err = configIni.NewSection(profile)
-		if err != nil {
-			return "", err
+func (h contextCreateAWSHelper) chooseRegion(region string, section ini.Section) (string, error) {
+	defaultRegion := region
+	if defaultRegion == "" && section.Name() != "" {
+		reg, err := section.GetKey("region")
+		if err == nil {
+			defaultRegion = reg.Value()
 		}
 	}
-	reg, err := section.GetKey("region")
-	if err == nil {
-		suggestion = reg.Value()
-	}
-	// promp user for region
-	region, err = h.user.Input("Region", suggestion)
-	if err != nil {
-		return "", err
-	}
-	if region == "" {
-		return "", fmt.Errorf("region cannot be empty")
-	}
-	// save selected/typed region under profile in ~/.aws/config
-	_, err = section.NewKey("region", region)
+	result, err := h.user.Input("Region", defaultRegion)
 	if err != nil {
 		return "", err
 	}
-	return region, configIni.SaveTo(awsConfig)
+	return result, nil
 }
 
 func (h contextCreateAWSHelper) askCredentials() (string, string, error) {
-	confirm, err := h.user.Confirm("Enter AWS credentials", false)
-	if err != nil {
-		return "", "", err
-	}
-	if !confirm {
-		return "", "", nil
-	}
-
-	accessKeyID, err := h.user.Input("AWS Access Key ID", "")
+	confirm, err := h.user.Confirm("Enter credentials", false)
 	if err != nil {
 		return "", "", err
 	}
-	secretAccessKey, err := h.user.Password("Enter AWS Secret Access Key")
-	if err != nil {
-		return "", "", err
-	}
-	// validate access ID and password
-	if len(accessKeyID) < 3 || len(secretAccessKey) < 3 {
-		return "", "", fmt.Errorf("AWS Access/Secret Access Key must have more than 3 characters")
-	}
-	return accessKeyID, secretAccessKey, nil
-}
-
-func contains(values []string, value string) bool {
-	for _, v := range values {
-		if v == value {
-			return true
+	if confirm {
+		accessKeyID, err := h.user.Input("AWS Access Key ID", "")
+		if err != nil {
+			return "", "", err
 		}
-	}
-	return false
-}
-
-func loadIniFile(path string, prefix bool) (map[string]ini.Section, error) {
-	profiles := map[string]ini.Section{}
-	credIni, err := ini.Load(path)
-	if err != nil {
-		return nil, err
-	}
-	for _, section := range credIni.Sections() {
-		if prefix && strings.HasPrefix(section.Name(), "profile ") {
-			profiles[section.Name()[len("profile "):]] = *section
-		} else if !prefix || section.Name() == "default" {
-			profiles[section.Name()] = *section
+		secretAccessKey, err := h.user.Password("Enter AWS Secret Access Key")
+		if err != nil {
+			return "", "", err
 		}
+		// validate password
+		if len(secretAccessKey) < 3 {
+			return "", "", fmt.Errorf("AWS Secret Access Key must have more than 3 characters")
+		}
+		return accessKeyID, secretAccessKey, nil
 	}
-	return profiles, nil
+	return "", "", nil
 }
diff --git a/ecs/sdk.go b/ecs/sdk.go
index 81cd7189..77ee60d4 100644
--- a/ecs/sdk.go
+++ b/ecs/sdk.go
@@ -25,6 +25,7 @@ import (
 
 	"github.com/docker/compose-cli/api/compose"
 	"github.com/docker/compose-cli/api/secrets"
+	"github.com/docker/compose-cli/errdefs"
 	"github.com/docker/compose-cli/internal"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -51,6 +52,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ssm"
 	"github.com/aws/aws-sdk-go/service/ssm/ssmiface"
 	"github.com/hashicorp/go-multierror"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
@@ -332,6 +334,7 @@ func (s sdk) ListStacks(ctx context.Context, name string) ([]compose.Stack, erro
 }
 
 func (s sdk) GetStackClusterID(ctx context.Context, stack string) (string, error) {
+	// Note: could use DescribeStackResource but we only can detect `does not exist` case by matching string error message
 	resources, err := s.CF.ListStackResourcesWithContext(ctx, &cloudformation.ListStackResourcesInput{
 		StackName: aws.String(stack),
 	})
@@ -343,7 +346,28 @@ func (s sdk) GetStackClusterID(ctx context.Context, stack string) (string, error
 			return aws.StringValue(r.PhysicalResourceId), nil
 		}
 	}
-	return "", nil
+	// stack is using user-provided cluster
+	res, err := s.CF.GetTemplateSummaryWithContext(ctx, &cloudformation.GetTemplateSummaryInput{
+		StackName: aws.String(stack),
+	})
+	if err != nil {
+		return "", err
+	}
+	c := aws.StringValue(res.Metadata)
+	var m templateMetadata
+	err = json.Unmarshal([]byte(c), &m)
+	if err != nil {
+		return "", err
+	}
+	if m.Cluster == "" {
+		return "", errors.Wrap(errdefs.ErrNotFound, "CloudFormation is missing cluster metadata")
+	}
+
+	return m.Cluster, nil
+}
+
+type templateMetadata struct {
+	Cluster string `json:",omitempty"`
 }
 
 func (s sdk) GetServiceTaskDefinition(ctx context.Context, cluster string, serviceArns []string) (map[string]string, error) {
@@ -645,6 +669,9 @@ func (s sdk) DescribeService(ctx context.Context, cluster string, arn string) (c
 		return compose.ServiceStatus{}, err
 	}
 
+	for _, f := range services.Failures {
+		return compose.ServiceStatus{}, errors.Wrapf(errdefs.ErrNotFound, "can't get service status %s: %s", aws.StringValue(f.Detail), aws.StringValue(f.Reason))
+	}
 	service := services.Services[0]
 	var name string
 	for _, t := range service.Tags {


Test patch:
diff --git a/aci/convert/convert_test.go b/aci/convert/convert_test.go
index 77f8cde9..c2beecb5 100644
--- a/aci/convert/convert_test.go
+++ b/aci/convert/convert_test.go
@@ -18,10 +18,7 @@ package convert
 
 import (
 	"context"
-	"fmt"
-	"io/ioutil"
 	"os"
-	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/mock"
@@ -715,151 +712,6 @@ func TestConvertContainerGroupStatus(t *testing.T) {
 	assert.Equal(t, "Unknown", GetStatus(container(nil), group(nil)))
 }
 
-func TestConvertSecrets(t *testing.T) {
-	serviceName := "testservice"
-	secretName := "testsecret"
-	absBasePath := "/home/user"
-	tmpFile, err := ioutil.TempFile(os.TempDir(), "TestConvertProjectSecrets-")
-	assert.NilError(t, err)
-	_, err = tmpFile.Write([]byte("test content"))
-	assert.NilError(t, err)
-	t.Cleanup(func() {
-		assert.NilError(t, os.Remove(tmpFile.Name()))
-	})
-
-	t.Run("mix default and absolute", func(t *testing.T) {
-		pSquashedDefaultAndAbs := projectAciHelper{
-			Services: []types.ServiceConfig{
-				{
-					Name: serviceName,
-					Secrets: []types.ServiceSecretConfig{
-						{
-							Source: secretName,
-							Target: "some_target1",
-						},
-						{
-							Source: secretName,
-						},
-						{
-							Source: secretName,
-							Target: filepath.Join(defaultSecretsPath, "some_target2"),
-						},
-						{
-							Source: secretName,
-							Target: filepath.Join(absBasePath, "some_target3"),
-						},
-						{
-							Source: secretName,
-							Target: filepath.Join(absBasePath, "some_target4"),
-						},
-					},
-				},
-			},
-			Secrets: map[string]types.SecretConfig{
-				secretName: {
-					File: tmpFile.Name(),
-				},
-			},
-		}
-		vs, err := pSquashedDefaultAndAbs.getAciSecretVolumes()
-		assert.NilError(t, err)
-		assert.Equal(t, len(vs), 2)
-
-		defaultVolumeName := getServiceSecretKey(serviceName, defaultSecretsPath)
-		assert.Equal(t, *vs[0].Name, defaultVolumeName)
-		assert.Equal(t, len(vs[0].Secret), 3)
-
-		homeVolumeName := getServiceSecretKey(serviceName, absBasePath)
-		assert.Equal(t, *vs[1].Name, homeVolumeName)
-		assert.Equal(t, len(vs[1].Secret), 2)
-
-		s := serviceConfigAciHelper(pSquashedDefaultAndAbs.Services[0])
-		vms, err := s.getAciSecretsVolumeMounts()
-		assert.NilError(t, err)
-		assert.Equal(t, len(vms), 2)
-
-		assert.Equal(t, *vms[0].Name, defaultVolumeName)
-		assert.Equal(t, *vms[0].MountPath, defaultSecretsPath)
-
-		assert.Equal(t, *vms[1].Name, homeVolumeName)
-		assert.Equal(t, *vms[1].MountPath, absBasePath)
-	})
-
-	t.Run("convert invalid target", func(t *testing.T) {
-		targetName := "some/invalid/relative/path/target"
-		pInvalidRelativePathTarget := projectAciHelper{
-			Services: []types.ServiceConfig{
-				{
-					Name: serviceName,
-					Secrets: []types.ServiceSecretConfig{
-						{
-							Source: secretName,
-							Target: targetName,
-						},
-					},
-				},
-			},
-			Secrets: map[string]types.SecretConfig{
-				secretName: {
-					File: tmpFile.Name(),
-				},
-			},
-		}
-		_, err := pInvalidRelativePathTarget.getAciSecretVolumes()
-		assert.Equal(t, err.Error(),
-			fmt.Sprintf(`in service %q, secret with source %q cannot have a relative path as target. Only absolute paths are allowed. Found %q`,
-				serviceName, secretName, targetName))
-	})
-
-	t.Run("convert colliding default targets", func(t *testing.T) {
-		targetName1 := filepath.Join(defaultSecretsPath, "target1")
-		targetName2 := filepath.Join(defaultSecretsPath, "sub/folder/target2")
-
-		service := serviceConfigAciHelper{
-			Name: serviceName,
-			Secrets: []types.ServiceSecretConfig{
-				{
-					Source: secretName,
-					Target: targetName1,
-				},
-				{
-					Source: secretName,
-					Target: targetName2,
-				},
-			},
-		}
-
-		_, err := service.getAciSecretsVolumeMounts()
-		assert.Equal(t, err.Error(),
-			fmt.Sprintf(`mount paths %q and %q collide. A volume mount cannot include another one.`,
-				filepath.Dir(targetName1), filepath.Dir(targetName2)))
-	})
-
-	t.Run("convert colliding absolute targets", func(t *testing.T) {
-		targetName1 := filepath.Join(absBasePath, "target1")
-		targetName2 := filepath.Join(absBasePath, "sub/folder/target2")
-
-		service := serviceConfigAciHelper{
-			Name: serviceName,
-			Secrets: []types.ServiceSecretConfig{
-				{
-					Source: secretName,
-					Target: targetName1,
-				},
-				{
-					Source: secretName,
-					Target: targetName2,
-				},
-			},
-		}
-
-		_, err := service.getAciSecretsVolumeMounts()
-		assert.Equal(t, err.Error(),
-			fmt.Sprintf(`mount paths %q and %q collide. A volume mount cannot include another one.`,
-				filepath.Dir(targetName1), filepath.Dir(targetName2)))
-	})
-}
-
 func container(status *string) containerinstance.Container {
 	var state *containerinstance.ContainerState = nil
 	if status != nil {
diff --git a/ecs/cloudformation_test.go b/ecs/cloudformation_test.go
index 06176822..06881df0 100644
--- a/ecs/cloudformation_test.go
+++ b/ecs/cloudformation_test.go
@@ -492,6 +492,18 @@ services:
 	}
 }
 
+func TestTemplateMetadata(t *testing.T) {
+	template := convertYaml(t, `
+x-aws-cluster: "arn:aws:ecs:region:account:cluster/name"
+services:
+  test:
+    image: nginx
+`, useDefaultVPC, func(m *MockAPIMockRecorder) {
+		m.ClusterExists(gomock.Any(), "arn:aws:ecs:region:account:cluster/name").Return(true, nil)
+	})
+	assert.Equal(t, template.Metadata["Cluster"], "arn:aws:ecs:region:account:cluster/name")
+}
+
 func convertYaml(t *testing.T, yaml string, fn ...func(m *MockAPIMockRecorder)) *cloudformation.Template {
 	project := loadConfig(t, yaml)
 	ctrl := gomock.NewController(t)
diff --git a/tests/aci-e2e/e2e-aci_test.go b/tests/aci-e2e/e2e-aci_test.go
index 9e77e257..ddb49105 100644
--- a/tests/aci-e2e/e2e-aci_test.go
+++ b/tests/aci-e2e/e2e-aci_test.go
@@ -564,12 +564,19 @@ func TestUpSecrets(t *testing.T) {
 		secret2Value = "another_password\n"
 	)
 	var (
-		basefilePath    = filepath.Join("..", "composefiles", composeProjectName)
-		composefilePath = filepath.Join(basefilePath, "compose.yml")
+		basefilePath                 = filepath.Join("..", "composefiles", composeProjectName)
+		composefilePath              = filepath.Join(basefilePath, "compose.yml")
+		composefileInvalidTargetPath = filepath.Join(basefilePath, "compose-invalid-target.yml")
 	)
 	c := NewParallelE2eCLI(t, binDir)
 	_, _, _ = setupTestResourceGroup(t, c)
 
+	t.Run("compose up invalid target", func(t *testing.T) {
+		res := c.RunDockerOrExitError("compose", "up", "-f", composefileInvalidTargetPath, "--project-name", composeProjectName)
+		assert.Equal(t, res.ExitCode, 1)
+		assert.Equal(t, res.Combined(), "in service \"web\", secret with source \"mysecret1\" cannot have a path as target. Found \"my/invalid/target1\"\n")
+	})
+
 	t.Run("compose up", func(t *testing.T) {
 		c.RunDockerCmd("compose", "up", "-f", composefilePath, "--project-name", composeProjectName)
 		res := c.RunDockerCmd("ps")
diff --git a/tests/composefiles/aci_secrets/compose-invalid-target.yml b/tests/composefiles/aci_secrets/compose-invalid-target.yml
new file mode 100644
index 00000000..cd67b126
--- /dev/null
+++ b/tests/composefiles/aci_secrets/compose-invalid-target.yml
@@ -0,0 +1,16 @@
+services:
+  web:
+    build: .
+    image: ulyssessouza/secrets_server
+    ports:
+      - "80:80"
+    secrets:
+      - source: mysecret1
+        target: my/invalid/target1
+      - mysecret2
+
+secrets:
+  mysecret1:
+    file: ./my_secret1.txt
+  mysecret2:
+    file: ./my_secret2.txt
diff --git a/tests/ecs-e2e/e2e-ecs_test.go b/tests/ecs-e2e/e2e-ecs_test.go
index 18460a92..60393cd6 100644
--- a/tests/ecs-e2e/e2e-ecs_test.go
+++ b/tests/ecs-e2e/e2e-ecs_test.go
@@ -152,16 +152,16 @@ func setupTest(t *testing.T) (*E2eCLI, string) {
 		if localTestProfile != "" {
 			region := os.Getenv("TEST_AWS_REGION")
 			assert.Check(t, region != "")
-			res = c.RunDockerCmd("context", "create", "ecs", contextName, "--profile", "default", "--region", region)
+			res = c.RunDockerCmd("context", "create", "ecs", contextName, "--profile", localTestProfile, "--region", region)
 		} else {
-			profile := "default"
+			profile := contextName
 			region := os.Getenv("AWS_DEFAULT_REGION")
 			secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
 			keyID := os.Getenv("AWS_ACCESS_KEY_ID")
 			assert.Check(t, keyID != "")
 			assert.Check(t, secretKey != "")
 			assert.Check(t, region != "")
-			res = c.RunDockerCmd("context", "create", "ecs", contextName, "--profile", profile, "--region", region)
+			res = c.RunDockerCmd("context", "create", "ecs", contextName, "--profile", profile, "--region", region, "--secret-key", secretKey, "--key-id", keyID)
 		}
 		res.Assert(t, icmd.Expected{Out: "Successfully created ecs context \"" + contextName + "\""})
 		res = c.RunDockerCmd("context", "use", contextName)