Repository: cortexlabs/cortex (language: go)

Issue:
Delete all cortex-created AWS resources when deleting a cluster
### Notes

Consider adding a flag which would allow the user to preserve cluster state. If so, add docs describing how to delete the resources later. It could be best if `cortex cluster down` deleted them even if the cluster was already deleted previously (without deleting the resources).

Depending on the above, update cluster uninstall docs accordingly.

- [ ] log groups (can potentially only be a single one, in which case the delete command will be simpler)
- [ ] rename keep-volumes to keep-aws-resources?
- [ ] make sure that deletes of resources are independent and best effort
- [ ] report errors to sentry at the end of the program's execution


* make sure that deletes of resources are independent and **best effort**
   * fully functional cluster
      * delete everything sequentially and log everything
   * partially deleted cluster
      * delete everything that hasn't been deleted for the cluster
   * completely deleted cluster
      * same as for the above (already deleted x,y,z).



Discussion / hints:

Fix patch:
diff --git a/cli/cmd/cluster.go b/cli/cmd/cluster.go
index f863236eee..8bacd94ab3 100644
--- a/cli/cmd/cluster.go
+++ b/cli/cmd/cluster.go
@@ -57,17 +57,17 @@ import (
 )
 
 var (
-	_flagClusterUpEnv             string
-	_flagClusterInfoEnv           string
-	_flagClusterScaleNodeGroup    string
-	_flagClusterScaleMinInstances int64
-	_flagClusterScaleMaxInstances int64
-	_flagClusterConfig            string
-	_flagClusterName              string
-	_flagClusterRegion            string
-	_flagClusterInfoDebug         bool
-	_flagClusterDisallowPrompt    bool
-	_flagClusterDownKeepVolumes   bool
+	_flagClusterUpEnv                string
+	_flagClusterInfoEnv              string
+	_flagClusterScaleNodeGroup       string
+	_flagClusterScaleMinInstances    int64
+	_flagClusterScaleMaxInstances    int64
+	_flagClusterConfig               string
+	_flagClusterName                 string
+	_flagClusterRegion               string
+	_flagClusterInfoDebug            bool
+	_flagClusterDisallowPrompt       bool
+	_flagClusterDownKeepAWSResources bool
 )
 
 var _eksctlPrefixRegex = regexp.MustCompile(`^.*[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} \[.+] {2}`)
@@ -100,7 +100,7 @@ func clusterInit() {
 	addClusterNameFlag(_clusterDownCmd)
 	addClusterRegionFlag(_clusterDownCmd)
 	_clusterDownCmd.Flags().BoolVarP(&_flagClusterDisallowPrompt, "yes", "y", false, "skip prompts")
-	_clusterDownCmd.Flags().BoolVar(&_flagClusterDownKeepVolumes, "keep-volumes", false, "keep cortex provisioned persistent volumes")
+	_clusterDownCmd.Flags().BoolVar(&_flagClusterDownKeepAWSResources, "keep-aws-resources", false, "skip deletion of resources that cortex provisioned on aws (bucket contents, ebs volumes, log group)")
 	_clusterCmd.AddCommand(_clusterDownCmd)
 
 	_clusterExportCmd.Flags().SortFlags = false
@@ -437,20 +437,33 @@ var _clusterDownCmd = &cobra.Command{
 		bucketName := clusterconfig.BucketName(accountID, accessConfig.ClusterName, accessConfig.Region)
 
 		warnIfNotAdmin(awsClient)
+		fmt.Println()
 
-		clusterState, err := clusterstate.GetClusterState(awsClient, accessConfig)
-		if err != nil && errors.GetKind(err) != clusterstate.ErrUnexpectedCloudFormationStatus {
-			exit.Error(err)
+		errorsList := []error{}
+
+		if _flagClusterDisallowPrompt {
+			fmt.Printf("your cluster named \"%s\" in %s will be spun down and all apis will be deleted\n\n", accessConfig.ClusterName, accessConfig.Region)
+		} else {
+			prompt.YesOrExit(fmt.Sprintf("your cluster named \"%s\" in %s will be spun down and all apis will be deleted, are you sure you want to continue?", accessConfig.ClusterName, accessConfig.Region), "", "")
 		}
-		if err == nil {
+
+		fmt.Print("￮ retrieving cluster ... ")
+		var clusterExists bool
+		clusterState, err := clusterstate.GetClusterState(awsClient, accessConfig)
+		if err != nil {
+			errorsList = append(errorsList, err)
+			fmt.Print("failed ✗")
+			fmt.Printf("\n\ncouldn't retrieve cluster state; check the cluster stacks in the cloudformation console: https://%s.console.aws.amazon.com/cloudformation\n", accessConfig.Region)
+			errors.PrintError(err)
+			fmt.Println()
+		} else {
 			switch clusterState.Status {
 			case clusterstate.StatusNotFound:
-				exit.Error(clusterstate.ErrorClusterDoesNotExist(accessConfig.ClusterName, accessConfig.Region))
+				fmt.Println("cluster doesn't exist ✓")
 			case clusterstate.StatusDeleteComplete:
-				// silently clean up
 				awsClient.DeleteQueuesWithPrefix(clusterconfig.SQSNamePrefix(accessConfig.ClusterName))
 				awsClient.DeletePolicy(clusterconfig.DefaultPolicyARN(accountID, accessConfig.ClusterName, accessConfig.Region))
-				if !_flagClusterDownKeepVolumes {
+				if !_flagClusterDownKeepAWSResources {
 					volumes, err := listPVCVolumesForCluster(awsClient, accessConfig.ClusterName)
 					if err == nil {
 						for _, volume := range volumes {
@@ -458,76 +471,121 @@ var _clusterDownCmd = &cobra.Command{
 						}
 					}
 				}
-				exit.Error(clusterstate.ErrorClusterAlreadyDeleted(accessConfig.ClusterName, accessConfig.Region))
+				fmt.Println("already deleted ✓")
+			default:
+				fmt.Println("found ✓")
+				clusterExists = true
 			}
 		}
 
 		// updating CLI env is best-effort, so ignore errors
 		loadBalancer, _ := getLoadBalancer(accessConfig.ClusterName, OperatorLoadBalancer, awsClient)
 
-		if _flagClusterDisallowPrompt {
-			fmt.Printf("your cluster named \"%s\" in %s will be spun down and all apis will be deleted\n\n", accessConfig.ClusterName, accessConfig.Region)
-		} else {
-			prompt.YesOrExit(fmt.Sprintf("your cluster named \"%s\" in %s will be spun down and all apis will be deleted, are you sure you want to continue?", accessConfig.ClusterName, accessConfig.Region), "", "")
-		}
-
-		fmt.Print("￮ deleting sqs queues ")
-		err = awsClient.DeleteQueuesWithPrefix(clusterconfig.SQSNamePrefix(accessConfig.ClusterName))
-		if err != nil {
+		fmt.Print("￮ deleting sqs queues ... ")
+		if queueExists, err := awsClient.DoesQueueExist(clusterconfig.SQSNamePrefix(accessConfig.ClusterName)); err != nil {
+			errorsList = append(errorsList, err)
+			fmt.Print("failed ✗")
 			fmt.Printf("\n\nfailed to delete all sqs queues; please delete queues starting with the name %s via the cloudwatch console: https://%s.console.aws.amazon.com/sqs/v2/home\n", clusterconfig.SQSNamePrefix(accessConfig.ClusterName), accessConfig.Region)
 			errors.PrintError(err)
 			fmt.Println()
+		} else if !queueExists {
+			fmt.Println("no sqs queues exist ✓")
 		} else {
-			fmt.Println("✓")
+			err = awsClient.DeleteQueuesWithPrefix(clusterconfig.SQSNamePrefix(accessConfig.ClusterName))
+			if err != nil {
+				fmt.Print("failed ✗")
+				errorsList = append(errorsList, err)
+				fmt.Printf("\n\nfailed to delete all sqs queues; please delete queues starting with the name %s via the cloudwatch console: https://%s.console.aws.amazon.com/sqs/v2/home\n", clusterconfig.SQSNamePrefix(accessConfig.ClusterName), accessConfig.Region)
+				errors.PrintError(err)
+				fmt.Println()
+			} else {
+				fmt.Println("✓")
+			}
 		}
 
-		fmt.Print("￮ spinning down the cluster ...")
-
-		out, exitCode, err := runManagerAccessCommand("/root/uninstall.sh", *accessConfig, awsClient, nil, nil)
-		if err != nil {
-			errors.PrintError(err)
+		clusterDoesntExist := !clusterExists
+		if clusterExists {
+			fmt.Print("￮ spinning down the cluster ...")
+			out, exitCode, err := runManagerAccessCommand("/root/uninstall.sh", *accessConfig, awsClient, nil, nil)
+			if err != nil {
+				errorsList = append(errorsList, err)
+				fmt.Println()
+				errors.PrintError(err)
+			} else if exitCode == nil || *exitCode != 0 {
+				template := "\nNote: if this error cannot be resolved, please ensure that all CloudFormation stacks for this cluster eventually become fully deleted (%s)."
+				template += " If the stack deletion process has failed, please delete the stacks directly from the AWS console (this may require manually deleting particular AWS resources that are blocking the stack deletion)."
+				template += " In addition to deleting the stacks manually from the AWS console, also make sure to empty and remove the %s bucket"
+				helpStr := fmt.Sprintf(template, clusterstate.CloudFormationURL(accessConfig.ClusterName, accessConfig.Region), bucketName)
+				fmt.Println(helpStr)
+				errorsList = append(errorsList, ErrorClusterDown(filterEKSCTLOutput(out)+helpStr))
+			} else {
+				clusterDoesntExist = true
+			}
 			fmt.Println()
-		} else if exitCode == nil || *exitCode != 0 {
-			out = filterEKSCTLOutput(out)
-			template := "\nNote: if this error cannot be resolved, please ensure that all CloudFormation stacks for this cluster eventually become fully deleted (%s)."
-			template += " If the stack deletion process has failed, please delete the stacks directly from the AWS console (this may require manually deleting particular AWS resources that are blocking the stack deletion)."
-			template += " In addition to deleting the stacks manually from the AWS console, also make sure to empty and remove the %s bucket"
-			helpStr := fmt.Sprintf(template, clusterstate.CloudFormationURL(accessConfig.ClusterName, accessConfig.Region), bucketName)
-			fmt.Println(helpStr)
-			exit.Error(ErrorClusterDown(out + helpStr))
 		}
 
 		// set lifecycle policy to clean the bucket
-		fmt.Printf("￮ setting lifecycle policy to empty the %s bucket ", bucketName)
-		err = setLifecycleRulesOnClusterDown(awsClient, bucketName)
-		if err != nil {
-			fmt.Printf("\n\nfailed to set lifecycle policy to empty the %s bucket; you can remove the bucket manually via the s3 console: https://s3.console.aws.amazon.com/s3/management/%s\n", bucketName, bucketName)
-			errors.PrintError(err)
-			fmt.Println()
+		var bucketExists bool
+		if !_flagClusterDownKeepAWSResources {
+			fmt.Printf("￮ setting lifecycle policy to empty the %s bucket ... ", bucketName)
+			bucketExists, err := awsClient.DoesBucketExist(bucketName)
+			if err != nil {
+				errorsList = append(errorsList, err)
+				fmt.Print("failed ✗")
+				fmt.Printf("\n\nfailed to set lifecycle policy to empty the %s bucket; you can remove the bucket manually via the s3 console: https://s3.console.aws.amazon.com/s3/management/%s\n", bucketName, bucketName)
+				errors.PrintError(err)
+				fmt.Println()
+			} else if !bucketExists {
+				fmt.Println("bucket doesn't exist ✗")
+			} else {
+				err = setLifecycleRulesOnClusterDown(awsClient, bucketName)
+				if err != nil {
+					errorsList = append(errorsList, err)
+					fmt.Print("failed ✗")
+					fmt.Printf("\n\nfailed to set lifecycle policy to empty the %s bucket; you can remove the bucket manually via the s3 console: https://s3.console.aws.amazon.com/s3/management/%s\n", bucketName, bucketName)
+					errors.PrintError(err)
+					fmt.Println()
+				} else {
+					fmt.Println("✓")
+				}
+			}
 		}
-		fmt.Println("✓")
 
 		// delete policy after spinning down the cluster (which deletes the roles) because policies can't be deleted if they are attached to roles
-		policyARN := clusterconfig.DefaultPolicyARN(accountID, accessConfig.ClusterName, accessConfig.Region)
-		fmt.Printf("￮ deleting auto-generated iam policy %s ", policyARN)
-		err = awsClient.DeletePolicy(policyARN)
-		if err != nil {
-			fmt.Printf("\n\nfailed to delete auto-generated cortex policy %s; please delete the policy via the iam console: https://console.aws.amazon.com/iam/home#/policies\n", policyARN)
-			errors.PrintError(err)
-			fmt.Println()
-		} else {
-			fmt.Println("✓")
+		if clusterDoesntExist {
+			policyARN := clusterconfig.DefaultPolicyARN(accountID, accessConfig.ClusterName, accessConfig.Region)
+			fmt.Printf("￮ deleting auto-generated iam policy %s ... ", policyARN)
+			if policy, err := awsClient.GetPolicyOrNil(policyARN); err != nil {
+				errorsList = append(errorsList, err)
+				fmt.Print("failed ✗")
+				fmt.Printf("\n\nfailed to delete auto-generated cortex policy %s; please delete the policy via the iam console: https://console.aws.amazon.com/iam/home#/policies\n", policyARN)
+				errors.PrintError(err)
+				fmt.Println()
+			} else if policy == nil {
+				fmt.Println("policy doesn't exist ✓")
+			} else {
+				err = awsClient.DeletePolicy(policyARN)
+				if err != nil {
+					errorsList = append(errorsList, err)
+					fmt.Print("failed ✗")
+					fmt.Printf("\n\nfailed to delete auto-generated cortex policy %s; please delete the policy via the iam console: https://console.aws.amazon.com/iam/home#/policies\n", policyARN)
+					errors.PrintError(err)
+					fmt.Println()
+				} else {
+					fmt.Println("✓")
+				}
+			}
 		}
 
-		// delete EBS volumes
-		if !_flagClusterDownKeepVolumes {
+		if !_flagClusterDownKeepAWSResources {
+			fmt.Print("￮ deleting ebs volumes ... ")
 			volumes, err := listPVCVolumesForCluster(awsClient, accessConfig.ClusterName)
 			if err != nil {
-				fmt.Println("\nfailed to list volumes for deletion; please delete any volumes associated with your cluster via the ec2 console: https://console.aws.amazon.com/ec2/v2/home?#Volumes")
+				errorsList = append(errorsList, err)
+				fmt.Println("\n\nfailed to list volumes for deletion; please delete any volumes associated with your cluster via the ec2 console: https://console.aws.amazon.com/ec2/v2/home?#Volumes")
 				errors.PrintError(err)
 				fmt.Println()
 			} else {
-				fmt.Print("￮ deleting ebs volumes ")
 				var failedToDeleteVolumes []string
 				var lastErr error
 				for _, volume := range volumes {
@@ -537,7 +595,10 @@ var _clusterDownCmd = &cobra.Command{
 						lastErr = err
 					}
 				}
-				if lastErr != nil {
+				if len(volumes) == 0 {
+					fmt.Println("no ebs volumes exist ✓")
+				} else if lastErr != nil {
+					errorsList = append(errorsList, lastErr)
 					fmt.Printf("\n\nfailed to delete %s %s; please delete %s via the ec2 console: https://console.aws.amazon.com/ec2/v2/home?#Volumes\n", s.PluralS("volume", len(failedToDeleteVolumes)), s.UserStrsAnd(failedToDeleteVolumes), s.PluralCustom("it", "them", len(failedToDeleteVolumes)))
 					errors.PrintError(lastErr)
 					fmt.Println()
@@ -545,8 +606,46 @@ var _clusterDownCmd = &cobra.Command{
 					fmt.Println("✓")
 				}
 			}
+
+			fmt.Printf("￮ deleting log group %s ... ", accessConfig.ClusterName)
+			logGroupExists, err := awsClient.DoesLogGroupExist(accessConfig.ClusterName)
+			if err != nil {
+				errorsList = append(errorsList, err)
+				fmt.Print("failed ✗")
+				fmt.Printf("\n\nfailed to list log group for deletion; please delete the log group associated with your cluster via the ec2 console: https://%s.console.aws.amazon.com/cloudwatch/home?#logsV2:log-groups\n", accessConfig.Region)
+				errors.PrintError(err)
+				fmt.Println()
+			} else {
+				if !logGroupExists {
+					fmt.Println("log group doesn't exist ✓")
+				} else {
+					err = awsClient.DeleteLogGroup(accessConfig.ClusterName)
+					if err != nil {
+						errorsList = append(errorsList, err)
+						fmt.Print("failed ✗")
+						fmt.Printf("\n\nfailed to delete log group %s; please delete the log group associated with your cluster via the ec2 console: https://%s.console.aws.amazon.com/cloudwatch/home?#logsV2:log-groups\n", accessConfig.ClusterName, accessConfig.Region)
+						errors.PrintError(err)
+						fmt.Println()
+					} else {
+						fmt.Println("✓")
+					}
+				}
+			}
 		}
 
+		// best-effort deletion of cached config
+		cachedClusterConfigPath := cachedClusterConfigPath(accessConfig.ClusterName, accessConfig.Region)
+		os.Remove(cachedClusterConfigPath)
+
+		if len(errorsList) > 0 {
+			exit.Error(errors.ListOfErrors(ErrClusterDown, false, errorsList...))
+		}
+		fmt.Printf("\nplease check CloudFormation to ensure that all resources for the %s cluster eventually become successfully deleted: %s\n", accessConfig.ClusterName, clusterstate.CloudFormationURL(accessConfig.ClusterName, accessConfig.Region))
+		if !_flagClusterDownKeepAWSResources && bucketExists {
+			fmt.Printf("\na lifecycle rule has been applied to the cluster's %s bucket to empty its contents within the next 24 hours; you can delete the %s bucket via the s3 console once it has been emptied (or you can empty and delete it now): https://s3.console.aws.amazon.com/s3/management/%s\n", bucketName, bucketName, bucketName)
+		}
+		fmt.Println()
+
 		// best-effort deletion of cli environment(s)
 		if loadBalancer != nil {
 			envNames, isDefaultEnv, _ := getEnvNamesByOperatorEndpoint(*loadBalancer.DNSName)
@@ -569,12 +668,6 @@ var _clusterDownCmd = &cobra.Command{
 				}
 			}
 		}
-
-		fmt.Printf("\nplease check CloudFormation to ensure that all resources for the %s cluster eventually become successfully deleted: %s\n", accessConfig.ClusterName, clusterstate.CloudFormationURL(accessConfig.ClusterName, accessConfig.Region))
-		fmt.Printf("\na lifecycle rule has been applied to the cluster’s %s bucket to empty its contents later today; you can delete the %s bucket via the s3 console once it has been emptied: https://s3.console.aws.amazon.com/s3/management/%s\n", bucketName, bucketName, bucketName)
-
-		cachedClusterConfigPath := cachedClusterConfigPath(accessConfig.ClusterName, accessConfig.Region)
-		os.Remove(cachedClusterConfigPath)
 	},
 }
 
diff --git a/dev/minimum_aws_policy.json b/dev/minimum_aws_policy.json
index 6e30adedbf..194f1cf04f 100644
--- a/dev/minimum_aws_policy.json
+++ b/dev/minimum_aws_policy.json
@@ -76,6 +76,7 @@
                 "iam:ListInstanceProfiles",
                 "logs:CreateLogGroup",
                 "logs:PutLogEvents",
+                "logs:DeleteLogGroup",
                 "iam:CreateOpenIDConnectProvider",
                 "iam:GetOpenIDConnectProvider",
                 "iam:GetRolePolicy"
diff --git a/docs/clients/cli.md b/docs/clients/cli.md
index 838c0f898a..8c48f98093 100644
--- a/docs/clients/cli.md
+++ b/docs/clients/cli.md
@@ -164,12 +164,12 @@ Usage:
   cortex cluster down [flags]
 
 Flags:
-  -c, --config string   path to a cluster configuration file
-  -n, --name string     name of the cluster
-  -r, --region string   aws region of the cluster
-  -y, --yes             skip prompts
-      --keep-volumes    keep cortex provisioned persistent volumes
-  -h, --help            help for down
+  -c, --config string        path to a cluster configuration file
+  -n, --name string          name of the cluster
+  -r, --region string        aws region of the cluster
+  -y, --yes                  skip prompts
+      --keep-aws-resources   skip deletion of resources that cortex provisioned on aws (bucket contents, ebs volumes, log group)
+  -h, --help                 help for down
 ```
 
 ## cluster export
diff --git a/docs/clusters/management/auth.md b/docs/clusters/management/auth.md
index 90d2687055..b01ff08a56 100644
--- a/docs/clusters/management/auth.md
+++ b/docs/clusters/management/auth.md
@@ -141,6 +141,7 @@ Replace the following placeholders with their respective values in the policy te
                 "iam:ListInstanceProfiles",
                 "logs:CreateLogGroup",
                 "logs:PutLogEvents",
+                "logs:DeleteLogGroup",
                 "iam:CreateOpenIDConnectProvider",
                 "iam:GetOpenIDConnectProvider",
                 "iam:GetRolePolicy"
diff --git a/docs/clusters/management/delete.md b/docs/clusters/management/delete.md
index 2b3480fde7..964ce779a3 100644
--- a/docs/clusters/management/delete.md
+++ b/docs/clusters/management/delete.md
@@ -4,34 +4,18 @@
 cortex cluster down
 ```
 
-## Delete metadata and log groups
+## Bucket Contents
 
-Since you may wish to have access to your data after spinning down your cluster, Cortex's bucket, log groups, and
-Prometheus volume are not automatically deleted when running `cortex cluster down`.
-
-To delete them:
-
-```bash
-# identify the name of your cortex S3 bucket
-aws s3 ls
-
-# delete the S3 bucket
-aws s3 rb --force s3://<bucket>
-
-# delete the log group (replace <cluster_name> with the name of your cluster, default: cortex)
-aws logs describe-log-groups --log-group-name-prefix=<cluster_name> --query logGroups[*].[logGroupName] --output text | xargs -I {} aws logs delete-log-group --log-group-name {}
-```
+When a Cortex cluster is created, an S3 bucket is created for its internal use. When running `cortex cluster down`, a lifecycle rule is applied to the bucket such that its entire contents are removed within the next 24 hours. You can safely delete the bucket at any time after `cortex cluster down` has finished running.
 
 ## Delete Certificates
 
 If you've configured a custom domain for your APIs, you can remove the SSL Certificate and Hosted Zone for the domain by
 following these [instructions](../networking/custom-domain.md#cleanup).
 
-## Keep Cortex Volumes
+## Keep Cortex Resources
 
-The volumes used by Cortex's Prometheus and Grafana instances are deleted by default on a cluster down operation.
-If you want to keep the metrics and dashboards volumes for any reason,
-you can pass the `--keep-volumes` flag to the `cortex cluster down` command.
+The contents of Cortex's S3 bucket, the EBS volumes (used by Cortex's Prometheus and Grafana instances), and the log group are deleted by default when running `cortex cluster down`. If you want to keep these resources, you can pass the `--keep-aws-resources` flag to the `cortex cluster down` command.
 
 ## Troubleshooting
 
diff --git a/pkg/lib/aws/cloudwatch.go b/pkg/lib/aws/cloudwatch.go
index 317972afce..507a8a1fca 100644
--- a/pkg/lib/aws/cloudwatch.go
+++ b/pkg/lib/aws/cloudwatch.go
@@ -105,6 +105,17 @@ func (c *Client) CreateLogGroup(logGroup string, tags map[string]string) error {
 	return nil
 }
 
+func (c *Client) DeleteLogGroup(logGroup string) error {
+	_, err := c.CloudWatchLogs().DeleteLogGroup(&cloudwatchlogs.DeleteLogGroupInput{
+		LogGroupName: aws.String(logGroup),
+	})
+	if err != nil {
+		return errors.Wrap(err, "log group "+logGroup)
+	}
+
+	return nil
+}
+
 func (c *Client) TagLogGroup(logGroup string, tagMap map[string]string) error {
 	tags := map[string]*string{}
 	for key, value := range tagMap {
diff --git a/pkg/lib/aws/iam.go b/pkg/lib/aws/iam.go
index 4490ab1902..547b36ad55 100644
--- a/pkg/lib/aws/iam.go
+++ b/pkg/lib/aws/iam.go
@@ -209,3 +209,20 @@ func (c *Client) DeletePolicy(policyARN string) error {
 	}
 	return nil
 }
+
+func (c *Client) GetPolicyOrNil(policyARN string) (*iam.Policy, error) {
+	policyOutput, err := c.IAM().GetPolicy(&iam.GetPolicyInput{
+		PolicyArn: aws.String(policyARN),
+	})
+	if err != nil {
+		if IsErrCode(err, iam.ErrCodeNoSuchEntityException) {
+			return nil, nil
+		}
+		return nil, errors.WithStack(err)
+	}
+
+	if policyOutput != nil {
+		return policyOutput.Policy, nil
+	}
+	return nil, nil
+}
diff --git a/pkg/lib/errors/errors.go b/pkg/lib/errors/errors.go
index 10ddd97b39..2758f23f7e 100644
--- a/pkg/lib/errors/errors.go
+++ b/pkg/lib/errors/errors.go
@@ -17,6 +17,7 @@ limitations under the License.
 package errors
 
 import (
+	"fmt"
 	"strings"
 
 	s "github.com/cortexlabs/cortex/pkg/lib/strings"
@@ -37,3 +38,20 @@ func ErrorUnexpected(msgs ...interface{}) error {
 		Message: strings.Join(strs, ": "),
 	})
 }
+
+func ListOfErrors(errKind string, shouldPrint bool, errors ...error) error {
+	var errorsContents string
+	for i, err := range errors {
+		if err != nil {
+			errorsContents += fmt.Sprintf("error #%d: %s\n", i+1, err.Error())
+		}
+	}
+	if errorsContents == "" {
+		return nil
+	}
+	return WithStack(&Error{
+		Kind:    errKind,
+		Message: errorsContents,
+		NoPrint: !shouldPrint,
+	})
+}


Test patch: