Repository: aws/amazon-ecs-cli-v2
Pull request #395 (2019-11-18 02:42:16)

Tighten up CloudFormation error handling
This fixes and enhances some cf error handling:
1. For create, if a stack was failed to have been crated,
   we'll delete it and then create the stack. This means
   if someone creates an env stack, it fails, and the run
   env init again, it'll delete the old failed stack and
   try to create it again. We only do this for stacks
   which failed cration.
2. For both update/create we error out if a stack is
   already being updated.

This doesn't fix the bug where if a stack does already exist
we don't write it to SSM.

<!-- Provide summary of changes -->

<!-- Issue number, if available. E.g. "Fixes #31", "Addresses #42, 77" -->

By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.



Patch:
diff --git a/internal/pkg/deploy/cloudformation/cloudformation.go b/internal/pkg/deploy/cloudformation/cloudformation.go
index 3fb0afd7..9377cbfe 100644
--- a/internal/pkg/deploy/cloudformation/cloudformation.go
+++ b/internal/pkg/deploy/cloudformation/cloudformation.go
@@ -168,46 +168,115 @@ func (cf CloudFormation) describeStackWithClient(describeStackInput *cloudformat
 	client cloudformationiface.CloudFormationAPI) (*cloudformation.Stack, error) {
 	describeStackOutput, err := client.DescribeStacks(describeStackInput)
 	if err != nil {
+		if stackDoesNotExist(err) {
+			return nil, &ErrStackNotFound{stackName: *describeStackInput.StackName}
+		}
 		return nil, err
 	}
 
 	if len(describeStackOutput.Stacks) == 0 {
-		return nil, fmt.Errorf("failed to find a stack named %s", *describeStackInput.StackName)
+		return nil, &ErrStackNotFound{stackName: *describeStackInput.StackName}
 	}
 
 	return describeStackOutput.Stacks[0], nil
 }
+
+// create will only spin up a stack if none exists or if a stack exists
+// but requires cleanup (meaning we failed to created it before). With
+// stacks that are failed to be created, you have to delete them if you
+// want to update them. In this case, we'll delete the failed stack
+// and then try creating it again.
+// If a stack already exists in another state, we'll return an ErrStackAlreadyExists
+// error.
 func (cf CloudFormation) create(stackConfig stackConfiguration) error {
-	return cf.deploy(stackConfig, cloudformation.ChangeSetTypeCreate)
+	describeStackInput := &cloudformation.DescribeStacksInput{StackName: aws.String(stackConfig.StackName())}
+	existingStack, err := cf.describeStack(describeStackInput)
+	// Create the stack if it doesn't already exists.
+	if err != nil {
+
+		var stackNotFound *ErrStackNotFound
+		if !errors.As(err, &stackNotFound) {
+			return err
+		}
+		// If there's no existing stack, we can go ahead and create it.
+		return cf.deploy(stackConfig, cloudformation.ChangeSetTypeCreate)
+	}
+
+	// If the stack exists, but failed to create, we'll clean it up and
+	// then re-create it.
+	if StackStatus(*existingStack.StackStatus).RequiresCleanup() {
+		if err := cf.delete(stackConfig.StackName()); err != nil {
+			return fmt.Errorf("cleaning up a previous failed stack: %w", err)
+		}
+		return cf.deploy(stackConfig, cloudformation.ChangeSetTypeCreate)
+	}
+
+	if StackStatus(*existingStack.StackStatus).InProgress() {
+		return &ErrStackUpdateInProgress{
+			stackName:   stackConfig.StackName(),
+			stackStatus: aws.StringValue(existingStack.StackStatus),
+		}
+	}
+
+	// If the stack exists and has been successfully created - return
+	// a ErrStackAlreadyExists error.
+	return &ErrStackAlreadyExists{
+		stackName: stackConfig.StackName(),
+		parentErr: fmt.Errorf("with status: %s", *existingStack.StackStatus),
+	}
 }
 
+// update will update a given stack, so long as the stack already exists
+// and it isn't already deploying something. If there's already some action
+// happening on this stack, we'll return ErrStackUpdateInProgress.
 func (cf CloudFormation) update(stackConfig stackConfiguration) error {
+	describeStackInput := &cloudformation.DescribeStacksInput{StackName: aws.String(stackConfig.StackName())}
+	existingStack, err := cf.describeStack(describeStackInput)
+	// If we can't find the stack to update, return an error.
+	if err != nil {
+		return err
+	}
+
+	// If the stack exists but is in progress, return an error.
+	if StackStatus(*existingStack.StackStatus).InProgress() {
+		return &ErrStackUpdateInProgress{
+			stackName:   stackConfig.StackName(),
+			stackStatus: aws.StringValue(existingStack.StackStatus),
+		}
+	}
+
 	return cf.deploy(stackConfig, cloudformation.ChangeSetTypeUpdate)
 }
 
+func (cf CloudFormation) delete(stackName string) error {
+	if _, err := cf.client.DeleteStack(&cloudformation.DeleteStackInput{
+		StackName: aws.String(stackName),
+	}); err != nil {
+		return fmt.Errorf("deleting stack %s: %w", stackName, err)
+	}
+
+	return cf.client.WaitUntilStackDeleteCompleteWithContext(context.Background(),
+		&cloudformation.DescribeStacksInput{StackName: aws.String(stackName)},
+		cf.waiters...)
+}
+
 func (cf CloudFormation) deploy(stackConfig stackConfiguration, createOrUpdate string) error {
 	template, err := stackConfig.Template()
 	if err != nil {
 		return fmt.Errorf("template creation: %w", err)
 	}
 
-	in, err := createChangeSetInput(stackConfig.StackName(), template, withChangeSetType(createOrUpdate), withTags(stackConfig.Tags()), withParameters(stackConfig.Parameters()))
+	in, err := createChangeSetInput(stackConfig.StackName(),
+		template,
+		withChangeSetType(createOrUpdate),
+		withTags(stackConfig.Tags()),
+		withParameters(stackConfig.Parameters()))
+
 	if err != nil {
 		return err
 	}
 
-	if err := cf.deployChangeSet(in); err != nil {
-		if stackExists(err) {
-			// Explicitly return a StackAlreadyExists error for the caller to decide if they want to ignore the
-			// operation or fail the program.
-			return &ErrStackAlreadyExists{
-				stackName: stackConfig.StackName(),
-				parentErr: err,
-			}
-		}
-		return err
-	}
-	return nil
+	return cf.deployChangeSet(in)
 }
 
 func (cf CloudFormation) deployChangeSet(in *cloudformation.CreateChangeSetInput) error {
@@ -237,31 +306,21 @@ func (cf CloudFormation) createChangeSet(in *cloudformation.CreateChangeSetInput
 	}, nil
 }
 
-// stackExists returns true if the underlying error is a stack already exists error.
-func stackExists(err error) bool {
-	currentErr := err
-	for {
-		if currentErr == nil {
-			break
-		}
-		if aerr, ok := currentErr.(awserr.Error); ok {
-			switch aerr.Code() {
-			case "ValidationError":
-				// A ValidationError occurs if we tried to create the stack with a change set.
-				if strings.Contains(aerr.Message(), "already exists") {
-					return true
-				}
-			case cloudformation.ErrCodeAlreadyExistsException:
-				// An AlreadyExists error occurs if we tried to create the stack with the CreateStack API.
+// stackDoesNotExist returns true if the underlying error is a stack doesn't exist.
+func stackDoesNotExist(err error) bool {
+	if aerr, ok := err.(awserr.Error); ok {
+		switch aerr.Code() {
+		case "ValidationError":
+			// A ValidationError occurs if we describe a stack which doesn't exist.
+			if strings.Contains(aerr.Message(), "does not exist") {
 				return true
 			}
 		}
-		currentErr = errors.Unwrap(currentErr)
 	}
 	return false
 }
 
-// stackExists returns true if the underlying error is a stack already exists error.
+// stackSetExists returns true if the underlying error is a stack already exists error.
 func stackSetExists(err error) bool {
 	if aerr, ok := err.(awserr.Error); ok {
 		switch aerr.Code() {
diff --git a/internal/pkg/deploy/cloudformation/errors.go b/internal/pkg/deploy/cloudformation/errors.go
index df5af42e..7de823b4 100644
--- a/internal/pkg/deploy/cloudformation/errors.go
+++ b/internal/pkg/deploy/cloudformation/errors.go
@@ -23,6 +23,25 @@ func (err *ErrStackAlreadyExists) Unwrap() error {
 	return err.parentErr
 }
 
+// ErrStackNotFound occurs when we can't find a particular CloudFormation stack.
+type ErrStackNotFound struct {
+	stackName string
+}
+
+func (err *ErrStackNotFound) Error() string {
+	return fmt.Sprintf("failed to find a stack named %s", err.stackName)
+}
+
+// ErrStackUpdateInProgress occurs when we try to update a stack that's already being updated.
+type ErrStackUpdateInProgress struct {
+	stackName   string
+	stackStatus string
+}
+
+func (err *ErrStackUpdateInProgress) Error() string {
+	return fmt.Sprintf("stack %s is currently being updated (status %s) and cannot be deployed to", err.stackName, err.stackStatus)
+}
+
 // ErrNotExecutableChangeSet occurs when the change set cannot be executed.
 type ErrNotExecutableChangeSet struct {
 	set *changeSet
@@ -39,7 +58,7 @@ type ErrTemplateNotFound struct {
 }
 
 func (err *ErrTemplateNotFound) Error() string {
-	return fmt.Sprintf("failed to find the cloudformation template at %s", err.templateLocation)
+	return fmt.Sprintf("find the cloudformation template at %s", err.templateLocation)
 }
 
 // Is returns true if the target's template location and parent error are equal to this error's template location and parent error.
diff --git a/internal/pkg/deploy/cloudformation/stack_status.go b/internal/pkg/deploy/cloudformation/stack_status.go
new file mode 100644
index 00000000..1561deb2
--- /dev/null
+++ b/internal/pkg/deploy/cloudformation/stack_status.go
@@ -0,0 +1,25 @@
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package cloudformation
+
+import (
+	"strings"
+
+	"github.com/aws/aws-sdk-go/service/cloudformation"
+)
+
+// StackStatus stacks
+type StackStatus string
+
+// RequiresCleanup indicates that the stack was created, but failed.
+// It should be deleted.
+func (s StackStatus) RequiresCleanup() bool {
+	return cloudformation.StackStatusRollbackComplete == string(s) ||
+		cloudformation.StackStatusRollbackFailed == string(s)
+}
+
+// InProgress that the stack is currently being updated.
+func (s StackStatus) InProgress() bool {
+	return strings.HasSuffix(string(s), "IN_PROGRESS")
+}


Test patch:
diff --git a/internal/pkg/deploy/cloudformation/cloudformation_integration_test.go b/internal/pkg/deploy/cloudformation/cloudformation_integration_test.go
index 0dcd1966..179f1f6a 100644
--- a/internal/pkg/deploy/cloudformation/cloudformation_integration_test.go
+++ b/internal/pkg/deploy/cloudformation/cloudformation_integration_test.go
@@ -412,7 +412,7 @@ func Test_Environment_Deployment_Integration(t *testing.T) {
 					"Should export CRNExecutionRole ARN")
 
 				require.True(t,
-					strings.HasSuffix(*output.OutputValue, fmt.Sprintf("role/%s-CFNExecutionRoleARN", envStackName)),
+					strings.HasSuffix(*output.OutputValue, fmt.Sprintf("role/%s-CFNExecutionRole", envStackName)),
 					"CRNExecutionRole ARN value should not be nil.")
 			},
 			"ClusterId": func(output *awsCF.Output) {
diff --git a/internal/pkg/deploy/cloudformation/cloudformation_test.go b/internal/pkg/deploy/cloudformation/cloudformation_test.go
index 5577b65c..b052befd 100644
--- a/internal/pkg/deploy/cloudformation/cloudformation_test.go
+++ b/internal/pkg/deploy/cloudformation/cloudformation_test.go
@@ -33,11 +33,10 @@ type mockCloudFormation struct {
 
 	t                                               *testing.T
 	mockCreateChangeSet                             func(t *testing.T, in *cloudformation.CreateChangeSetInput) (*cloudformation.CreateChangeSetOutput, error)
-	mockWaitUntilChangeSetCreateCompleteWithContext func(t *testing.T, in *cloudformation.DescribeChangeSetInput) error
 	mockExecuteChangeSet                            func(t *testing.T, in *cloudformation.ExecuteChangeSetInput) (*cloudformation.ExecuteChangeSetOutput, error)
 	mockDescribeChangeSet                           func(t *testing.T, in *cloudformation.DescribeChangeSetInput) (*cloudformation.DescribeChangeSetOutput, error)
-	mockWaitUntilStackCreateCompleteWithContext     func(t *testing.T, in *cloudformation.DescribeStacksInput) error
 	mockDescribeStacks                              func(t *testing.T, in *cloudformation.DescribeStacksInput) (*cloudformation.DescribeStacksOutput, error)
+	mockDeleteStack                                 func(t *testing.T, in *cloudformation.DeleteStackInput) (*cloudformation.DeleteStackOutput, error)
 	mockCreateStackSet                              func(t *testing.T, in *cloudformation.CreateStackSetInput) (*cloudformation.CreateStackSetOutput, error)
 	mockDescribeStackSet                            func(t *testing.T, in *cloudformation.DescribeStackSetInput) (*cloudformation.DescribeStackSetOutput, error)
 	mockUpdateStackSet                              func(t *testing.T, in *cloudformation.UpdateStackSetInput) (*cloudformation.UpdateStackSetOutput, error)
@@ -46,7 +45,10 @@ type mockCloudFormation struct {
 	mockDescribeStackSetOperation                   func(t *testing.T, in *cloudformation.DescribeStackSetOperationInput) (*cloudformation.DescribeStackSetOperationOutput, error)
 	mockDescribeStackEvents                         func(t *testing.T, in *cloudformation.DescribeStackEventsInput) (*cloudformation.DescribeStackEventsOutput, error)
 	mockCreateStack                                 func(t *testing.T, in *cloudformation.CreateStackInput) (*cloudformation.CreateStackOutput, error)
+	mockWaitUntilChangeSetCreateCompleteWithContext func(t *testing.T, in *cloudformation.DescribeChangeSetInput) error
+	mockWaitUntilStackCreateCompleteWithContext     func(t *testing.T, in *cloudformation.DescribeStacksInput) error
 	mockWaitUntilStackUpdateCompleteWithContext     func(t *testing.T, in *cloudformation.DescribeStacksInput) error
+	mockWaitUntilStackDeleteCompleteWithContext     func(t *testing.T, in *cloudformation.DescribeStacksInput) error
 }
 
 func (cf mockCloudFormation) CreateChangeSet(in *cloudformation.CreateChangeSetInput) (*cloudformation.CreateChangeSetOutput, error) {
@@ -57,6 +59,10 @@ func (cf mockCloudFormation) ExecuteChangeSet(in *cloudformation.ExecuteChangeSe
 	return cf.mockExecuteChangeSet(cf.t, in)
 }
 
+func (cf mockCloudFormation) DeleteStack(in *cloudformation.DeleteStackInput) (*cloudformation.DeleteStackOutput, error) {
+	return cf.mockDeleteStack(cf.t, in)
+}
+
 func (cf mockCloudFormation) DescribeChangeSet(in *cloudformation.DescribeChangeSetInput) (*cloudformation.DescribeChangeSetOutput, error) {
 	return cf.mockDescribeChangeSet(cf.t, in)
 }
@@ -109,6 +115,10 @@ func (cf mockCloudFormation) WaitUntilStackCreateCompleteWithContext(context con
 	return cf.mockWaitUntilStackCreateCompleteWithContext(cf.t, in)
 }
 
+func (cf mockCloudFormation) WaitUntilStackDeleteCompleteWithContext(context context.Context, in *cloudformation.DescribeStacksInput, opts ...request.WaiterOption) error {
+	return cf.mockWaitUntilStackDeleteCompleteWithContext(cf.t, in)
+}
+
 type mockStackConfiguration struct {
 	mockTemplate   func() (string, error)
 	mockParameters func() []*cloudformation.Parameter
@@ -132,41 +142,317 @@ func (sc mockStackConfiguration) StackName() string {
 	return sc.mockStackName()
 }
 
-func TestDeploy(t *testing.T) {
+func TestUpdate(t *testing.T) {
 	mockStackConfig := getMockStackConfiguration()
 	testCases := map[string]struct {
 		cf    CloudFormation
 		input stackConfiguration
 		want  error
 	}{
-		"should wrap error returned from CreateChangeSet call": {
+		"should deploy when there is an existing stack": {
+			cf: CloudFormation{
+				client: &mockCloudFormation{
+					t: t,
+					mockCreateChangeSet: func(t *testing.T, in *cloudformation.CreateChangeSetInput) (*cloudformation.CreateChangeSetOutput, error) {
+						require.Equal(t, mockStackConfig.StackName(), *in.StackName)
+						require.True(t, isValidChangeSetName(*in.ChangeSetName))
+						require.Equal(t, mockTemplate, *in.TemplateBody)
+						require.Equal(t, cloudformation.ChangeSetTypeUpdate, *in.ChangeSetType)
+
+						return &cloudformation.CreateChangeSetOutput{
+							Id:      aws.String(mockChangeSetID),
+							StackId: aws.String(mockStackID),
+						}, nil
+					},
+					mockWaitUntilChangeSetCreateCompleteWithContext: func(t *testing.T, in *cloudformation.DescribeChangeSetInput) error {
+						require.Equal(t, mockStackID, *in.StackName)
+						require.Equal(t, mockChangeSetID, *in.ChangeSetName)
+						return nil
+					},
+					mockDescribeChangeSet: func(t *testing.T, in *cloudformation.DescribeChangeSetInput) (*cloudformation.DescribeChangeSetOutput, error) {
+						return &cloudformation.DescribeChangeSetOutput{
+							ExecutionStatus: aws.String(cloudformation.ExecutionStatusAvailable),
+						}, nil
+					},
+					mockExecuteChangeSet: func(t *testing.T, in *cloudformation.ExecuteChangeSetInput) (output *cloudformation.ExecuteChangeSetOutput, e error) {
+						require.Equal(t, mockStackID, *in.StackName)
+						require.Equal(t, mockChangeSetID, *in.ChangeSetName)
+						return nil, nil
+					},
+					mockDescribeStacks: func(t *testing.T, in *cloudformation.DescribeStacksInput) (*cloudformation.DescribeStacksOutput, error) {
+						return &cloudformation.DescribeStacksOutput{
+							Stacks: []*cloudformation.Stack{
+								&cloudformation.Stack{
+									StackStatus: aws.String("UPDATE_COMPLETE"),
+									StackId:     aws.String(fmt.Sprintf("arn:aws:cloudformation:eu-west-3:000000000:stack/%s", *in.StackName)),
+								},
+							},
+						}, nil
+					},
+				},
+				box: boxWithTemplateFile(),
+			},
 			input: mockStackConfig,
+			want:  nil,
+		},
+		"should bubble up errors describing stack": {
+			cf: CloudFormation{
+				client: &mockCloudFormation{
+					t: t,
+					mockDescribeStacks: func(t *testing.T, in *cloudformation.DescribeStacksInput) (*cloudformation.DescribeStacksOutput, error) {
+						return nil, fmt.Errorf("error")
+					},
+				},
+				box: boxWithTemplateFile(),
+			},
+			input: mockStackConfig,
+			want:  fmt.Errorf("error"),
+		},
+		"should return ErrStackUpdateInProgress when the stack is already updating": {
+			cf: CloudFormation{
+				client: &mockCloudFormation{
+					t: t,
+					mockDescribeStacks: func(t *testing.T, in *cloudformation.DescribeStacksInput) (*cloudformation.DescribeStacksOutput, error) {
+						return &cloudformation.DescribeStacksOutput{
+							Stacks: []*cloudformation.Stack{
+								&cloudformation.Stack{
+									StackStatus: aws.String("UPDATE_IN_PROGRESS"),
+									StackId:     aws.String(fmt.Sprintf("arn:aws:cloudformation:eu-west-3:000000000:stack/%s", *in.StackName)),
+								},
+							},
+						}, nil
+					},
+				},
+				box: boxWithTemplateFile(),
+			},
+			input: mockStackConfig,
+			want:  fmt.Errorf("stack mockStackID is currently being updated (status UPDATE_IN_PROGRESS) and cannot be deployed to"),
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			got := tc.cf.update(tc.input)
+
+			if tc.want != nil {
+				require.EqualError(t, got, tc.want.Error())
+			} else {
+				require.NoError(t, got)
+			}
+		})
+	}
+}
+
+func TestCreate(t *testing.T) {
+	mockStackConfig := getMockStackConfiguration()
+	testCases := map[string]struct {
+		cf    CloudFormation
+		input stackConfiguration
+		want  error
+	}{
+		"should deploy when there is no existing stack": {
 			cf: CloudFormation{
 				client: &mockCloudFormation{
 					t: t,
 					mockCreateChangeSet: func(t *testing.T, in *cloudformation.CreateChangeSetInput) (*cloudformation.CreateChangeSetOutput, error) {
-						return nil, fmt.Errorf("some AWS error")
+						require.Equal(t, mockStackConfig.StackName(), *in.StackName)
+						require.True(t, isValidChangeSetName(*in.ChangeSetName))
+						require.Equal(t, mockTemplate, *in.TemplateBody)
+						require.Equal(t, cloudformation.ChangeSetTypeCreate, *in.ChangeSetType)
+
+						return &cloudformation.CreateChangeSetOutput{
+							Id:      aws.String(mockChangeSetID),
+							StackId: aws.String(mockStackID),
+						}, nil
+					},
+					mockWaitUntilChangeSetCreateCompleteWithContext: func(t *testing.T, in *cloudformation.DescribeChangeSetInput) error {
+						require.Equal(t, mockStackID, *in.StackName)
+						require.Equal(t, mockChangeSetID, *in.ChangeSetName)
+						return nil
+					},
+					mockDescribeChangeSet: func(t *testing.T, in *cloudformation.DescribeChangeSetInput) (*cloudformation.DescribeChangeSetOutput, error) {
+						return &cloudformation.DescribeChangeSetOutput{
+							ExecutionStatus: aws.String(cloudformation.ExecutionStatusAvailable),
+						}, nil
+					},
+					mockExecuteChangeSet: func(t *testing.T, in *cloudformation.ExecuteChangeSetInput) (output *cloudformation.ExecuteChangeSetOutput, e error) {
+						require.Equal(t, mockStackID, *in.StackName)
+						require.Equal(t, mockChangeSetID, *in.ChangeSetName)
+						return nil, nil
+					},
+					mockDescribeStacks: func(t *testing.T, in *cloudformation.DescribeStacksInput) (*cloudformation.DescribeStacksOutput, error) {
+						return &cloudformation.DescribeStacksOutput{}, nil
 					},
 				},
 				box: boxWithTemplateFile(),
 			},
-			want: fmt.Errorf("failed to create changeSet for stack %s: %s", mockStackConfig.StackName(), "some AWS error"),
+			input: mockStackConfig,
+			want:  nil,
 		},
-		"should return a ErrStackAlreadyExists if the stack already exists": {
+		"should delete a failed stack then deploy": {
 			cf: CloudFormation{
 				client: &mockCloudFormation{
 					t: t,
 					mockCreateChangeSet: func(t *testing.T, in *cloudformation.CreateChangeSetInput) (*cloudformation.CreateChangeSetOutput, error) {
-						msg := fmt.Sprintf("Stack [%s-%s] already exists and cannot be created again with the changeSet [ecscli-%s]", mockProjectName, mockEnvironmentName, mockChangeSetID)
-						return nil, awserr.New("ValidationError", msg, nil)
+						require.Equal(t, mockStackConfig.StackName(), *in.StackName)
+						require.True(t, isValidChangeSetName(*in.ChangeSetName))
+						require.Equal(t, mockTemplate, *in.TemplateBody)
+						require.Equal(t, cloudformation.ChangeSetTypeCreate, *in.ChangeSetType)
+
+						return &cloudformation.CreateChangeSetOutput{
+							Id:      aws.String(mockChangeSetID),
+							StackId: aws.String(mockStackID),
+						}, nil
+					},
+					mockWaitUntilChangeSetCreateCompleteWithContext: func(t *testing.T, in *cloudformation.DescribeChangeSetInput) error {
+						require.Equal(t, mockStackID, *in.StackName)
+						require.Equal(t, mockChangeSetID, *in.ChangeSetName)
+						return nil
+					},
+					mockDescribeChangeSet: func(t *testing.T, in *cloudformation.DescribeChangeSetInput) (*cloudformation.DescribeChangeSetOutput, error) {
+						return &cloudformation.DescribeChangeSetOutput{
+							ExecutionStatus: aws.String(cloudformation.ExecutionStatusAvailable),
+						}, nil
+					},
+					mockExecuteChangeSet: func(t *testing.T, in *cloudformation.ExecuteChangeSetInput) (output *cloudformation.ExecuteChangeSetOutput, e error) {
+						require.Equal(t, mockStackID, *in.StackName)
+						require.Equal(t, mockChangeSetID, *in.ChangeSetName)
+						return nil, nil
+					},
+					mockDescribeStacks: func(t *testing.T, in *cloudformation.DescribeStacksInput) (*cloudformation.DescribeStacksOutput, error) {
+						return &cloudformation.DescribeStacksOutput{
+							Stacks: []*cloudformation.Stack{
+								&cloudformation.Stack{
+									StackStatus: aws.String(cloudformation.StackStatusRollbackComplete),
+									StackId:     aws.String(fmt.Sprintf("arn:aws:cloudformation:eu-west-3:000000000:stack/%s", *in.StackName)),
+								},
+							},
+						}, nil
+					},
+					mockDeleteStack: func(t *testing.T, in *cloudformation.DeleteStackInput) (*cloudformation.DeleteStackOutput, error) {
+						require.Equal(t, mockStackConfig.StackName(), *in.StackName)
+						return nil, nil
+					},
+					mockWaitUntilStackDeleteCompleteWithContext: func(t *testing.T, in *cloudformation.DescribeStacksInput) error {
+						return nil
+					},
+				},
+				box: boxWithTemplateFile(),
+			},
+			input: mockStackConfig,
+			want:  nil,
+		},
+		"should bubble up errors describing stack": {
+			cf: CloudFormation{
+				client: &mockCloudFormation{
+					t: t,
+					mockDescribeStacks: func(t *testing.T, in *cloudformation.DescribeStacksInput) (*cloudformation.DescribeStacksOutput, error) {
+						return nil, fmt.Errorf("error")
 					},
 				},
 				box: boxWithTemplateFile(),
 			},
 			input: mockStackConfig,
-			want: &ErrStackAlreadyExists{
-				stackName: mockStackConfig.StackName(),
+			want:  fmt.Errorf("error"),
+		},
+		"should bubble up errors deleting old stacks": {
+			cf: CloudFormation{
+				client: &mockCloudFormation{
+					t: t,
+					mockDescribeStacks: func(t *testing.T, in *cloudformation.DescribeStacksInput) (*cloudformation.DescribeStacksOutput, error) {
+						return &cloudformation.DescribeStacksOutput{
+							Stacks: []*cloudformation.Stack{
+								&cloudformation.Stack{
+									StackStatus: aws.String(cloudformation.StackStatusRollbackComplete),
+									StackId:     aws.String(fmt.Sprintf("arn:aws:cloudformation:eu-west-3:000000000:stack/%s", *in.StackName)),
+								},
+							},
+						}, nil
+					},
+					mockDeleteStack: func(t *testing.T, in *cloudformation.DeleteStackInput) (*cloudformation.DeleteStackOutput, error) {
+						require.Equal(t, mockStackConfig.StackName(), *in.StackName)
+						return nil, fmt.Errorf("error")
+					},
+				},
+				box: boxWithTemplateFile(),
+			},
+			input: mockStackConfig,
+			want:  fmt.Errorf("cleaning up a previous failed stack: deleting stack mockStackID: error"),
+		},
+		"should return ErrStackAlreadyExists for stacks not in a failed to create state": {
+			cf: CloudFormation{
+				client: &mockCloudFormation{
+					t: t,
+					mockDescribeStacks: func(t *testing.T, in *cloudformation.DescribeStacksInput) (*cloudformation.DescribeStacksOutput, error) {
+						return &cloudformation.DescribeStacksOutput{
+							Stacks: []*cloudformation.Stack{
+								&cloudformation.Stack{
+									StackStatus: aws.String("UPDATE_COMPLETE"),
+									StackId:     aws.String(fmt.Sprintf("arn:aws:cloudformation:eu-west-3:000000000:stack/%s", *in.StackName)),
+								},
+							},
+						}, nil
+					},
+				},
+				box: boxWithTemplateFile(),
 			},
+			input: mockStackConfig,
+			want:  fmt.Errorf("stack mockStackID already exists"),
+		},
+		"should return ErrStackUpdateInProgress when the stack is already updating": {
+			cf: CloudFormation{
+				client: &mockCloudFormation{
+					t: t,
+					mockDescribeStacks: func(t *testing.T, in *cloudformation.DescribeStacksInput) (*cloudformation.DescribeStacksOutput, error) {
+						return &cloudformation.DescribeStacksOutput{
+							Stacks: []*cloudformation.Stack{
+								&cloudformation.Stack{
+									StackStatus: aws.String("UPDATE_IN_PROGRESS"),
+									StackId:     aws.String(fmt.Sprintf("arn:aws:cloudformation:eu-west-3:000000000:stack/%s", *in.StackName)),
+								},
+							},
+						}, nil
+					},
+				},
+				box: boxWithTemplateFile(),
+			},
+			input: mockStackConfig,
+			want:  fmt.Errorf("stack mockStackID is currently being updated (status UPDATE_IN_PROGRESS) and cannot be deployed to"),
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			got := tc.cf.create(tc.input)
+
+			if tc.want != nil {
+				require.EqualError(t, got, tc.want.Error())
+			} else {
+				require.NoError(t, got)
+			}
+		})
+	}
+}
+
+func TestDeploy(t *testing.T) {
+	mockStackConfig := getMockStackConfiguration()
+	testCases := map[string]struct {
+		cf    CloudFormation
+		input stackConfiguration
+		want  error
+	}{
+		"should wrap error returned from CreateChangeSet call": {
+			input: mockStackConfig,
+			cf: CloudFormation{
+				client: &mockCloudFormation{
+					t: t,
+					mockCreateChangeSet: func(t *testing.T, in *cloudformation.CreateChangeSetInput) (*cloudformation.CreateChangeSetOutput, error) {
+						return nil, fmt.Errorf("some AWS error")
+					},
+				},
+				box: boxWithTemplateFile(),
+			},
+			want: fmt.Errorf("failed to create changeSet for stack %s: %s", mockStackConfig.StackName(), "some AWS error"),
 		},
 		"should wrap error returned from WaitUntilChangeSetCreateComplete call": {
 			input: mockStackConfig,
@@ -405,6 +691,33 @@ func TestWaitForStackCreation(t *testing.T) {
 	}
 }
 
+func TestStackDoesNotExistError(t *testing.T) {
+	testCases := map[string]struct {
+		input error
+		want  bool
+	}{
+		"does not exist error": {
+			input: awserr.New("ValidationError", "stack does not exist", nil),
+			want:  true,
+		},
+		"other validation error": {
+			input: awserr.New("ValidationError", "stack exploded", nil),
+			want:  false,
+		},
+		"non aws error": {
+			input: fmt.Errorf("different error"),
+			want:  false,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			got := stackDoesNotExist(tc.input)
+			require.Equal(t, tc.want, got)
+		})
+	}
+}
+
 func getMockWaitStackCreateCFClient(t *testing.T, stackName string, shouldThrowError, shouldReturnEmptyStacks bool) CloudFormation {
 	return CloudFormation{
 		client: &mockCloudFormation{
diff --git a/internal/pkg/deploy/cloudformation/project_test.go b/internal/pkg/deploy/cloudformation/project_test.go
index ae889821..de1bd474 100644
--- a/internal/pkg/deploy/cloudformation/project_test.go
+++ b/internal/pkg/deploy/cloudformation/project_test.go
@@ -37,6 +37,9 @@ func TestCreateProjectResources(t *testing.T) {
 					mockCreateChangeSet: func(t *testing.T, in *cloudformation.CreateChangeSetInput) (*cloudformation.CreateChangeSetOutput, error) {
 						return nil, fmt.Errorf("error creating stack")
 					},
+					mockDescribeStacks: func(t *testing.T, in *cloudformation.DescribeStacksInput) (*cloudformation.DescribeStacksOutput, error) {
+						return &cloudformation.DescribeStacksOutput{}, nil
+					},
 				}
 			},
 			want: fmt.Errorf("failed to create changeSet for stack testproject-infrastructure-roles: error creating stack"),
@@ -45,13 +48,19 @@ func TestCreateProjectResources(t *testing.T) {
 			mockCFClient: func() *mockCloudFormation {
 				return &mockCloudFormation{
 					t: t,
-					mockCreateChangeSet: func(t *testing.T, in *cloudformation.CreateChangeSetInput) (*cloudformation.CreateChangeSetOutput, error) {
-						msg := fmt.Sprintf("Stack [%s-%s] already exists and cannot be created again with the changeSet [ecscli-%s]", mockProjectName, mockEnvironmentName, mockChangeSetID)
-						return nil, awserr.New("ValidationError", msg, nil)
-					},
 					mockCreateStackSet: func(t *testing.T, in *cloudformation.CreateStackSetInput) (*cloudformation.CreateStackSetOutput, error) {
 						return nil, nil
 					},
+					mockDescribeStacks: func(t *testing.T, in *cloudformation.DescribeStacksInput) (*cloudformation.DescribeStacksOutput, error) {
+						return &cloudformation.DescribeStacksOutput{
+							Stacks: []*cloudformation.Stack{
+								&cloudformation.Stack{
+									StackStatus: aws.String("UPDATE_COMPLETE"),
+									StackId:     aws.String(fmt.Sprintf("arn:aws:cloudformation:eu-west-3:902697171733:stack/%s", *in.StackName)),
+								},
+							},
+						}, nil
+					},
 				}
 			},
 		},
@@ -1133,6 +1142,7 @@ func TestDelegateDNSPermissions(t *testing.T) {
 
 // Useful for mocking a successfully deployed stack
 func getMockSuccessfulDeployCFClient(t *testing.T, stackName string) *mockCloudFormation {
+	times := 0
 	return &mockCloudFormation{
 		t: t,
 		mockCreateChangeSet: func(t *testing.T, in *cloudformation.CreateChangeSetInput) (*cloudformation.CreateChangeSetOutput, error) {
@@ -1157,6 +1167,11 @@ func getMockSuccessfulDeployCFClient(t *testing.T, stackName string) *mockCloudF
 			return nil, nil
 		},
 		mockDescribeStacks: func(t *testing.T, input *cloudformation.DescribeStacksInput) (*cloudformation.DescribeStacksOutput, error) {
+			if times == 0 {
+				times = times + 1
+				return &cloudformation.DescribeStacksOutput{}, nil
+			}
+
 			return &cloudformation.DescribeStacksOutput{
 				Stacks: []*cloudformation.Stack{
 					&cloudformation.Stack{
@@ -1208,8 +1223,9 @@ func mockProjectRolesStack(stackArn string, parameters map[string]string) *cloud
 	}
 
 	return &cloudformation.Stack{
-		StackId:    aws.String(stackArn),
-		Parameters: parametersList,
+		StackId:     aws.String(stackArn),
+		StackStatus: aws.String("UPDATE_COMPLETE"),
+		Parameters:  parametersList,
 	}
 }