Repository: gchq/sleeper
Issue: WaitForStackToDelete doesn't deal with failed deletes

### Description

When running `./scripts/test/tearDown.sh`, the `WaitForStackToDelete` class got stuck as the stack entered a state of `DELETE_FAILED`.

### Steps to reproduce

The stack failed to delete because there were some files in the system test bucket. It's not clear why that was so it's hard to reproduce.

### Expected behaviour

`WaitForStackToDelete` should detect that the stack has entered the state of `DELETE_FAILED`. It's not clear what's best to do after that. It could retry the delete or move on to the next part of the tear down process.



Changed files:
java/clients/src/test/java/sleeper/clients/teardown/WaitForStackToDeleteIT.java, java/clients/src/main/java/sleeper/clients/teardown/WaitForStackToDelete.java

diff --git a/java/clients/src/main/java/sleeper/clients/teardown/WaitForStackToDelete.java b/java/clients/src/main/java/sleeper/clients/teardown/WaitForStackToDelete.java
index aed4032fd..b0b359257 100644
--- a/java/clients/src/main/java/sleeper/clients/teardown/WaitForStackToDelete.java
+++ b/java/clients/src/main/java/sleeper/clients/teardown/WaitForStackToDelete.java
@@ -57,6 +57,9 @@ public class WaitForStackToDelete {
         try {
             Stack stack = cloudFormationClient.describeStacks(builder -> builder.stackName(stackName)).stacks()
                     .stream().findFirst().orElseThrow();
+            if (StackStatus.DELETE_FAILED.equals(stack.stackStatus())) {
+                throw new DeleteFailedException(stackName);
+            }
             LOGGER.info("Stack {} is currently in state {}", stackName, stack.stackStatus());
             return stack.stackStatus().equals(StackStatus.DELETE_COMPLETE);
         } catch (CloudFormationException e) {
@@ -64,4 +67,10 @@ public class WaitForStackToDelete {
             return true;
         }
     }
+
+    public static class DeleteFailedException extends RuntimeException {
+        DeleteFailedException(String stackName) {
+            super("Failed to delete stack \\"" + stackName + "\\"");
+        }
+    }
 }
diff --git a/java/clients/src/test/java/sleeper/clients/teardown/WaitForStackToDeleteIT.java b/java/clients/src/test/java/sleeper/clients/teardown/WaitForStackToDeleteIT.java
index 5d7a780ea..afe87e2fc 100644
--- a/java/clients/src/test/java/sleeper/clients/teardown/WaitForStackToDeleteIT.java
+++ b/java/clients/src/test/java/sleeper/clients/teardown/WaitForStackToDeleteIT.java
@@ -74,6 +74,18 @@ class WaitForStackToDeleteIT {
                 .isInstanceOf(PollWithRetries.TimedOutException.class);
     }
 
+    @Test
+    void shouldThrowExceptionWhenDeleteFailed(WireMockRuntimeInfo runtimeInfo) {
+        // Given
+        stubFor(describeStacksRequestWithStackName("test-stack")
+                .willReturn(aResponseWithStackName("test-stack", StackStatus.DELETE_FAILED)));
+
+        // When/Then
+        assertThatThrownBy(() -> waitForStacksToDelete(runtimeInfo, "test-stack"))
+                .isInstanceOf(WaitForStackToDelete.DeleteFailedException.class)
+                .hasMessage("Failed to delete stack \\"test-stack\\"");
+    }
+
     private static void waitForStacksToDelete(WireMockRuntimeInfo runtimeInfo, String stackName) throws InterruptedException {
         WaitForStackToDelete.from(
                         PollWithRetries.intervalAndMaxPolls(0, 1),