Repository: DataBiosphere/toil

Issue:
Flaky Test: AWSManagedAutoscaleTest.testAutoScale can fail because the cluster can't be destroyed


┆Issue is synchronized with this [Jira Story](https://ucsc-cgl.atlassian.net/browse/TOIL-1245)
┆Issue Number: TOIL-1245



Discussion:
In https://ucsc-ci.com/databiosphere/toil/-/jobs/26996 I got this failure:

```
____________________ AWSManagedAutoscaleTest.testAutoScale _____________________
Traceback (most recent call last):
  File "/builds/databiosphere/toil/src/toil/test/provisioners/clusterTest.py", line 79, in tearDown
    self.destroyCluster()
  File "/builds/databiosphere/toil/src/toil/test/provisioners/clusterTest.py", line 64, in destroyCluster
    subprocess.check_call(['toil', 'destroy-cluster', '-p=aws', '-z', self.zone, self.clusterName])
  File "/usr/lib/python3.9/subprocess.py", line 373, in check_call
    raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['toil', 'destroy-cluster', '-p=aws', '-z', 'us-west-2a', 'provisioner-test-66b40bd0-30b9-4dd4-9838-b9c9e08f5185']' returned non-zero exit status 1.
```

It looks like the actual error is:

```
[2022-11-16T03:01:58+0000] [MainThread] [I] [toil.utils.toilDestroyCluster] Destroying cluster provisioner-test-66b40bd0-30b9-4dd4-9838-b9c9e08f5185
[2022-11-16T03:01:59+0000] [MainThread] [E] [toil.lib.retry] Got a <class 'botocore.exceptions.ClientError'>: An error occurred (ValidationError) when calling the DeleteAutoScalingGroup operation: AutoScalingGroup name not found - AutoScalingGroup 'toil-a087ca19-84fa-4e1a-bfee-66548c8ed73a' not found which is not retriable according to <function expectedShutdownErrors at 0x7fae086b98b0>
Traceback (most recent call last):
  File "/builds/databiosphere/toil/venv/bin/toil", line 33, in <module>
    sys.exit(load_entry_point('toil', 'console_scripts', 'toil')())
  File "/builds/databiosphere/toil/src/toil/utils/toilMain.py", line 32, in main
    get_or_die(module, 'main')()
  File "/builds/databiosphere/toil/src/toil/utils/toilDestroyCluster.py", line 33, in main
    cluster.destroyCluster()
  File "/builds/databiosphere/toil/src/toil/provisioners/aws/awsProvisioner.py", line 614, in destroyCluster
    self.aws.client(self._region, 'autoscaling').delete_auto_scaling_group(AutoScalingGroupName=asgName, ForceDelete=True)
  File "/builds/databiosphere/toil/venv/lib/python3.9/site-packages/botocore/client.py", line 515, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "/builds/databiosphere/toil/venv/lib/python3.9/site-packages/botocore/client.py", line 934, in _make_api_call
    raise error_class(parsed_response, operation_name)
botocore.exceptions.ClientError: An error occurred (ValidationError) when calling the DeleteAutoScalingGroup operation: AutoScalingGroup name not found - AutoScalingGroup 'toil-a087ca19-84fa-4e1a-bfee-66548c8ed73a' not found
```

So we probably need to add this to the expected errors to get at shutdown, because an ASG we want to destroy might not exist or might be being destroyed.

Resolving patch:
diff --git a/src/toil/provisioners/aws/awsProvisioner.py b/src/toil/provisioners/aws/awsProvisioner.py
--- a/src/toil/provisioners/aws/awsProvisioner.py
+++ b/src/toil/provisioners/aws/awsProvisioner.py
@@ -68,6 +68,7 @@
 from toil.lib.retry import (ErrorCondition,
                             get_error_body,
                             get_error_code,
+                            get_error_message,
                             get_error_status,
                             old_retry,
                             retry)
@@ -116,7 +117,15 @@ def awsRetryPredicate(e):
     return False
 
 
-def expectedShutdownErrors(e):
+def expectedShutdownErrors(e: Exception) -> bool:
+    """
+    Matches errors that we expect to occur during shutdown, and which indicate
+    that we need to wait or try again.
+
+    Should *not* match any errors which indicate that an operation is
+    impossible or unnecessary (such as errors resulting from a thing not
+    existing to be deleted).
+    """
     return get_error_status(e) == 400 and 'dependent object' in get_error_body(e)
 
 
@@ -249,7 +258,7 @@ def _write_file_to_cloud(self, key: str, contents: bytes) -> str:
             s3_client.head_bucket(Bucket=bucket_name)
             bucket = s3.Bucket(bucket_name)
         except ClientError as err:
-            if err.response.get('ResponseMetadata', {}).get('HTTPStatusCode') == 404:
+            if get_error_status(err) == 404:
                 bucket = create_s3_bucket(s3, bucket_name=bucket_name, region=self._region)
                 bucket.wait_until_exists()
                 bucket.Versioning().enable()
@@ -276,7 +285,7 @@ def _read_file_from_cloud(self, key: str) -> bytes:
         try:
             return obj.get().get('Body').read()
         except ClientError as e:
-            if e.response.get('ResponseMetadata', {}).get('HTTPStatusCode') == 404:
+            if get_error_status(e) == 404:
                 logger.warning(f'Trying to read non-existent file "{key}" from {bucket_name}.')
             raise
 
@@ -615,12 +624,21 @@ def destroyCluster(self) -> None:
 
         logger.debug('Deleting autoscaling groups ...')
         removed = False
+
         for attempt in old_retry(timeout=300, predicate=expectedShutdownErrors):
             with attempt:
                 for asgName in self._getAutoScalingGroupNames():
-                    # We delete the group and all the instances via ForceDelete.
-                    self.aws.client(self._region, 'autoscaling').delete_auto_scaling_group(AutoScalingGroupName=asgName, ForceDelete=True)
-                    removed = True
+                    try:
+                        # We delete the group and all the instances via ForceDelete.
+                        self.aws.client(self._region, 'autoscaling').delete_auto_scaling_group(AutoScalingGroupName=asgName, ForceDelete=True)
+                        removed = True
+                    except ClientError as e:
+                        if get_error_code(e) == 'ValidationError' and 'AutoScalingGroup name not found' in get_error_message(e):
+                            # This ASG does not need to be removed (or a
+                            # previous delete returned an error but also
+                            # succeeded).
+                            pass
+
         if removed:
             logger.debug('... Successfully deleted autoscaling groups')
 
@@ -709,7 +727,7 @@ def destroyCluster(self) -> None:
                 except s3.meta.client.exceptions.NoSuchBucket:
                     pass
                 except ClientError as e:
-                    if e.response.get('ResponseMetadata', {}).get('HTTPStatusCode') == 404:
+                    if get_error_status(e) == 404:
                         pass
                     else:
                         raise  # retry this


Test patch: