Repository: DataBiosphere/toil

Issue:
`toil destroy-cluster` doesn't destroy the cluster if the leader is stopped
I tried to clean up a Toil cluster on AWS where the leader node was stopped (because we manually stopped it because the cluster wasn't going to be used for a while).

```
$ toil destroy-cluster -p aws -z us-west-2a jxu-autoscale-test2
[2022-06-15T07:21:26-0700] [MainThread] [I] [toil.utils.toilDestroyCluster] Destroying cluster jxu-autoscale-test2
Enter MFA code for arn:aws:iam::652235167018:mfa/anovak@soe.ucsc.edu: 
[2022-06-15T07:21:38-0700] [MainThread] [I] [toil.provisioners.aws.awsProvisioner] Terminating any remaining workers ...
[2022-06-15T07:21:39-0700] [MainThread] [I] [toil.provisioners.aws.awsProvisioner] Deleting launch templates ...
[2022-06-15T07:21:39-0700] [MainThread] [I] [toil.provisioners.aws.awsProvisioner] Deleting IAM roles ...
[2022-06-15T07:21:40-0700] [MainThread] [I] [toil.provisioners.aws.awsProvisioner] Deleting security group ...
[2022-06-15T07:21:41-0700] [MainThread] [I] [toil.provisioners.aws.awsProvisioner] Deleting S3 buckets ...
[2022-06-15T07:21:42-0700] [MainThread] [I] [toil.utils.toilDestroyCluster] Cluster jxu-autoscale-test2 is now gone.
```

Toil is lying to me; the leader node is still there, in its stopped state, and it still has the security group belonging to the cluster. The IAM roles I think are being left behind as well.

We should handle the case where the leader node is in the stopped state, and we shouldn't return successfully from destroy-cluster if stuff belonging to the cluster is still there.

┆Issue is synchronized with this [Jira Story](https://ucsc-cgl.atlassian.net/browse/TOIL-1185)
┆friendlyId: TOIL-1185



Discussion:

Resolving patch:
diff --git a/src/toil/provisioners/aws/awsProvisioner.py b/src/toil/provisioners/aws/awsProvisioner.py
--- a/src/toil/provisioners/aws/awsProvisioner.py
+++ b/src/toil/provisioners/aws/awsProvisioner.py
@@ -619,7 +619,7 @@ def destroyCluster(self) -> None:
         # Do the workers after the ASGs because some may belong to ASGs
         logger.info('Terminating any remaining workers ...')
         removed = False
-        instances = self._getNodesInCluster(both=True)
+        instances = self._get_nodes_in_cluster(include_stopped_nodes=True)
         spotIDs = self._getSpotRequestIDs()
         if spotIDs:
             self.aws.boto2(self._region, 'ec2').cancel_spot_instance_requests(request_ids=spotIDs)
@@ -869,7 +869,7 @@ def addManagedNodes(self, nodeTypes: Set[str], minNodes, maxNodes, preemptable,
 
     def getProvisionedWorkers(self, instance_type: Optional[str] = None, preemptable: Optional[bool] = None) -> List[Node]:
         assert self._leaderPrivateIP
-        entireCluster = self._getNodesInCluster(instance_type=instance_type, both=True)
+        entireCluster = self._get_nodes_in_cluster(instance_type=instance_type)
         logger.debug('All nodes in cluster: %s', entireCluster)
         workerInstances = [i for i in entireCluster if i.private_ip_address != self._leaderPrivateIP]
         logger.debug('All workers found in cluster: %s', workerInstances)
@@ -925,7 +925,7 @@ def _getLeaderInstance(self) -> Boto2Instance:
         """
         Get the Boto 2 instance for the cluster's leader.
         """
-        instances = self._getNodesInCluster(both=True)
+        instances = self._get_nodes_in_cluster(include_stopped_nodes=True)
         instances.sort(key=lambda x: x.launch_time)
         try:
             leader = instances[0]  # assume leader was launched first
@@ -988,7 +988,7 @@ def _terminateInstances(self, instances: List[Boto2Instance]):
         self._terminateIDs(instanceIDs)
         logger.info('... Waiting for instance(s) to shut down...')
         for instance in instances:
-            wait_transition(instance, {'pending', 'running', 'shutting-down'}, 'terminated')
+            wait_transition(instance, {'pending', 'running', 'shutting-down', 'stopping', 'stopped'}, 'terminated')
         logger.info('Instance(s) terminated.')
 
     @awsRetry
@@ -1089,24 +1089,34 @@ def _getBoto3BlockDeviceMappings(cls, type_info: InstanceType, rootVolSize: int
         return bdms
 
     @awsRetry
-    def _getNodesInCluster(self, instance_type: Optional[str] = None, preemptable=False, both=False) -> List[Boto2Instance]:
+    def _get_nodes_in_cluster(self, instance_type: Optional[str] = None, include_stopped_nodes=False) -> List[Boto2Instance]:
         """
         Get Boto2 instance objects for all nodes in the cluster.
         """
 
-        allInstances = self.aws.boto2(self._region, 'ec2').get_only_instances(filters={'instance.group-name': self.clusterName})
+        all_instances = self.aws.boto2(self._region, 'ec2').get_only_instances(filters={'instance.group-name': self.clusterName})
+
         def instanceFilter(i):
             # filter by type only if nodeType is true
             rightType = not instance_type or i.instance_type == instance_type
             rightState = i.state == 'running' or i.state == 'pending'
+            if include_stopped_nodes:
+                rightState = rightState or i.state == 'stopping' or i.state == 'stopped'
             return rightType and rightState
-        filteredInstances = [i for i in allInstances if instanceFilter(i)]
-        if not preemptable and not both:
-            return [i for i in filteredInstances if i.spot_instance_request_id is None]
-        elif preemptable and not both:
-            return [i for i in filteredInstances if i.spot_instance_request_id is not None]
-        elif both:
-            return filteredInstances
+
+        return [i for i in all_instances if instanceFilter(i)]
+
+    def _filter_nodes_in_cluster(self, instance_type: Optional[str] = None, preemptable: bool = False) -> List[Boto2Instance]:
+        """
+        Get Boto2 instance objects for the nodes in the cluster filtered by preemptability.
+        """
+
+        instances = self._get_nodes_in_cluster(instance_type, include_stopped_nodes=False)
+
+        if preemptable:
+            return [i for i in instances if i.spot_instance_request_id is not None]
+
+        return [i for i in instances if i.spot_instance_request_id is None]
 
     def _getSpotRequestIDs(self) -> List[str]:
         """


Test patch:
diff --git a/src/toil/test/provisioners/aws/awsProvisionerTest.py b/src/toil/test/provisioners/aws/awsProvisionerTest.py
--- a/src/toil/test/provisioners/aws/awsProvisionerTest.py
+++ b/src/toil/test/provisioners/aws/awsProvisionerTest.py
@@ -113,7 +113,7 @@ def rsyncUtil(self, src, dest):
         subprocess.check_call(['toil', 'rsync-cluster', '--insecure', '-p=aws', '-z', self.zone, self.clusterName] + [src, dest])
 
     def getRootVolID(self):
-        instances = self.cluster._getNodesInCluster(both=True)
+        instances = self.cluster._get_nodes_in_cluster()
         instances.sort(key=lambda x: x.launch_time)
         leader = instances[0]  # assume leader was launched first
 
@@ -303,7 +303,7 @@ def launchCluster(self):
         # visible to EC2 read requests immediately after the create returns,
         # which is the last thing that starting the cluster does.
         time.sleep(10)
-        nodes = self.cluster._getNodesInCluster(both=True)
+        nodes = self.cluster._get_nodes_in_cluster()
         nodes.sort(key=lambda x: x.launch_time)
         # assuming that leader is first
         workers = nodes[1:]