Repository: DataBiosphere/toil

Issue:
AWS runs: Identifying clusterName in VPC with multiple security groups
CJ and all;
I've made progress in setting up AWS Toil CWL runs and am currently running into an issue when the AWSProvisioner class bootstraps. The origin of the problem is here:

https://github.com/BD2KGenomics/toil/blob/master/src/toil/provisioners/aws/awsProvisioner.py#L51

where it tries to infer the cluster name from security groups in the VPC. If you have any other security groups present in the VPC (default, plus any specialized ones for other applications) you get this error:
```
ip-10-0-0-210.ec2.internal 2017-02-13 19:09:09,739 MainThread INFO toil.common: Using AWS provisioner.
I0213 19:09:09.777951   152 sched.cpp:1987] Asked to stop the driver
I0213 19:09:09.778194   222 sched.cpp:1187] Stopping framework '90df95d0-8968-43ea-8994-d125f78ce8a1-0001'
Traceback (most recent call last):
  File "/usr/local/bin/cwltoil", line 11, in <module>
    sys.exit(main())
  File "/usr/local/lib/python2.7/dist-packages/toil/cwl/cwltoil.py", line 727, in main
    outobj = toil.start(wf1)
  File "/usr/local/lib/python2.7/dist-packages/toil/common.py", line 675, in start
    self._setProvisioner()
  File "/usr/local/lib/python2.7/dist-packages/toil/common.py", line 712, in _setProvisioner
    self._provisioner = AWSProvisioner(self.config, self._batchSystem)
  File "/usr/local/lib/python2.7/dist-packages/toil/provisioners/aws/awsProvisioner.py", line 52, in __init__
    self.ctx = self._buildContext(clusterName=self.clusterName)
  File "/usr/local/lib/python2.7/dist-packages/toil/provisioners/aws/awsProvisioner.py", line 97, in _buildContext
    return Context(availability_zone=zone, namespace=cls._toNameSpace(clusterName))
  File "/usr/local/lib/python2.7/dist-packages/toil/provisioners/aws/awsProvisioner.py", line 221, in _toNameSpace
    assert isinstance(clusterName, str), repr(clusterName)
AssertionError: ['toil-bcbio', 'default', 'bcbio_cluster_sg']
```
Are there approaches we can use to input the clusterName, or otherwise query for it, so we avoid the need to infer this from security_groups? Thanks for any suggestions/directions on how best to approach this.


Discussion:
@chapmanb this is related to compatibility with older AWS accounts, right? I think the easiest work around is for the Toil leader to look at its own tags and look for the 'Name' tag. That's similar to what CGCloud does for leader discovery.
Quick clarification - the 'Name' tag corresponds to the cluster name, which is why it will be useful here. 

Resolving patch:
diff --git a/src/toil/provisioners/aws/__init__.py b/src/toil/provisioners/aws/__init__.py
--- a/src/toil/provisioners/aws/__init__.py
+++ b/src/toil/provisioners/aws/__init__.py
@@ -42,7 +42,12 @@ def _getCurrentAWSZone(spotBid=None, nodeType=None, ctx=None):
         pass
     else:
         zone = os.environ.get('TOIL_AWS_ZONE', None)
-        if spotBid:
+        if not zone and runningOnEC2():
+            try:
+                zone = get_instance_metadata()['placement']['availability-zone']
+            except KeyError:
+                pass
+        if not zone and spotBid:
             # if spot bid is present, all the other parameters must be as well
             assert bool(spotBid) == bool(nodeType) == bool(ctx)
             # if the zone is unset and we are using the spot market, optimize our
@@ -52,11 +57,6 @@ def _getCurrentAWSZone(spotBid=None, nodeType=None, ctx=None):
             zone = boto.config.get('Boto', 'ec2_region_name')
             if zone is not None:
                 zone += 'a'  # derive an availability zone in the region
-        if not zone and runningOnEC2():
-            try:
-                zone = get_instance_metadata()['placement']['availability-zone']
-            except KeyError:
-                pass
     return zone
 
 
@@ -113,10 +113,13 @@ def choose_spot_zone(zones, bid, spot_history):
     for zone in zones:
         zone_histories = filter(lambda zone_history:
                                 zone_history.availability_zone == zone.name, spot_history)
-        price_deviation = std_dev([history.price for history in zone_histories])
-        recent_price = zone_histories[0]
+        if zone_histories:
+            price_deviation = std_dev([history.price for history in zone_histories])
+            recent_price = zone_histories[0].price
+        else:
+            price_deviation, recent_price = 0.0, bid
         zone_tuple = ZoneTuple(name=zone.name, price_deviation=price_deviation)
-        (markets_over_bid, markets_under_bid)[recent_price.price < bid].append(zone_tuple)
+        (markets_over_bid, markets_under_bid)[recent_price < bid].append(zone_tuple)
 
     return min(markets_under_bid or markets_over_bid,
                key=attrgetter('price_deviation')).name
@@ -127,7 +130,8 @@ def optimize_spot_bid(ctx, instance_type, spot_bid):
     Check whether the bid is sane and makes an effort to place the instance in a sensible zone.
     """
     spot_history = _get_spot_history(ctx, instance_type)
-    _check_spot_bid(spot_bid, spot_history)
+    if spot_history:
+        _check_spot_bid(spot_bid, spot_history)
     zones = ctx.ec2.get_all_zones()
     most_stable_zone = choose_spot_zone(zones, spot_bid, spot_history)
     logger.info("Placing spot instances in zone %s.", most_stable_zone)
diff --git a/src/toil/provisioners/aws/awsProvisioner.py b/src/toil/provisioners/aws/awsProvisioner.py
--- a/src/toil/provisioners/aws/awsProvisioner.py
+++ b/src/toil/provisioners/aws/awsProvisioner.py
@@ -26,6 +26,7 @@
 from six.moves import xrange
 
 from bd2k.util import memoize
+import boto.ec2
 from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType
 from boto.exception import BotoServerError, EC2ResponseError
 from cgcloud.lib.ec2 import (ec2_instance_types, a_short_time, create_ondemand_instances,
@@ -48,7 +49,7 @@ class AWSProvisioner(AbstractProvisioner):
     def __init__(self, config, batchSystem):
         super(AWSProvisioner, self).__init__(config, batchSystem)
         self.instanceMetaData = get_instance_metadata()
-        self.clusterName = self.instanceMetaData['security-groups']
+        self.clusterName = self._getClusterNameFromTags(self.instanceMetaData)
         self.ctx = self._buildContext(clusterName=self.clusterName)
         self.spotBid = None
         assert config.preemptableNodeType or config.nodeType
@@ -63,6 +64,18 @@ def __init__(self, config, batchSystem):
         self.masterPublicKey = self.setSSH()
         self.tags = self._getLeader(self.clusterName).tags
 
+    def _getClusterNameFromTags(self, md):
+        """Retrieve cluster name from current instance tags
+        """
+        instance = self._getClusterInstance(md)
+        return str(instance.tags["Name"])
+
+    def _getClusterInstance(self, md):
+        zone = getCurrentAWSZone()
+        region = Context.availability_zone_re.match(zone).group(1)
+        conn = boto.ec2.connect_to_region(region)
+        return conn.get_all_instances(instance_ids=[md["instance-id"]])[0].instances[0]
+
     @staticmethod
     def retryPredicate(e):
         return AWSProvisioner.throttlePredicate(e)
@@ -115,7 +128,7 @@ def sshLeader(cls, clusterName, args=None, zone=None, **kwargs):
         logger.info('SSH ready')
         kwargs['tty'] = sys.stdin.isatty()
         command = args if args else ['bash']
-        cls._sshAppliance(leader.ip_address, *command, **kwargs)
+        cls._sshAppliance(leader.public_dns_name, *command, **kwargs)
 
     def _remainingBillingInterval(self, instance):
         return awsRemainingBillingInterval(instance)
@@ -207,7 +220,7 @@ def _coreSSH(cls, nodeIP, *args, **kwargs):
     @classmethod
     def rsyncLeader(cls, clusterName, args, zone=None):
         leader = cls._getLeader(clusterName, zone=zone)
-        cls._rsyncNode(leader.ip_address, args)
+        cls._rsyncNode(leader.public_dns_name, args)
 
     @classmethod
     def _rsyncNode(cls, ip, args, applianceName='toil_leader'):
@@ -266,7 +279,7 @@ def _addTags(cls, instances, tags):
     def _waitForNode(cls, instance, role):
         # returns the node's IP
         cls._waitForIP(instance)
-        instanceIP = instance.ip_address
+        instanceIP = instance.public_dns_name
         cls._waitForSSHPort(instanceIP)
         cls._waitForSSHKeys(instanceIP)
         # wait here so docker commands can be used reliably afterwards
@@ -353,13 +366,14 @@ def _waitForSSHPort(cls, ip_address):
                 s.close()
 
     @classmethod
-    def launchCluster(cls, instanceType, keyName, clusterName, spotBid=None, userTags=None, zone=None):
+    def launchCluster(cls, instanceType, keyName, clusterName, spotBid=None, userTags=None, zone=None,
+                      vpcSubnet=None):
         if userTags is None:
             userTags = {}
         ctx = cls._buildContext(clusterName=clusterName, zone=zone)
         profileARN = cls._getProfileARN(ctx)
         # the security group name is used as the cluster identifier
-        cls._createSecurityGroup(ctx, clusterName)
+        sgs = cls._createSecurityGroup(ctx, clusterName, vpcSubnet)
         bdm = cls._getBlockDeviceMapping(ec2_instance_types[instanceType])
         leaderData = dict(role='leader',
                           image=applianceSelf(),
@@ -367,10 +381,12 @@ def launchCluster(cls, instanceType, keyName, clusterName, spotBid=None, userTag
                           sshKey='AAAAB3NzaC1yc2Enoauthorizedkeyneeded',
                           args=leaderArgs.format(name=clusterName))
         userData = awsUserData.format(**leaderData)
-        kwargs = {'key_name': keyName, 'security_groups': [clusterName],
+        kwargs = {'key_name': keyName, 'security_group_ids': [sg.id for sg in sgs],
                   'instance_type': instanceType,
                   'user_data': userData, 'block_device_map': bdm,
                   'instance_profile_arn': profileARN}
+        if vpcSubnet:
+            kwargs["subnet_id"] = vpcSubnet
         if not spotBid:
             logger.info('Launching non-preemptable leader')
             create_ondemand_instances(ctx.ec2, image_id=cls._discoverAMI(ctx),
@@ -402,21 +418,28 @@ def expectedShutdownErrors(e):
         if spotIDs:
             ctx.ec2.cancel_spot_instance_requests(request_ids=spotIDs)
         instancesToTerminate = awsFilterImpairedNodes(instances, ctx.ec2)
+        vpcId = None
         if instancesToTerminate:
+            vpcId = instancesToTerminate[0].vpc_id
             cls._deleteIAMProfiles(instances=instancesToTerminate, ctx=ctx)
             cls._terminateInstances(instances=instancesToTerminate, ctx=ctx)
         if len(instances) == len(instancesToTerminate):
             logger.info('Deleting security group...')
+            removed = False
             for attempt in retry(timeout=300, predicate=expectedShutdownErrors):
                 with attempt:
-                    try:
-                        ctx.ec2.delete_security_group(name=clusterName)
-                    except BotoServerError as e:
-                        if e.error_code == 'InvalidGroup.NotFound':
-                            pass
-                        else:
-                            raise
-            logger.info('... Succesfully deleted security group')
+                    for sg in ctx.ec2.get_all_security_groups():
+                        if sg.name == clusterName and vpcId and sg.vpc_id == vpcId:
+                            try:
+                                ctx.ec2.delete_security_group(group_id=sg.id)
+                                removed = True
+                            except BotoServerError as e:
+                                if e.error_code == 'InvalidGroup.NotFound':
+                                    pass
+                                else:
+                                    raise
+            if removed:
+                logger.info('... Succesfully deleted security group')
         else:
             assert len(instances) > len(instancesToTerminate)
             # the security group can't be deleted until all nodes are terminated
@@ -510,12 +533,14 @@ def _addNodes(self, instances, numNodes, preemptable=False):
                           sshKey=self.masterPublicKey,
                           args=workerArgs.format(ip=self.leaderIP, preemptable=preemptable, keyPath=keyPath))
         userData = awsUserData.format(**workerData)
+        sgs = [sg for sg in self.ctx.ec2.get_all_security_groups() if sg.name == self.clusterName]
         kwargs = {'key_name': self.keyName,
-                  'security_groups': [self.clusterName],
+                  'security_group_ids': [sg.id for sg in sgs],
                   'instance_type': self.instanceType.name,
                   'user_data': userData,
                   'block_device_map': bdm,
                   'instance_profile_arn': arn}
+        kwargs["subnet_id"] = self._getClusterInstance(self.instanceMetaData).subnet_id
 
         instancesLaunched = []
 
@@ -615,14 +640,19 @@ def _getSpotRequestIDs(cls, ctx, clusterName):
         return [request for request in requests if request.id in idsToCancel]
 
     @classmethod
-    def _createSecurityGroup(cls, ctx, name):
+    def _createSecurityGroup(cls, ctx, name, vpcSubnet=None):
         def groupNotFound(e):
             retry = (e.status == 400 and 'does not exist in default VPC' in e.body)
             return retry
-
+        vpcId = None
+        if vpcSubnet:
+            conn = boto.connect_vpc(region=ctx.ec2.region)
+            subnets = conn.get_all_subnets(subnet_ids=[vpcSubnet])
+            if len(subnets) > 0:
+                vpcId = subnets[0].vpc_id
         # security group create/get. ssh + all ports open within the group
         try:
-            web = ctx.ec2.create_security_group(name, 'Toil appliance security group')
+            web = ctx.ec2.create_security_group(name, 'Toil appliance security group', vpc_id=vpcId)
         except EC2ResponseError as e:
             if e.status == 400 and 'already exists' in e.body:
                 pass  # group exists- nothing to do
@@ -637,6 +667,11 @@ def groupNotFound(e):
                 with attempt:
                     # the following authorizes all port access within the web security group
                     web.authorize(ip_protocol='tcp', from_port=0, to_port=65535, src_group=web)
+        out = []
+        for sg in ctx.ec2.get_all_security_groups():
+            if sg.name == name and vpcId is None or sg.vpc_id == vpcId:
+                out.append(sg)
+        return out
 
     @classmethod
     def _getProfileARN(cls, ctx):
diff --git a/src/toil/utils/toilLaunchCluster.py b/src/toil/utils/toilLaunchCluster.py
--- a/src/toil/utils/toilLaunchCluster.py
+++ b/src/toil/utils/toilLaunchCluster.py
@@ -49,6 +49,9 @@ def main():
                              "      \"Name\": clusterName,"
                              "      \"Owner\": IAM username"
                              " }. ")
+    parser.add_argument("--vpcSubnet",
+                        help="VPC subnet ID to launch cluster in. Uses default subnet if not specified."
+                        "This subnet needs to have auto assign IPs turned on.")
     config = parseBasicOptions(parser)
     setLoggingFromOptions(config)
     tagsDict = None if config.tags is None else createTagsDict(config.tags)
@@ -70,4 +73,5 @@ def main():
         assert False
 
     provisioner.launchCluster(instanceType=config.nodeType, clusterName=config.clusterName,
-                              keyName=config.keyPairName, spotBid=spotBid, userTags=tagsDict, zone=config.zone)
+                              keyName=config.keyPairName, spotBid=spotBid, userTags=tagsDict, zone=config.zone,
+                              vpcSubnet=config.vpcSubnet)


Test patch: