Repository: DataBiosphere/toil

Issue:
Add an option to support extra security groups for AWS
Currently, we create a single security group (with name equal to the cluster name) and assign default settings. However, it's sometimes necessary to assign multiple security groups that were configured outside of the main toil workflow (e.g. when limiting access to a DB).

Task is to create an option, `--awsEc2ExtraSecurityGroupIds`, that would add additional security groups to cluster. Note that we still want to keep the existing behavior for making a new security group per cluster as it's the most reliable way of identifying workers in the cluster (tags are not reliable and can fail to be created in a large cluster).

┆Issue is synchronized with this [Jira Task](https://ucsc-cgl.atlassian.net/browse/TOIL-521)
┆Issue Number: TOIL-521



Discussion:

Resolving patch:
diff --git a/src/toil/provisioners/aws/awsProvisioner.py b/src/toil/provisioners/aws/awsProvisioner.py
--- a/src/toil/provisioners/aws/awsProvisioner.py
+++ b/src/toil/provisioners/aws/awsProvisioner.py
@@ -131,6 +131,10 @@ def _readClusterSettings(self):
         self._tags = self.getLeader().tags
         self._masterPublicKey = self._setSSH()
         self._leaderProfileArn = instanceMetaData['iam']['info']['InstanceProfileArn']
+        # The existing metadata API returns a single string if there is one security group, but
+        # a list when there are multiple: change the format to always be a list.
+        rawSecurityGroups = instanceMetaData['security-groups']
+        self._leaderSecurityGroupNames = [rawSecurityGroups] if not isinstance(rawSecurityGroups, list) else rawSecurityGroups
 
     def launchCluster(self, leaderNodeType, leaderStorage, owner, **kwargs):
         """
@@ -155,11 +159,13 @@ def launchCluster(self, leaderNodeType, leaderStorage, owner, **kwargs):
             # Spot-market provisioning requires bytes for user data.
             # We probably won't have a spot-market leader, but who knows!
             userData = userData.encode('utf-8')
-        specKwargs = {'key_name': self._keyName, 'security_group_ids': [sg.id for sg in sgs],
-                  'instance_type': leaderNodeType,
-                  'user_data': userData, 'block_device_map': bdm,
-                  'instance_profile_arn': profileArn,
-                  'placement': self._zone}
+        specKwargs = {'key_name': self._keyName,
+                      'security_group_ids': [sg.id for sg in sgs] + kwargs.get('awsEc2ExtraSecurityGroupIds', []),
+                      'instance_type': leaderNodeType,
+                      'user_data': userData,
+                      'block_device_map': bdm,
+                      'instance_profile_arn': profileArn,
+                      'placement': self._zone}
         if self._vpcSubnet:
             specKwargs["subnet_id"] = self._vpcSubnet
         instances = create_ondemand_instances(self._ctx.ec2, image_id=self._discoverAMI(),
@@ -289,7 +295,7 @@ def addNodes(self, nodeType, numNodes, preemptable, spotBid=None):
         if isinstance(userData, text_type):
             # Spot-market provisioning requires bytes for user data.
             userData = userData.encode('utf-8')
-        sgs = [sg for sg in self._ctx.ec2.get_all_security_groups() if sg.name == self.clusterName]
+        sgs = [sg for sg in self._ctx.ec2.get_all_security_groups() if sg.name in self._leaderSecurityGroupNames]
         kwargs = {'key_name': self._keyName,
                   'security_group_ids': [sg.id for sg in sgs],
                   'instance_type': instanceType.name,
diff --git a/src/toil/utils/toilLaunchCluster.py b/src/toil/utils/toilLaunchCluster.py
--- a/src/toil/utils/toilLaunchCluster.py
+++ b/src/toil/utils/toilLaunchCluster.py
@@ -88,6 +88,10 @@ def main():
                         help="If provided, the specified ARN is used as the instance profile for EC2 instances."
                              "Useful for setting custom IAM profiles. If not specified, a new IAM role is created "
                              "by default with sufficient access to perform basic cluster operations.")
+    parser.add_argument('--awsEc2ExtraSecurityGroupId', dest='awsEc2ExtraSecurityGroupIds', default=[], action='append',
+                        help="Any additional security groups to attach to EC2 instances. Note that a security group "
+                             "with its name equal to the cluster name will always be created, thus ensure that "
+                             "the extra security groups do not have the same name as the cluster name.")
     config = parseBasicOptions(parser)
     tagsDict = None if config.tags is None else createTagsDict(config.tags)
     checkValidNodeTypes(config.provisioner, config.nodeTypes)
@@ -146,7 +150,8 @@ def main():
                           botoPath=config.botoPath,
                           userTags=tagsDict,
                           vpcSubnet=config.vpcSubnet,
-                          awsEc2ProfileArn=config.awsEc2ProfileArn)
+                          awsEc2ProfileArn=config.awsEc2ProfileArn,
+                          awsEc2ExtraSecurityGroupIds=config.awsEc2ExtraSecurityGroupIds)
 
     for nodeType, workers in zip(nodeTypes, numNodes):
         cluster.addNodes(nodeType=nodeType, numNodes=workers, preemptable=False)


Test patch: