Repository: DataBiosphere/toil

Issue:
Review EC2 availability zone placement policy
Is zone hardcoded?
Should it be configurable?
Try to prevent:
http://jenkins.cgcloud.info/job/toil-pull-request-integration-tests/347/testReport/junit/src.toil.test.provisioners.cgcloud.provisionerTest/CGCloudRNASeqTest/testStaticCluster/
Default `TOIL_APPLIANCE_SELF` tag isn't found in quay
```
eduroam-169-233-244-140.ucsc.edu 2016-11-30 12:07:46,976 MainThread INFO toil.lib.bioio: Root logger is at level 'DEBUG', 'toil' logger at level 'DEBUG'.
eduroam-169-233-244-140.ucsc.edu 2016-11-30 12:07:46,977 MainThread INFO toil.lib.bioio: Root logger is at level 'DEBUG', 'toil' logger at level 'DEBUG'.
eduroam-169-233-244-140.ucsc.edu 2016-11-30 12:07:46,977 MainThread INFO toil.utils.toilLaunchCluster: Using aws provisioner.
eduroam-169-233-244-140.ucsc.edu 2016-11-30 12:07:50,394 MainThread DEBUG toil.provisioners.aws.awsProvisioner: Device mapping: {}
eduroam-169-233-244-140.ucsc.edu 2016-11-30 12:07:50,395 MainThread INFO toil: Using default docker registry of quay.io/ucsc_cgl as TOIL_DOCKER_REGISTRY is not set.
eduroam-169-233-244-140.ucsc.edu 2016-11-30 12:07:50,395 MainThread INFO toil: Using default docker name of toil as TOIL_DOCKER_NAME is not set.
eduroam-169-233-244-140.ucsc.edu 2016-11-30 12:07:50,395 MainThread INFO toil: Using default docker appliance of quay.io/ucsc_cgl/toil:3.5.0a1-d978eb71c0d76d2c759f9245074b1edebcb35719-dirty as TOIL_APPLIANCE_SELF is not set.
```

Should not be tagged as dirty?

cc: @cket


Discussion:

For the record, the workaround is to set TOIL_APPLIANCE_SELF to a value that includes a valid tag on quay.io. I will add documentation for developers on building and pushing their own appliance images and once we release 3.5 we will set a 'latest' tag on quay that will make finding a default easier.

Resolving patch:
diff --git a/src/toil/provisioners/aws/__init__.py b/src/toil/provisioners/aws/__init__.py
--- a/src/toil/provisioners/aws/__init__.py
+++ b/src/toil/provisioners/aws/__init__.py
@@ -11,10 +11,27 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import logging
 import os
+from collections import namedtuple
+from operator import attrgetter
+import datetime
+from cgcloud.lib.util import std_dev, mean
+
+logger = logging.getLogger(__name__)
+
+ZoneTuple = namedtuple('ZoneTuple', ['name', 'price_deviation'])
+
+
+def getSpotZone(spotBid, nodeType, ctx):
+    return _getCurrentAWSZone(spotBid, nodeType, ctx)
 
 
 def getCurrentAWSZone():
+    return _getCurrentAWSZone()
+
+
+def _getCurrentAWSZone(spotBid=None, nodeType=None, ctx=None):
     zone = None
     try:
         import boto
@@ -23,6 +40,12 @@ def getCurrentAWSZone():
         pass
     else:
         zone = os.environ.get('TOIL_AWS_ZONE', None)
+        if spotBid:
+            # if spot bid is present, all the other parameters must be as well
+            assert bool(spotBid) == bool(nodeType) == bool(ctx)
+            # if the zone is unset and we are using the spot market, optimize our
+            # choice based on the spot history
+            return optimize_spot_bid(ctx=ctx, instance_type=nodeType, spot_bid=spotBid)
         if not zone:
             zone = boto.config.get('Boto', 'ec2_region_name')
             if zone is not None:
@@ -34,6 +57,128 @@ def getCurrentAWSZone():
                 pass
     return zone
 
+
+def choose_spot_zone(zones, bid, spot_history):
+    """
+    Returns the zone to put the spot request based on, in order of priority:
+
+       1) zones with prices currently under the bid
+
+       2) zones with the most stable price
+
+    :param list[boto.ec2.zone.Zone] zones:
+    :param float bid:
+    :param list[boto.ec2.spotpricehistory.SpotPriceHistory] spot_history:
+
+    :rtype: str
+    :return: the name of the selected zone
+
+    >>> from collections import namedtuple
+    >>> FauxHistory = namedtuple( 'FauxHistory', [ 'price', 'availability_zone' ] )
+    >>> ZoneTuple = namedtuple( 'ZoneTuple', [ 'name' ] )
+
+    >>> zones = [ ZoneTuple( 'us-west-2a' ), ZoneTuple( 'us-west-2b' ) ]
+    >>> spot_history = [ FauxHistory( 0.1, 'us-west-2a' ), \
+                         FauxHistory( 0.2,'us-west-2a'), \
+                         FauxHistory( 0.3,'us-west-2b'), \
+                         FauxHistory( 0.6,'us-west-2b')]
+    >>> # noinspection PyProtectedMember
+    >>> choose_spot_zone( zones, 0.15, spot_history )
+    'us-west-2a'
+
+    >>> spot_history=[ FauxHistory( 0.3, 'us-west-2a' ), \
+                       FauxHistory( 0.2, 'us-west-2a' ), \
+                       FauxHistory( 0.1, 'us-west-2b'), \
+                       FauxHistory( 0.6, 'us-west-2b') ]
+    >>> # noinspection PyProtectedMember
+    >>> choose_spot_zone(zones, 0.15, spot_history)
+    'us-west-2b'
+
+    >>> spot_history={ FauxHistory( 0.1, 'us-west-2a' ), \
+                       FauxHistory( 0.7, 'us-west-2a' ), \
+                       FauxHistory( 0.1, "us-west-2b" ), \
+                       FauxHistory( 0.6, 'us-west-2b' ) }
+    >>> # noinspection PyProtectedMember
+    >>> choose_spot_zone(zones, 0.15, spot_history)
+    'us-west-2b'
+   """
+
+    # Create two lists of tuples of form: [ (zone.name, std_deviation), ... ] one for zones
+    # over the bid price and one for zones under bid price. Each are sorted by increasing
+    # standard deviation values.
+    #
+    markets_under_bid, markets_over_bid = [], []
+    for zone in zones:
+        zone_histories = filter(lambda zone_history:
+                                zone_history.availability_zone == zone.name, spot_history)
+        price_deviation = std_dev([history.price for history in zone_histories])
+        recent_price = zone_histories[0]
+        zone_tuple = ZoneTuple(name=zone.name, price_deviation=price_deviation)
+        (markets_over_bid, markets_under_bid)[recent_price.price < bid].append(zone_tuple)
+
+    return min(markets_under_bid or markets_over_bid,
+               key=attrgetter('price_deviation')).name
+
+
+def optimize_spot_bid(ctx, instance_type, spot_bid):
+    """
+    Check whether the bid is sane and makes an effort to place the instance in a sensible zone.
+    """
+    spot_history = _get_spot_history(ctx, instance_type)
+    _check_spot_bid(spot_bid, spot_history)
+    zones = ctx.ec2.get_all_zones()
+    most_stable_zone = choose_spot_zone(zones, spot_bid, spot_history)
+    logger.info("Placing spot instances in zone %s.", most_stable_zone)
+    return most_stable_zone
+
+
+def _check_spot_bid(spot_bid, spot_history):
+    """
+    Prevents users from potentially over-paying for instances
+
+    Note: this checks over the whole region, not a particular zone
+
+    :param spot_bid: float
+
+    :type spot_history: list[SpotPriceHistory]
+
+    :raises UserError: if bid is > 2X the spot price's average
+
+    >>> from collections import namedtuple
+    >>> FauxHistory = namedtuple( "FauxHistory", [ "price", "availability_zone" ] )
+    >>> spot_data = [ FauxHistory( 0.1, "us-west-2a" ), \
+                      FauxHistory( 0.2, "us-west-2a" ), \
+                      FauxHistory( 0.3, "us-west-2b" ), \
+                      FauxHistory( 0.6, "us-west-2b" ) ]
+    >>> # noinspection PyProtectedMember
+    >>> _check_spot_bid( 0.1, spot_data )
+    >>> # noinspection PyProtectedMember
+
+    # >>> Box._check_spot_bid( 2, spot_data )
+    Traceback (most recent call last):
+    ...
+    UserError: Your bid $ 2.000000 is more than double this instance type's average spot price ($ 0.300000) over the last week
+    """
+    average = mean([datum.price for datum in spot_history])
+    if spot_bid > average * 2:
+        logger.warn("Your bid $ %f is more than double this instance type's average "
+                 "spot price ($ %f) over the last week", spot_bid, average)
+
+def _get_spot_history(ctx, instance_type):
+    """
+    Returns list of 1,000 most recent spot market data points represented as SpotPriceHistory
+    objects. Note: The most recent object/data point will be first in the list.
+
+    :rtype: list[SpotPriceHistory]
+    """
+
+    one_week_ago = datetime.datetime.now() - datetime.timedelta(days=7)
+    spot_data = ctx.ec2.get_spot_price_history(start_time=one_week_ago.isoformat(),
+                                               instance_type=instance_type,
+                                               product_description="Linux/UNIX")
+    spot_data.sort(key=attrgetter("timestamp"), reverse=True)
+    return spot_data
+
 ec2FullPolicy = dict(Version="2012-10-17", Statement=[
     dict(Effect="Allow", Resource="*", Action="ec2:*")])
 
diff --git a/src/toil/provisioners/aws/awsProvisioner.py b/src/toil/provisioners/aws/awsProvisioner.py
--- a/src/toil/provisioners/aws/awsProvisioner.py
+++ b/src/toil/provisioners/aws/awsProvisioner.py
@@ -374,9 +374,11 @@ def _addNodes(self, instances, numNodes, preemptable=False):
                           entrypoint='mesos-slave',
                           args=workerArgs.format(ip=self.leaderIP, preemptable=preemptable))
         userData = awsUserData.format(**workerData)
-        kwargs = {'key_name': self.keyName, 'security_groups': [self.clusterName],
+        kwargs = {'key_name': self.keyName,
+                  'security_groups': [self.clusterName],
                   'instance_type': self.instanceType.name,
-                  'user_data': userData, 'block_device_map': bdm,
+                  'user_data': userData,
+                  'block_device_map': bdm,
                   'instance_profile_arn': arn}
 
         instancesLaunched = []
@@ -387,13 +389,16 @@ def _addNodes(self, instances, numNodes, preemptable=False):
                                       spec=kwargs, num_instances=1)
         else:
             logger.info('Launching %s preemptable nodes', numNodes)
+            kwargs['placement'] = getSpotZone(self.spotBid, self.instanceType.name, self.ctx)
             # force generator to evaluate
             instancesLaunched = list(create_spot_instances(ec2=self.ctx.ec2,
-                                       price=self.spotBid,
-                                       image_id=self._discoverAMI(self.ctx),
-                                       tags={'clusterName': self.clusterName},
-                                       spec=kwargs,
-                                       num_instances=numNodes))
+                                                           price=self.spotBid,
+                                                           image_id=self._discoverAMI(self.ctx),
+                                                           tags={'clusterName': self.clusterName},
+                                                           spec=kwargs,
+                                                           num_instances=numNodes,
+                                                           tentative=True)
+                                     )
         wait_instances_running(self.ctx.ec2, instancesLaunched)
         logger.info('Launched %s new instance(s)', numNodes)
         return len(instancesLaunched)
@@ -470,10 +475,6 @@ def groupNotFound(e):
                 with attempt:
                     # the following authorizes all port access within the web security group
                     web.authorize(ip_protocol='tcp', from_port=0, to_port=65535, src_group=web)
-            for attempt in retry(predicate=groupNotFound, timeout=300):
-                with attempt:
-                    # open port 5050-5051 for mesos web interface
-                    web.authorize(ip_protocol='tcp', from_port=5050, to_port=5051, cidr_ip='0.0.0.0/0')
 
     @classmethod
     def _getProfileARN(cls, ctx):


Test patch: