Repository: DataBiosphere/toil

Issue:
GCE Provisioner TODO

- [ ]  - libcloud ex_create_multiple_nodes bug that doesn't allow multi nodes
     - submit issue?
     - Or just keep class with machine name as uuid?
- [ ] copy SSE keys for server side encryption
- [ ] rename to Google (instead of GCE?)



Discussion:

Resolving patch:
diff --git a/src/toil/provisioners/gceProvisioner.py b/src/toil/provisioners/gceProvisioner.py
--- a/src/toil/provisioners/gceProvisioner.py
+++ b/src/toil/provisioners/gceProvisioner.py
@@ -312,10 +312,10 @@ def launchCluster(self, leaderNodeType, leaderSpotBid, nodeTypes, preemptableNod
         # Throws an error if cluster exists
         self.instanceGroup = driver.ex_create_instancegroup(clusterName, zone)
 
-        preemptible = False
+        preemptable = False
         if leaderSpotBid:
             logger.info('Launching preemptable leader')
-            preemptible = True
+            preemptable = True
         else:
             logger.info('Launching non-preemptable leader')
 
@@ -336,7 +336,7 @@ def launchCluster(self, leaderNodeType, leaderSpotBid, nodeTypes, preemptableNod
                                 ex_subnetwork=vpcSubnet,
                                 ex_disks_gce_struct = [disk],
                                 description=self.tags,
-                                ex_preemptible=preemptible)
+                                ex_preemptible=preemptable)
 
         self.instanceGroup.add_instances([leader])
 
@@ -535,11 +535,21 @@ def addNodes(self, nodeType, numNodes, preemptable):
     def getProvisionedWorkers(self, nodeType, preemptable):
         entireCluster = self._getNodesInCluster(nodeType=nodeType)
         logger.debug('All nodes in cluster: %s', entireCluster)
-        workerInstances = [i for i in entireCluster if i.private_ips[0] != self.leaderIP]
+        workerInstances = []
+        for instance in entireCluster:
+            scheduling = instance.extra.get('scheduling')
+            # If this field is not found in the extra meta-data, assume the node is not preemptable.
+            if scheduling and scheduling.get('preemptible', False) != preemptable:
+                continue
+            isWorker = True
+            for ip in instance.private_ips:
+                if ip == self.leaderIP:
+                    isWorker = False
+                    break # don't include the leader
+            if isWorker and instance.state == 'running':
+                workerInstances.append(instance)
+
         logger.debug('All workers found in cluster: %s', workerInstances)
-        # TODO: get spot workers
-        #workerInstances = [i for i in workerInstances if preemptable != (i.spot_instance_request_id is None)]
-        #logger.debug('%spreemptable workers found in cluster: %s', 'non-' if not preemptable else '', workerInstances)
         return [Node(publicIP=i.public_ips[0], privateIP=i.private_ips[0],
                      name=i.name, launchTime=i.created_at, nodeType=i.size,
                      preemptable=preemptable)
@@ -686,6 +696,7 @@ def ex_create_multiple_nodes(
         """
          Monkey patch to gce.py in libcloud to allow disk and images to be specified.
          Also changed name to a uuid below.
+         The prefix 'wp' identifies preemptible nodes and 'wn' non-preemtible nodes.
         """
         # if image and ex_disks_gce_struct:
         #    raise ValueError("Cannot specify both 'image' and "
@@ -739,7 +750,8 @@ def ex_create_multiple_nodes(
         status_list = []
 
         for i in range(number):
-            name = 'w' + bytes(uuid.uuid4()) #'%s-%03d' % (base_name, i)
+            name = 'wp' if ex_preemptible else 'wn'
+            name += bytes(uuid.uuid4()) #'%s-%03d' % (base_name, i)
             status = {'name': name, 'node_response': None, 'node': None}
             status_list.append(status)
 
@@ -770,8 +782,6 @@ def ex_create_multiple_nodes(
         return node_list
 
 
-    ## UNCHANGED CLASSES
-
     @classmethod
     def _waitForSSHKeys(cls, instanceIP, keyName='core'):
         # the propagation of public ssh keys vs. opening the SSH port is racey, so this method blocks until
@@ -779,7 +789,7 @@ def _waitForSSHKeys(cls, instanceIP, keyName='core'):
         startTime = time.time()
         while True:
             if time.time() - startTime > cls.maxWaitTime:
-                logger.error("Key propagation failed on machine with ip" % instanceIP)
+                logger.error("Key propagation failed on machine with ip %s" % instanceIP)
                 return False
             try:
                 logger.info('Attempting to establish SSH connection...')
@@ -800,7 +810,7 @@ def _waitForDockerDaemon(cls, ip_address, keyName='core'):
         startTime = time.time()
         while True:
             if time.time() - startTime > cls.maxWaitTime:
-                logger.error("Docker daemon failed to start on machine with ip" % ip_address)
+                logger.error("Docker daemon failed to start on machine with ip %s" % ip_address)
                 return False
             try:
                 output = cls._sshInstance(ip_address, '/usr/bin/ps', 'aux', sshOptions=['-oBatchMode=yes'], user=keyName)
@@ -828,7 +838,9 @@ def _waitForAppliance(cls, ip_address, role, keyName='core'):
         startTime = time.time()
         while True:
             if time.time() - startTime > cls.maxWaitTime:
-                logger.error("Appliance failed to start on machine with ip" % ip_address)
+                logger.error("Appliance failed to start on machine with ip %s" % ip_address)
+                logger.error("Check if the appliance is valid, e.g. check if the environment variable"
+                              " TOIL_APPLIANCE_SELF is set correctly and the container exists.")
                 return False
             try:
                 output = cls._sshInstance(ip_address, '/usr/bin/docker', 'ps',


Test patch:
diff --git a/src/toil/test/provisioners/gceProvisionerTest.py b/src/toil/test/provisioners/gceProvisionerTest.py
--- a/src/toil/test/provisioners/gceProvisionerTest.py
+++ b/src/toil/test/provisioners/gceProvisionerTest.py
@@ -172,7 +172,7 @@ def _test(self, preemptableJobs=False):
 
 
 
-@pytest.mark.timeout(1200)
+@pytest.mark.timeout(1600)
 class GCEAutoscaleTest(AbstractGCEAutoscaleTest):
 
     def __init__(self, name):
@@ -218,11 +218,13 @@ def testAutoScale(self):
     @needs_google
     def testSpotAutoScale(self):
         self.instanceTypes = ["n1-standard-2:%f" % self.spotBid]
-        self.numWorkers = ['2']
+        # Some spot workers have a stopped state after being started, strangely.
+        # This could be the natural preemption process, but it seems too rapid.
+        self.numWorkers = ['3'] # Try 3 to account for a stopped node.
         self._test(preemptableJobs=True)
 
 
-@pytest.mark.timeout(1200)
+@pytest.mark.timeout(1600)
 class GCEStaticAutoscaleTest(GCEAutoscaleTest):
     """
     Runs the tests on a statically provisioned cluster with autoscaling enabled.