Repository: DataBiosphere/toil

Issue:
Error running the sort.py example with autoscaling on AWS.

Hi there,

When i tried to run the `sort.py` example with autoscaling on AWS by following the steps in the Documentation, after i ran `python /tmp/sort.py aws:us-west-2:autoscaling-sort-jobstore --provisioner aws --nodeTypes c3.large --batchSystem mesos --mesosMaster <private-IP>:5050 --logLevel DEBUG`, the error message was as following:
```
Traceback (most recent call last):
  File "/tmp/sort.py", line 219, in <module>
    main()
  File "/tmp/sort.py", line 214, in main
    memory=sortMemory))
  File "/usr/local/lib/python2.7/dist-packages/toil/common.py", line 727, in start
    return self._runMainLoop(rootJobGraph)
  File "/usr/local/lib/python2.7/dist-packages/toil/common.py", line 997, in _runMainLoop
    jobCache=self._jobCache).run()
  File "/usr/local/lib/python2.7/dist-packages/toil/leader.py", line 147, in __init__
    self.clusterScaler = None if self.provisioner is None else ClusterScaler(self.provisioner, self, self.config)
  File "/usr/local/lib/python2.7/dist-packages/toil/provisioners/clusterScaler.py", line 254, in __init__
    require(sum(config.maxNodes) > 0, 'Not configured to create nodes of any type.')
  File "/usr/local/lib/python2.7/dist-packages/bd2k/util/exceptions.py", line 81, in require
    raise RequirementError( message)
bd2k.util.exceptions.RequirementError: Not configured to create nodes of any type.

```

BTW, the new `--nodeType` flag has been changed to `--nodeTypes`, but no matter i used `--nodeTypes c3.large` or `--nodeType c3.large`, the same error was raised. 

Thank you,

bettie


Discussion:

Hi again,

BTW, even if i removed `--nodeTypes` this flag, the same error was raised.
```
ip-172-31-8-166.us-west-2.compute.internal 2017-10-04 23:35:37,435 MainThread DEBUG toil.batchSystems.mesos.batchSystem: Joined Mesos driver
2017-10-04 23:35:37,435 - toil.batchSystems.mesos.batchSystem - DEBUG - Joined Mesos driver
ip-172-31-8-166.us-west-2.compute.internal 2017-10-04 23:35:37,436 MainThread DEBUG toil.common: ... finished shutting down the batch system in 0.00110697746277 seconds.
2017-10-04 23:35:37,436 - toil.common - DEBUG - ... finished shutting down the batch system in 0.00110697746277 seconds.
Traceback (most recent call last):
  File "/tmp/sort.py", line 219, in <module>
    main()
  File "/tmp/sort.py", line 214, in main
    memory=sortMemory))
  File "/usr/local/lib/python2.7/dist-packages/toil/common.py", line 727, in start
    return self._runMainLoop(rootJobGraph)
  File "/usr/local/lib/python2.7/dist-packages/toil/common.py", line 997, in _runMainLoop
    jobCache=self._jobCache).run()
  File "/usr/local/lib/python2.7/dist-packages/toil/leader.py", line 147, in __init__
    self.clusterScaler = None if self.provisioner is None else ClusterScaler(self.provisioner, self, self.config)
  File "/usr/local/lib/python2.7/dist-packages/toil/provisioners/clusterScaler.py", line 254, in __init__
    require(sum(config.maxNodes) > 0, 'Not configured to create nodes of any type.')
  File "/usr/local/lib/python2.7/dist-packages/bd2k/util/exceptions.py", line 81, in require
    raise RequirementError( message)
bd2k.util.exceptions.RequirementError: Not configured to create nodes of any type.
```

Thank you,

bettie
This works for me. Try localhost:5050 for --mesosMaster
You need the --maxNodes flag as well, otherwise it defaults to zero nodes of each type. We could change the default to 1, maybe.

Resolving patch:
diff --git a/src/toil/common.py b/src/toil/common.py
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -88,7 +88,7 @@ def __init__(self):
         self.nodeTypes = []
         self.nodeOptions = None
         self.minNodes = None
-        self.maxNodes = []
+        self.maxNodes = [10]
         self.alphaPacking = 0.8
         self.betaInertia = 1.2
         self.scaleInterval = 30
@@ -376,12 +376,13 @@ def _addOptions(addGroupFn, config):
     addOptionFn('--minNodes', default=None, 
                  help="Mininum number of nodes of each type in the cluster, if using "
                               "auto-scaling. This should be provided as a comma-separated "
-                              "list of the same length as the list of node types.")
+                              "list of the same length as the list of node types. default=0")
 
     addOptionFn('--maxNodes', default=None,
                 help="Maximum number of nodes of each type in the cluster, if using "
-                "autoscaling. This should be provided as a comma-separated list of the same "
-                "length as the list of node types.")
+                "autoscaling, provided as a comma-separated list. The first value is used "
+                "as a default if the list length is less than the number of nodeTypes. "
+                "default=%s" % config.maxNodes[0])
 
     # TODO: DESCRIBE THE FOLLOWING TWO PARAMETERS
     addOptionFn("--alphaPacking", dest="alphaPacking", default=None,
diff --git a/src/toil/provisioners/clusterScaler.py b/src/toil/provisioners/clusterScaler.py
--- a/src/toil/provisioners/clusterScaler.py
+++ b/src/toil/provisioners/clusterScaler.py
@@ -366,9 +366,11 @@ def __init__(self, scaler):
         self.jobShapes = RecentJobShapes(scaler.config, nodeShape=self.nodeShapes[0])
         # Minimum/maximum number of either preemptable or non-preemptable nodes in the cluster
         minNodes = scaler.config.minNodes
-        maxNodes = scaler.config.maxNodes
         if minNodes is None:
             minNodes = [0 for node in self.nodeTypes]
+        maxNodes = scaler.config.maxNodes
+        while len(maxNodes) < len(self.nodeTypes):
+            maxNodes.append(maxNodes[0])
         self.minNodes = dict(zip(self.nodeShapes, minNodes))
         self.maxNodes = dict(zip(self.nodeShapes, maxNodes))
 


Test patch:
diff --git a/src/toil/test/provisioners/clusterScalerTest.py b/src/toil/test/provisioners/clusterScalerTest.py
--- a/src/toil/test/provisioners/clusterScalerTest.py
+++ b/src/toil/test/provisioners/clusterScalerTest.py
@@ -104,7 +104,7 @@ def _testClusterScaling(self, config, numJobs, numPreemptableJobs, jobShape):
             logger.info("Creating test jobs")
             list(map(lambda x: mock.addJob(jobShape=jobShape), list(range(numJobs))))
             list(map(lambda x: mock.addJob(jobShape=jobShape, preemptable=True), list(range(numPreemptableJobs))))
-    
+
             # Add some completed jobs
             for preemptable in (True, False):
                 if preemptable and numPreemptableJobs > 0 or not preemptable and numJobs > 0:
@@ -119,7 +119,7 @@ def _testClusterScaling(self, config, numJobs, numPreemptableJobs, jobShape):
                                      command=None,
                                      jobName='testClusterScaling', unitName='')
                         clusterScaler.addCompletedJob(iJ, random.choice(list(range(1, x.wallTime))))
-    
+
             logger.info("Waiting for jobs to be processed")
             startTime = time.time()
             # Wait while the cluster the process chunks through the jobs
@@ -173,7 +173,7 @@ def testClusterScaling(self):
         config.scaleInterval = 3
 
         self._testClusterScaling(config, numJobs=100, numPreemptableJobs=0, jobShape=config.nodeTypes[0])
-        
+
     @slow
     def testClusterScalingMultipleNodeTypes(self):
 
@@ -182,7 +182,7 @@ def testClusterScalingMultipleNodeTypes(self):
         largeNode = Shape(20, 20, 10, 10, False)
 
         numJobs = 100
-        
+
         config = Config()
 
         # Make defaults dummy values
@@ -198,13 +198,13 @@ def testClusterScalingMultipleNodeTypes(self):
         #Make sure the node types don't have to be ordered
         config.nodeTypes = [largeNode, smallNode, mediumNode]
         config.minNodes = [0, 0, 0]
-        config.maxNodes = [10, 10, 10]
+        config.maxNodes = [10, 10] # test expansion of this list
 
         # Algorithm parameters
         config.alphaPacking = 0.8
         config.betaInertia = 1.2
         config.scaleInterval = 3
-        
+
         mock = MockBatchSystemAndProvisioner(config, secondsPerJob=2.0)
         clusterScaler = ClusterScaler(mock, mock, config)
         clusterScaler.start()
@@ -214,7 +214,7 @@ def testClusterScalingMultipleNodeTypes(self):
             #Add small jobs
             list(map(lambda x: mock.addJob(jobShape=smallNode), list(range(numJobs))))
             list(map(lambda x: mock.addJob(jobShape=mediumNode), list(range(numJobs))))
-            
+
             #Add medium completed jobs
             for i in range(1000):
                 iJ = JobNode(jobStoreID=1,
@@ -242,7 +242,7 @@ def testClusterScalingMultipleNodeTypes(self):
         self.assertTrue(mock.maxWorkers[mediumNode] > 0)
 
         self.assertEqual(mock.maxWorkers[largeNode], 0)
-        
+
     @slow
     def testClusterScalingWithPreemptableJobs(self):
         """
@@ -311,7 +311,7 @@ def shutDown(self):
         self.running = False
         self.leaderThread.join()
 
-    
+
     # Stub out all AbstractBatchSystem methods since they are never called
 
     for name, value in iteritems(AbstractBatchSystem.__dict__):
@@ -381,7 +381,7 @@ def getNumberOfJobsIssued(self, preemptable=None):
 
     def getJobs(self):
         return self.jobBatchSystemIDToIssuedJob.values()
-    
+
     # AbstractScalableBatchSystem functionality
 
     def getNodes(self, preemptable=False, timeout=None):
@@ -418,7 +418,7 @@ def _leaderFn(self):
             if updatedJobID:
                 del self.jobBatchSystemIDToIssuedJob[updatedJobID]
             time.sleep(0.1)
-            
+
 
     def _addNodes(self, numNodes, nodeType, preemptable=False):
         nodeShape = self.getNodeShape(nodeType=nodeType, preemptable=preemptable)
@@ -474,4 +474,4 @@ def getNumberOfNodes(self, nodeType=None, preemptable=None):
             return len(self.workers[nodeShape])
         else:
             return len(self.nodesToWorker)
-                
+