Repository: DataBiosphere/toil

Issue:
Can't specify disk requirement without specifying cache also
Running the following user script

```
import sys
from toil.job import Job

def hello( name ):
    return "Hello, %s!" % name

if __name__ == '__main__':
    options = Job.Runner.getDefaultArgumentParser().parse_args()
    job = Job.wrapFn( hello, "world", cores=1, memory=1e6, disk=1e6 )
    result = Job.Runner.startToil( job, options )
    assert result == 'Hello, world!'
```

using this command line

```
python2.7 hello_world.py ./foo
```

Produces the following exception.

```
Traceback (most recent call last):
  File "hello_world.py", line 10, in <module>
    result = Job.Runner.startToil( job, options )
  File "/usr/local/lib/python2.7/dist-packages/toil/job.py", line 436, in startToil
    rootJob = job._serialiseFirstJob(jobStore)
  File "/usr/local/lib/python2.7/dist-packages/toil/job.py", line 1246, in _serialiseFirstJob
    predecessorNumber=0)
  File "/usr/local/lib/python2.7/dist-packages/toil/job.py", line 1104, in _createEmptyJobForJob
    " than the disk requirement for the job! (disk: %s)" % (cache, disk))
RuntimeError: Trying to allocate a cache (cache: 2147483648.0) larger than the disk requirement for the job! (disk: 1000000)
```

The user shouldn't need to specify cache. It should have a sensible default derived from the disk requirement.

Can't specify disk requirement without specifying cache also
Running the following user script

```
import sys
from toil.job import Job

def hello( name ):
    return "Hello, %s!" % name

if __name__ == '__main__':
    options = Job.Runner.getDefaultArgumentParser().parse_args()
    job = Job.wrapFn( hello, "world", cores=1, memory=1e6, disk=1e6 )
    result = Job.Runner.startToil( job, options )
    assert result == 'Hello, world!'
```

using this command line

```
python2.7 hello_world.py ./foo
```

Produces the following exception.

```
Traceback (most recent call last):
  File "hello_world.py", line 10, in <module>
    result = Job.Runner.startToil( job, options )
  File "/usr/local/lib/python2.7/dist-packages/toil/job.py", line 436, in startToil
    rootJob = job._serialiseFirstJob(jobStore)
  File "/usr/local/lib/python2.7/dist-packages/toil/job.py", line 1246, in _serialiseFirstJob
    predecessorNumber=0)
  File "/usr/local/lib/python2.7/dist-packages/toil/job.py", line 1104, in _createEmptyJobForJob
    " than the disk requirement for the job! (disk: %s)" % (cache, disk))
RuntimeError: Trying to allocate a cache (cache: 2147483648.0) larger than the disk requirement for the job! (disk: 1000000)
```

The user shouldn't need to specify cache. It should have a sensible default derived from the disk requirement.



Discussion:
Adding `cache=1e6` prevents the exception.

Maybe modifying [these two lines in job.py](https://github.com/BD2KGenomics/toil/blob/master/src/toil/job.py#L1099-L1100) to

```
cache=(self.cache if self.cache is not None
              else min(float(jobStore.config.defaultCache), disk))
```

will help?

I.e. if cache isn't provided, use the lower of disk, and default cache.

Adding `cache=1e6` prevents the exception.

Maybe modifying [these two lines in job.py](https://github.com/BD2KGenomics/toil/blob/master/src/toil/job.py#L1099-L1100) to

```
cache=(self.cache if self.cache is not None
              else min(float(jobStore.config.defaultCache), disk))
```

will help?

I.e. if cache isn't provided, use the lower of disk, and default cache.


Resolving patch:
diff --git a/src/toil/common.py b/src/toil/common.py
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -19,6 +19,7 @@
 import sys
 import cPickle
 from argparse import ArgumentParser
+from bd2k.util.humanize import bytes2human
 
 from toil.lib.bioio import addLoggingOptions, getLogLevelString, absSymPath
 from toil.batchSystems.parasol import ParasolBatchSystem
@@ -59,7 +60,7 @@ def __init__(self):
         self.defaultMemory = 2147483648
         self.defaultCores = 1
         self.defaultDisk = 2147483648
-        self.defaultCache = 2147483648 #Cache is 2GB
+        self.defaultCache = self.defaultDisk
         self.maxCores = sys.maxint
         self.maxMemory = sys.maxint
         self.maxDisk = sys.maxint
@@ -88,10 +89,10 @@ def setOption(varName, parsingFn=None, checkFn=None):
             #If options object has the option "varName" specified
             #then set the "varName" attrib to this value in the config object
             x = getattr(options, varName, None)
-            if x != None:
-                if parsingFn != None:
+            if x is not None:
+                if parsingFn is not None:
                     x = parsingFn(x)
-                if checkFn != None:
+                if checkFn is not None:
                     try:
                         checkFn(x)
                     except AssertionError:
@@ -231,27 +232,41 @@ def _addOptions(addGroupFn, config):
     #Resource requirements
     #
     addOptionFn = addGroupFn("toil options for cores/memory requirements",
-                             "The options to specify default cores/memory requirements (if not specified by the jobs themselves), and to limit the total amount of memory/cores requested from the batch system.")
-    addOptionFn("--defaultMemory", dest="defaultMemory", default=None,
-                      help=("The default amount of memory to request for a job (in bytes), "
-                            "by default is 2^31 = 2 gigabytes, default=%s" % config.defaultMemory))
-    addOptionFn("--defaultCores", dest="defaultCores", default=None,
-                      help="The default number of cpu cores to dedicate a job. default=%s" % config.defaultCores)
-    addOptionFn("--defaultDisk", dest="defaultDisk", default=None,
-                      help="The default amount of disk space to dedicate a job (in bytes). default=%s" % config.defaultDisk)
-    addOptionFn("--defaultCache", dest="defaultCache", default=None,
-                help=("The default amount of disk space to use in caching "
-                      "files shared between jobs. This must be less than the disk requirement "
-                      "for the job default=%s" % config.defaultCache))
-    addOptionFn("--maxCores", dest="maxCores", default=None,
-                      help=("The maximum number of cpu cores to request from the batch system at any "
-                            "one time. default=%s" % config.maxCores))
-    addOptionFn("--maxMemory", dest="maxMemory", default=None,
-                      help=("The maximum amount of memory to request from the batch \
-                      system at any one time. default=%s" % config.maxMemory))
-    addOptionFn("--maxDisk", dest="maxDisk", default=None,
-                      help=("The maximum amount of disk space to request from the batch \
-                      system at any one time. default=%s" % config.maxDisk))
+                             "The options to specify default cores/memory requirements (if not "
+                             "specified by the jobs themselves), and to limit the total amount of "
+                             "memory/cores requested from the batch system.")
+    addOptionFn('--defaultMemory', dest='defaultMemory', default=None, metavar='INT',
+                help='The default amount of memory to request for a job. Only applicable to jobs '
+                     'that do not specify an explicit value for this requirement. Standard '
+                     'suffixes like K, Ki, M, Mi, G or Gi are supported. Default is %s' %
+                     bytes2human( config.defaultMemory, symbols='iec' ))
+    addOptionFn('--defaultCores', dest='defaultCores', default=None, metavar='FLOAT',
+                help='The default number of CPU cores to dedicate a job. Only applicable to jobs '
+                     'that do not specify an explicit value for this requirement. Fractions of a '
+                     'core (for example 0.1) are supported on some batch systems, namely Mesos '
+                     'and singleMachine. Default is %.1f ' % config.defaultCores)
+    addOptionFn('--defaultDisk', dest='defaultDisk', default=None, metavar='INT',
+                help='The default amount of disk space to dedicate a job. Only applicable to jobs '
+                     'that do not specify an explicit value for this requirement. Standard '
+                     'suffixes like K, Ki, M, Mi, G or Gi are supported. Default is %s' %
+                     bytes2human( config.defaultDisk, symbols='iec' ))
+    addOptionFn('--defaultCache', dest='defaultCache', default=None, metavar='INT',
+                help='The default amount of disk space to use for caching files shared between '
+                     'jobs. Only applicable to jobs that do not specify an explicit value for '
+                     'this requirement. Standard suffixes like K, Ki, M, Mi, G or Gi are '
+                     'supported. Default is %s' % bytes2human( config.defaultCache, symbols='iec' ))
+    addOptionFn('--maxCores', dest='maxCores', default=None, metavar='INT',
+                help='The maximum number of CPU cores to request from the batch system at any one '
+                     'time. Standard suffixes like K, Ki, M, Mi, G or Gi are supported. Default '
+                     'is %s' % bytes2human(config.maxCores, symbols='iec'))
+    addOptionFn('--maxMemory', dest='maxMemory', default=None, metavar='INT',
+                help="The maximum amount of memory to request from the batch system at any one "
+                     "time. Standard suffixes like K, Ki, M, Mi, G or Gi are supported. Default "
+                     "is %s" % bytes2human( config.maxMemory, symbols='iec'))
+    addOptionFn('--maxDisk', dest='maxDisk', default=None, metavar='INT',
+                help='The maximum amount of disk space to request from the batch system at any '
+                     'one time. Standard suffixes like K, Ki, M, Mi, G or Gi are supported. '
+                     'Default is %s' % bytes2human(config.maxDisk, symbols='iec'))
 
     #
     #Retrying/rescuing jobs
diff --git a/src/toil/job.py b/src/toil/job.py
--- a/src/toil/job.py
+++ b/src/toil/job.py
@@ -30,6 +30,7 @@
 import inspect
 from threading import Thread, Semaphore, Event
 from Queue import Queue, Empty
+from bd2k.util.expando import Expando
 from bd2k.util.humanize import human2bytes
 from io import BytesIO
 from toil.resource import ModuleDescriptor
@@ -64,7 +65,7 @@ def __init__(self, memory=None, cores=None, disk=None, cache=None):
         :type memory: int or string convertable by bd2k.util.humanize.human2bytes to an int
         """
         self.cores = cores
-        parse = lambda x : human2bytes(str(x)) if x is not None else x
+        parse = lambda x : x if x is None else human2bytes(str(x))
         self.memory = parse(memory)
         self.disk = parse(disk)
         self.cache = parse(cache)
@@ -1085,28 +1086,35 @@ def removePredecessor(job):
             t2.addFollowOnJobFn(deleteFileStoreIDs, map(lambda i : i.stopFileStoreID, self._services))
             self._services = [] #Defensive
 
-    def _createEmptyJobForJob(self, jobStore, command=None,
-                                 predecessorNumber=0):
+    def _createEmptyJobForJob(self, jobStore, command=None, predecessorNumber=0):
         """
         Create an empty job for the job.
         """
-        memory=(self.memory if self.memory is not None
-               else float(jobStore.config.defaultMemory))
-        cores=(self.cores if self.cores is not None
-               else float(jobStore.config.defaultCores))
-        disk=(self.disk if self.disk is not None
-              else float(jobStore.config.defaultDisk))
-        cache=(self.cache if self.cache is not None
-              else float(jobStore.config.defaultCache))
-        
-        if cache > disk:
-            raise RuntimeError("Trying to allocate a cache (cache: %s) larger"
-                               " than the disk requirement for the job! (disk: %s)" % (cache, disk))
+        requirements = self.effectiveRequirements(jobStore.config)
+        del requirements.cache
+        return jobStore.create(command=command, predecessorNumber=predecessorNumber, **requirements)
+
+    def effectiveRequirements(self, config):
+        """
+        Determine and validate the effective requirements for this job, substituting a missing
+        explict requirement with a default from the configuration.
+
+        :rtype: Expando
+        :return: a dictionary/object hybrid with one entry/attribute for each requirement
+        """
+        requirements = Expando(
+            memory=float(config.defaultMemory) if self.memory is None else self.memory,
+            cores=float(config.defaultCores) if self.cores is None else self.cores,
+            disk=float(config.defaultDisk) if self.disk is None else self.disk)
+        if self.cache is None:
+            requirements.cache = min(requirements.disk, float(config.defaultCache))
+        else:
+            requirements.cache = self.cache
+        if requirements.cache > requirements.disk:
+            raise RuntimeError("Trying to allocate a cache ({cache}) larger than the disk "
+                               "requirement for the job ({disk})".format(**requirements))
+        return requirements
 
-        return jobStore.create(command=command,
-                               memory=memory, cores=cores, disk=disk,
-                               predecessorNumber=predecessorNumber)
-        
     def _makeJobWrappers(self, jobWrapper, jobStore):
         """
         Creates a job for each job in the job graph, recursively.
diff --git a/src/toil/worker.py b/src/toil/worker.py
--- a/src/toil/worker.py
+++ b/src/toil/worker.py
@@ -266,7 +266,7 @@ def badWorker():
                     job = Job._loadJob(jobWrapper.command, jobStore)
                     
                     #Cleanup the cache from the previous job
-                    cleanCacheFn(job.cache if job.cache != None else config.defaultCache)
+                    cleanCacheFn(job.effectiveRequirements(jobStore.config).cache)
                     
                     #Create a fileStore object for the job
                     fileStore = Job.FileStore(jobStore, jobWrapper, localTempDir, 


Test patch: