Repository: DataBiosphere/toil

Issue:
resource parsing can't handle nums in scientific notation
A collaborator of mine is seeing this problem.

```
WARNING: toil.leader: R/N/jobzPnr5J:    Traceback (most recent call last):
WARNING: toil.leader: R/N/jobzPnr5J:      File "/neoantigen_prediction/protect/venv/lib/python2.7/site-packages/toil/worker.py", line 330, in main
WARNING: toil.leader: R/N/jobzPnr5J:        job._runner(jobWrapper=jobWrapper, jobStore=jobStore, fileStore=fileStore)
WARNING: toil.leader: R/N/jobzPnr5J:      File "/neoantigen_prediction/protect/venv/lib/python2.7/site-packages/toil/job.py", line 1043, in _runner
WARNING: toil.leader: R/N/jobzPnr5J:        returnValues = self._run(jobWrapper, fileStore)
WARNING: toil.leader: R/N/jobzPnr5J:      File "/neoantigen_prediction/protect/venv/lib/python2.7/site-packages/toil/job.py", line 991, in _run
WARNING: toil.leader: R/N/jobzPnr5J:        return self.run(fileStore)
WARNING: toil.leader: R/N/jobzPnr5J:      File "/neoantigen_prediction/protect/venv/lib/python2.7/site-packages/toil/job.py", line 1189, in run
WARNING: toil.leader: R/N/jobzPnr5J:        return self.addChildJobFn(userFunction, *self._args, **self._promisedKwargs).rv()
WARNING: toil.leader: R/N/jobzPnr5J:      File "/neoantigen_prediction/protect/venv/lib/python2.7/site-packages/toil/job.py", line 229, in addChildJobFn
WARNING: toil.leader: R/N/jobzPnr5J:        return self.addChild(JobFunctionWrappingJob(fn, *args, **kwargs))
WARNING: toil.leader: R/N/jobzPnr5J:      File "/neoantigen_prediction/protect/venv/lib/python2.7/site-packages/toil/job.py", line 1095, in __init__
WARNING: toil.leader: R/N/jobzPnr5J:        checkpoint=kwargs.pop("checkpoint") if "checkpoint" in kwargs else False)
WARNING: toil.leader: R/N/jobzPnr5J:      File "/neoantigen_prediction/protect/venv/lib/python2.7/site-packages/toil/job.py", line 69, in __init__
WARNING: toil.leader: R/N/jobzPnr5J:        self.disk = parse(disk)
WARNING: toil.leader: R/N/jobzPnr5J:      File "/neoantigen_prediction/protect/venv/lib/python2.7/site-packages/toil/job.py", line 67, in <lambda>
WARNING: toil.leader: R/N/jobzPnr5J:        parse = lambda x : x if x is None else human2bytes(str(x))
WARNING: toil.leader: R/N/jobzPnr5J:      File "/neoantigen_prediction/protect/venv/lib/python2.7/site-packages/bd2k/util/humanize.py", line 122, in human2bytes
WARNING: toil.leader: R/N/jobzPnr5J:        raise ValueError("can't interpret %r" % init)
WARNING: toil.leader: R/N/jobzPnr5J:    ValueError: can't interpret '1.15102817192e+11'
```

It arises from a resource request (in this case, disk) being passed a float in the scientific format.
The fix lies in modifying and updating bd2k-python-lib.

This PR should fix it: https://github.com/BD2KGenomics/bd2k-python-lib/pull/9



Discussion:
@hannes-ucsc could you look at this please?

What's the user script that leads to this? Stuffing float literals down human2bytes' throat is probably just treating a symptom.

This comes from ProTECT. The result of dynamic disk resource requests can be floats.

> The result of dynamic disk resource requests can be floats.

This should have been in the issue description.

d'oh!


Resolving patch:
diff --git a/src/toil/job.py b/src/toil/job.py
--- a/src/toil/job.py
+++ b/src/toil/job.py
@@ -63,10 +63,9 @@ def __init__(self, memory=None, cores=None, disk=None, preemptable=None, checkpo
         :type cache: int or string convertable by bd2k.util.humanize.human2bytes to an int
         :type memory: int or string convertable by bd2k.util.humanize.human2bytes to an int
         """
-        self.cores = cores
-        parse = lambda x : x if x is None else human2bytes(str(x))
-        self.memory = parse(memory)
-        self.disk = parse(disk)
+        self.cores = self._parseResource('cores', cores)
+        self.memory = self._parseResource('memory', memory)
+        self.disk = self._parseResource('disk', disk)
         self.checkpoint = checkpoint
         self.preemptable = preemptable
         #Private class variables
@@ -92,6 +91,49 @@ def __init__(self, memory=None, cores=None, disk=None, preemptable=None, checkpo
         self._rvs = collections.defaultdict(list)
         self._promiseJobStore = None
 
+    @staticmethod
+    def _parseResource(name, value):
+        """
+        Parse a Toil job's resource requirement value and apply resource-specific type checks. If the
+        value is a string, a binary or metric unit prefix in it will be evaluated and the
+        corresponding integral value will be returned.
+
+        :param str name: The name of the resource
+
+        :param None|str|float|int value: The resource value
+
+        :rtype: int|float|None
+
+        >>> Job._parseResource('cores', None)
+        >>> Job._parseResource('cores', 1), Job._parseResource('disk', 1), \
+        Job._parseResource('memory', 1)
+        (1, 1, 1)
+        >>> Job._parseResource('cores', '1G'), Job._parseResource('disk', '1G'), \
+        Job._parseResource('memory', '1G')
+        (1073741824, 1073741824, 1073741824)
+        >>> Job._parseResource('cores', 1.1)
+        1.1
+        >>> Job._parseResource('disk', 1.1)
+        Traceback (most recent call last):
+        ...
+        TypeError: The 'disk' requirement does not accept values that are of <type 'float'>
+        >>> Job._parseResource('memory', object())
+        Traceback (most recent call last):
+        ...
+        TypeError: The 'memory' requirement does not accept values that are of <type 'object'>
+        """
+        assert name in ('memory', 'disk', 'cores')
+        if value is None:
+            return value
+        elif isinstance(value, str):
+            value = human2bytes(value)
+        if isinstance(value, int):
+            return value
+        elif isinstance(value, float) and name == 'cores':
+            return value
+        else:
+            raise TypeError("The '%s' requirement does not accept values that are of %s"
+                            % (name, type(value)))
 
     def run(self, fileStore):
         """


Test patch:
diff --git a/src/toil/test/batchSystems/batchSystemTest.py b/src/toil/test/batchSystems/batchSystemTest.py
--- a/src/toil/test/batchSystems/batchSystemTest.py
+++ b/src/toil/test/batchSystems/batchSystemTest.py
@@ -636,15 +636,15 @@ def testNestedResourcesDoNotBlock(self):
 
         # Should block off 50% of memory while waiting for it's 3 cores
         firstJobChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=0,
-                                   writeVal='fJC', cores=3, memory=(availableMemory/2), disk='1M')
+                                   writeVal='fJC', cores=3, memory=int(availableMemory/2), disk='1M')
 
         # These two shouldn't be able to run before B because there should be only
         # (50% of memory - 1M) available (firstJobChild should be blocking 50%)
         secondJobChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5,
-                                    writeVal='sJC', cores=2, memory=(availableMemory/1.5),
+                                    writeVal='sJC', cores=2, memory=int(availableMemory/1.5),
                                     disk='1M')
         secondJobGrandChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5,
-                                         writeVal='sJGC', cores=2, memory=(availableMemory/1.5),
+                                         writeVal='sJGC', cores=2, memory=int(availableMemory/1.5),
                                          disk='1M')
 
         root.addChild(blocker)