Repository: DataBiosphere/toil

Issue:
Allow memory requirements to be specified as resource string



Discussion:
What exactly does this mean? Do you mean storage?

It is annoying to have to specify everything in bytes. It would be good to be able to specify things in other byte units, e.g. "10GB", "1000MB", etc. I assume

(oops, continuing) I assume... this is possible? I think Hannes mentioned a standard module for interpreting such strings.

This is in bd2k-python-lib

https://github.com/BD2KGenomics/bd2k-python-lib/blob/master/src/bd2k/util/humanize.py

We can either steal it from there or add bd2k-python-lib as a dependency. I'd prefer the latter even though we recently got rid of sonLib. The difference is that bd2k-python-lib is already on PyPI.


Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
     author='Benedict Paten',
     author_email='benedict@soe.usc.edu',
     url="https://github.com/BD2KGenomics/toil",
-    install_requires=[ ],
+    install_requires=['bd2k-python-lib>=1.7.dev1'],
     extras_require={
         'mesos': [
             'mesos.interface==0.22.0',
diff --git a/src/toil/common.py b/src/toil/common.py
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -27,6 +27,7 @@
 from argparse import ArgumentParser
 from optparse import OptionContainer, OptionGroup
 
+from bd2k.util.humanize import human2bytes
 from toil.lib.bioio import addLoggingOptions, getLogLevelString, system, absSymPath
 from toil.batchSystems.parasol import ParasolBatchSystem
 from toil.batchSystems.gridengine import GridengineBatchSystem
@@ -218,13 +219,13 @@ def createConfig(options):
     config.attrib["max_job_duration"] = str(float(options.maxJobDuration))
     config.attrib["batch_system"] = options.batchSystem
     config.attrib["job_time"] = str(float(options.jobTime))
-    config.attrib["max_log_file_size"] = str(int(options.maxLogFileSize))
-    config.attrib["default_memory"] = str(int(options.defaultMemory))
+    config.attrib["max_log_file_size"] = str(human2bytes(str(options.maxLogFileSize)))
+    config.attrib["default_memory"] = str(human2bytes(str(options.defaultMemory)))
     config.attrib["default_cpu"] = str(int(options.defaultCpu))
-    config.attrib["default_disk"] = str(int(options.defaultDisk))
+    config.attrib["default_disk"] = str(human2bytes(str(options.defaultDisk)))
     config.attrib["max_cpus"] = str(int(options.maxCpus))
-    config.attrib["max_memory"] = str(int(options.maxMemory))
-    config.attrib["max_disk"] = str(int(options.maxDisk))
+    config.attrib["max_memory"] = str(human2bytes(str(options.maxMemory)))
+    config.attrib["max_disk"] = str(human2bytes(str(options.maxDisk)))
     config.attrib["scale"] = str(float(options.scale))
     if options.bigBatchSystem is not None:
         config.attrib["big_batch_system"] = options.bigBatchSystem
diff --git a/src/toil/job.py b/src/toil/job.py
--- a/src/toil/job.py
+++ b/src/toil/job.py
@@ -30,6 +30,7 @@
 import uuid
 import time
 from toil.resource import ModuleDescriptor
+from bd2k.util.humanize import human2bytes
 
 try:
     import cPickle 
@@ -59,9 +60,12 @@ def __init__(self, memory=sys.maxint, cpu=sys.maxint, disk=sys.maxint):
         Memory is the maximum number of bytes of memory the job will
         require to run. Cpu is the number of cores required. 
         """
-        self.memory = memory
         self.cpu = cpu
-        self.disk = disk
+        # passing sys.maxint to human2bytes seems to result in some float imprecision, and returns a value 1
+        # larger than sys.maxint. We later assume that any value here not equal to sys.maxint must be the user's value
+        # so it is passed to the batch system, which cannot allocate that many resources.
+        self.memory = human2bytes(str(memory)) if memory!=sys.maxint else sys.maxint
+        self.disk = human2bytes(str(disk)) if disk!=sys.maxint else sys.maxint
         #Private class variables
         
         #See Job.addChild


Test patch: