Repository: DataBiosphere/toil

Issue:
LSF clusters have varying resource units -- toil needs to detect this and adapt accordingly
Use of the correct units is needed at 
https://github.com/BD2KGenomics/toil/blob/105aa24/src/toil/batchSystems/lsf.py#L38
> `mem = '' if mem is None else '-R "select[type==X86_64 && mem > ' + str(int(mem/ 1000000)) + '] rusage[mem=' + str(int(mem/ 1000000)) + ']" -M' + str(int(mem/ 1000000)) + '000'`

For example -- the trailing ` + '000'` needs to be left off when running on EBI's LSF clusters


Discussion:
One way to discover the units is with the following command
```
grep LSF_UNIT_FOR_LIMITS ${LSF_ENVDIR:-/etc}/lsf.conf | awk -F= '{print $2}'
```
Could some other LSF users verify this and report back what they see?

For example:
```
[mcrusoe@ebi-cli-001 /]$ grep LSF_UNIT_FOR_LIMITS ${LSF_ENVDIR:-/etc}/lsf.conf | awk -F= '{print $2}'
MB
```
Here's the comprehensive solution from `ipython-cluster-helper` for determining varying the LSF unit size

https://github.com/roryk/ipython-cluster-helper/blob/2042f4c/cluster_helper/lsf.py#L94
Other helpful LSF code at https://github.com/roryk/ipython-cluster-helper/blob/8a6a8a0/cluster_helper/cluster.py#L142
grep LSF_UNIT_FOR_LIMITS ${LSF_ENVDIR:-/etc}/lsf.conf | awk -F= '{print $2}'
GB



Resolving patch:
diff --git a/src/toil/batchSystems/abstractBatchSystem.py b/src/toil/batchSystems/abstractBatchSystem.py
--- a/src/toil/batchSystems/abstractBatchSystem.py
+++ b/src/toil/batchSystems/abstractBatchSystem.py
@@ -96,15 +96,7 @@ def issueBatchJob(self, jobNode):
         """
         Issues a job with the specified command to the batch system and returns a unique jobID.
 
-        :param str command: the string to run as a command,
-
-        :param int memory: int giving the number of bytes of memory the job needs to run
-
-        :param float cores: the number of cores needed for the job
-
-        :param int disk: int giving the number of bytes of disk space the job needs to run
-
-        :param bool preemptable: True if the job can be run on a preemptable node
+        :param jobNode a toil.job.JobNode
 
         :return: a unique jobID that can be used to reference the newly issued job
         :rtype: int
diff --git a/src/toil/batchSystems/lsf.py b/src/toil/batchSystems/lsf.py
--- a/src/toil/batchSystems/lsf.py
+++ b/src/toil/batchSystems/lsf.py
@@ -30,15 +30,32 @@
 
 from toil.batchSystems import MemoryString
 from toil.batchSystems.abstractBatchSystem import BatchSystemSupport
+from toil.batchSystems.lsfHelper import parse_memory, per_core_reservation
 
 logger = logging.getLogger( __name__ )
 
 
 
 def prepareBsub(cpu, mem):
-    mem = '' if mem is None else '-R "select[type==X86_64 && mem > ' + str(int(mem/ 1000000)) + '] rusage[mem=' + str(int(mem/ 1000000)) + ']" -M' + str(int(mem/ 1000000)) + '000'
-    cpu = '' if cpu is None else '-n ' + str(int(cpu))
-    bsubline = ["bsub", mem, cpu,"-cwd", ".", "-o", "/dev/null", "-e", "/dev/null"]
+    """
+    Make a bsub commandline to execute.
+
+    params:
+      cpu: number of cores needed
+      mem: number of bytes of memory needed
+    """
+    if mem:
+        if per_core_reservation():
+            mem = parse_memory(float(mem)/1024**3/int(cpu))
+        else:
+            mem = parse_memory(float(mem)/1024**3)
+        bsubMem = '-R "select[type==X86_64 && mem > ' + str(mem) + '] '\
+            'rusage[mem=' + str(mem) + '] -M' +str(mem)
+    else:
+        bsubMem = ''
+    cpuStr = '' if cpu is None else '-n ' + str(int(cpu))
+    bsubline = ["bsub", bsubMem, cpuStr,"-cwd", ".", "-o", "/dev/null", "-e",
+        "/dev/null"]
     lsfArgs = os.getenv('TOIL_LSF_ARGS')
     if lsfArgs:
         bsubline.extend(lsfArgs.split())
diff --git a/src/toil/batchSystems/lsfHelper.py b/src/toil/batchSystems/lsfHelper.py
new file mode 100755
--- /dev/null
+++ b/src/toil/batchSystems/lsfHelper.py
@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+# Adapted from https://github.com/roryk/ipython-cluster-helper
+# ipython-cluster-helper is licensed under the MIT license
+#
+# Copyright 2013-2017 "Rory Kirchne" <rory.kirchner@gmail.com> and contributors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import math
+import os
+import subprocess
+import fnmatch
+
+LSB_PARAMS_FILENAME = "lsb.params"
+LSF_CONF_FILENAME = "lsf.conf"
+LSF_CONF_ENV = ["LSF_CONFDIR", "LSF_ENVDIR"]
+DEFAULT_LSF_UNITS = "KB"
+DEFAULT_RESOURCE_UNITS = "MB"
+
+def find(basedir, string):
+    """
+    walk basedir and return all files matching string
+    """
+    matches = []
+    for root, dirnames, filenames in os.walk(basedir):
+        for filename in fnmatch.filter(filenames, string):
+            matches.append(os.path.join(root, filename))
+    return matches
+
+def find_first_match(basedir, string):
+    """
+    return the first file that matches string starting from basedir
+    """
+    matches = find(basedir, string)
+    return matches[0] if matches else matches
+
+def get_conf_file(filename, env):
+    conf_path = os.environ.get(env)
+    if not conf_path:
+        return None
+    conf_file = find_first_match(conf_path, filename)
+    return conf_file
+
+def apply_conf_file(fn, conf_filename):
+    for env in LSF_CONF_ENV:
+        conf_file = get_conf_file(conf_filename, env)
+        if conf_file:
+            with open(conf_file) as conf_handle:
+                value = fn(conf_handle)
+            if value:
+                return value
+    return None
+
+def per_core_reserve_from_stream(stream):
+    for k, v in tokenize_conf_stream(stream):
+        if k == "RESOURCE_RESERVE_PER_SLOT":
+            return v.upper()
+    return None
+
+def get_lsf_units_from_stream(stream):
+    for k, v in tokenize_conf_stream(stream):
+        if k == "LSF_UNIT_FOR_LIMITS":
+            return v
+    return None
+
+def tokenize_conf_stream(conf_handle):
+    """
+    convert the key=val pairs in a LSF config stream to tuples of tokens
+    """
+    for line in conf_handle:
+        if line.startswith("#"):
+            continue
+        tokens = line.split("=")
+        if len(tokens) != 2:
+            continue
+        yield (tokens[0].strip(), tokens[1].strip())
+
+def apply_bparams(fn):
+    """
+    apply fn to each line of bparams, returning the result
+    """
+    cmd = ["bparams", "-a"]
+    try:
+        output = subprocess.check_output(cmd)
+    except:
+        return None
+    return fn(output.split("\n"))
+
+def apply_lsadmin(fn):
+    """
+    apply fn to each line of lsadmin, returning the result
+    """
+    cmd = ["lsadmin", "showconf", "lim"]
+    try:
+        output = subprocess.check_output(cmd)
+    except:
+        return None
+    return fn(output.split("\n"))
+
+
+def get_lsf_units(resource=False):
+    """
+    check if we can find LSF_UNITS_FOR_LIMITS in lsadmin and lsf.conf
+    files, preferring the value in bparams, then lsadmin, then the lsf.conf file
+    """
+    lsf_units = apply_bparams(get_lsf_units_from_stream)
+    if lsf_units:
+        return lsf_units
+
+    lsf_units = apply_lsadmin(get_lsf_units_from_stream)
+    if lsf_units:
+        return lsf_units
+
+    lsf_units = apply_conf_file(get_lsf_units_from_stream, LSF_CONF_FILENAME)
+    if lsf_units:
+        return lsf_units
+
+    # -R usage units are in MB, not KB by default
+    if resource:
+        return DEFAULT_RESOURCE_UNITS
+    else:
+        return DEFAULT_LSF_UNITS
+
+def parse_memory(mem):
+    """
+    Parse memory parameter
+    """
+    lsf_unit = get_lsf_units(resource=True)
+    return convert_mb(float(mem) * 1024, lsf_unit)
+
+
+def per_core_reservation():
+    """
+    returns True if the cluster is configured for reservations to be per core,
+    False if it is per job
+    """
+    per_core = apply_bparams(per_core_reserve_from_stream)
+    if per_core:
+        if per_core.upper() == "Y":
+            return True
+        else:
+            return False
+
+    per_core = apply_lsadmin(per_core_reserve_from_stream)
+    if per_core:
+        if per_core.upper() == "Y":
+            return True
+        else:
+            return False
+
+    per_core = apply_conf_file(per_core_reserve_from_stream, LSB_PARAMS_FILENAME)
+    if per_core and per_core.upper() == "Y":
+        return True
+    else:
+        return False
+    return False
+
+def convert_mb(kb, unit):
+    UNITS = {"B": -2,
+             "KB": -1,
+             "MB": 0,
+             "GB": 1,
+             "TB": 2}
+    assert unit in UNITS, ("%s not a valid unit, valid units are %s."
+                           % (unit, UNITS.keys()))
+    return int(float(kb) / float(math.pow(1024, UNITS[unit])))
+
+
+
+if __name__ == "__main__":
+    print (get_lsf_units())
+    print (per_core_reservation())


Test patch: