Repository: DataBiosphere/toil

Issue:
Memory Limit value Zero
https://github.com/DataBiosphere/toil/blob/f159848b4e0c06dc27cccd6c3389dd6efbbad750/src/toil/batchSystems/lsf.py#L252

```
Traceback (most recent call last):
  File "/nfs/production/panda/ensembl/thiagogenez/usr/linuxbrew/Cellar/cactus/HEAD-71f6430/libexec/lib/python3.9/site-packages/toil/batchSystems/abstractGridEngineBatchSystem.py", line 222, in run
    while self._runStep():
  File "/nfs/production/panda/ensembl/thiagogenez/usr/linuxbrew/Cellar/cactus/HEAD-71f6430/libexec/lib/python3.9/site-packages/toil/batchSystems/abstractGridEngineBatchSystem.py", line 211, in _runStep
    activity |= self.createJobs(newJob)
  File "/nfs/production/panda/ensembl/thiagogenez/usr/linuxbrew/Cellar/cactus/HEAD-71f6430/libexec/lib/python3.9/site-packages/toil/batchSystems/abstractGridEngineBatchSystem.py", line 115, in createJobs
    batchJobID = self.boss.with_retries(self.submitJob, subLine)
  File "/nfs/production/panda/ensembl/thiagogenez/usr/linuxbrew/Cellar/cactus/HEAD-71f6430/libexec/lib/python3.9/site-packages/toil/batchSystems/abstractGridEngineBatchSystem.py", line 448, in with_retries
    raise err
  File "/nfs/production/panda/ensembl/thiagogenez/usr/linuxbrew/Cellar/cactus/HEAD-71f6430/libexec/lib/python3.9/site-packages/toil/batchSystems/abstractGridEngineBatchSystem.py", line 439, in with_retries
    return operation(*args, **kwargs)
  File "/nfs/production/panda/ensembl/thiagogenez/usr/linuxbrew/Cellar/cactus/HEAD-71f6430/libexec/lib/python3.9/site-packages/toil/batchSystems/lsf.py", line 90, in submitJob
    stdout = call_command(subLine, env=combinedEnv)
  File "/nfs/production/panda/ensembl/thiagogenez/usr/linuxbrew/Cellar/cactus/HEAD-71f6430/libexec/lib/python3.9/site-packages/toil/lib/misc.py", line 226, in call_command
    raise CalledProcessErrorStderr(proc.returncode, cmd, output=stdout, stderr=stderr)
toil.lib.misc.CalledProcessErrorStderr: Command '['bsub', '-cwd', '.', '-J', 'toil_job_0', '-R', 'select[mem > 0] rusage[mem=0]', '-M', '0', '-n', '1', '-o', '/scratch/toil_workflow_bca7db08-c60c-4d2c-ab39-4c4943bce9a0_job_0_batch_lsf_%J_std_output.log', '-e', '/scratch/toil_workflow_bca7db08-c60c-4d2c-ab39-4c4943bce9a0_job_0_batch_lsf_%J_std_error.log', '_toil_worker RunSelfBlast file:/hps/nobackup2/production/ensembl/thiagogenez/mohab/cactus/jobStore kind-RunSelfBlast/instance-52384sl0 --context gASVtAAAAAAAAACMJXRvaWwuYmF0Y2hTeXN0ZW1zLmFic3RyYWN0QmF0Y2hTeXN0ZW2UjBRXb3JrZXJDbGVhbnVwQ29udGV4dJSTlCmBlH2UKIwRd29ya2VyQ2xlYW51cEluZm+UaACMEVdvcmtlckNsZWFudXBJbmZvlJOUTowkYmNhN2RiMDgtYzYwYy00ZDJjLWFiMzktNGM0OTQzYmNlOWEwlIwGYWx3YXlzlIeUgZSMBWFyZW5hlE51Yi4=']' exit status 255: MEMLIMIT value <0> is not valid. Limit must be a positive integer. You can use the following units for the limit: KB (or K), MB (or M), GB (or G), TB (or T), PB (or P), EB (or E), ZB (or Z). Job not submitted.

```
Problem:
```
>>> mem = 104857600
>>> mem
104857600
>>> mem = float(mem) // 1024**3
>>> mem
0.0
```

Suggestion for fix would be:
```
            bsubMem = []
            if mem:
                if per_core_reservation():
                    mem = float(mem)/1024**3/math.ceil(cpu)
                    mem_resource = parse_memory_resource(mem)
                    mem_limit = parse_memory_limit(mem)
                else:
                    mem = float(mem) // 1024**3
                    logger.info("INFO Transformation: {}".format(mem))
                    mem_resource = parse_memory_resource(mem)
                    mem_limit = parse_memory_limit(mem)

                if mem > 0:
                  bsubMem = ['-R', 'select[mem > {m}] '
                           'rusage[mem={m}]'.format(m=mem_resource),
                           '-M', str(mem_limit)]
```

Will do a pull request for this issue.

┆Issue is synchronized with this [Jira Task](https://ucsc-cgl.atlassian.net/browse/TOIL-761)
┆Issue Number: TOIL-761



Discussion:
@thiagogenez Sorry, this one is my fault!
@mr-c  No worries. Glad to help!  
May I leave this issue with you? (I will not pull a request then.)
If you have a tested fix, it would be better coming from you! I'm guessing
the double division operator is to blame here.

>

Okay - Will do a bit of more tests and pull a request. 

Resolving patch:
diff --git a/src/toil/batchSystems/lsf.py b/src/toil/batchSystems/lsf.py
--- a/src/toil/batchSystems/lsf.py
+++ b/src/toil/batchSystems/lsf.py
@@ -36,7 +36,8 @@
 from toil.batchSystems.lsfHelper import (check_lsf_json_output_supported,
                                          parse_memory_limit,
                                          parse_memory_resource,
-                                         per_core_reservation)
+                                         per_core_reservation,
+                                         parse_mem_and_cmd_from_output)
 from toil.lib.misc import call_command
 
 logger = logging.getLogger(__name__)
@@ -286,29 +287,22 @@ def parseMaxMem(self, jobID):
             """
             Parse the maximum memory from job.
 
-            params:
-            jobID: ID number of the job
+            :param jobID: ID number of the job
             """
-            memargs = ["bjobs", "-l", str(jobID)]
             try:
-                bjobs = subprocess.check_output(memargs, universal_newlines=True)
-                memregex = r"MAX MEM: (.*?);"
-                meminfo = re.search(memregex, bjobs)
-                s = " ".join(bjobs.split())
-                command = re.search(r"Command <(.*?)>", s)
-                if meminfo:
-                    if not command:
-                        logger.info("Cannot Parse Max Memory Due to Missing Command String: %s", bjobs)
-                    else:
-                        logger.info("[job ID %s, Command %s] the maximum memory used was: %s",
-                                    str(jobID), command.group(1), meminfo.group(1))
+                output = subprocess.check_output(["bjobs", "-l", str(jobID)], universal_newlines=True)
+                max_mem, command = parse_mem_and_cmd_from_output(output=output)
+                if not max_mem:
+                    logger.warning(f"[job ID {jobID}] Unable to Collect Maximum Memory Usage: {output}")
+                    return
+
+                if not command:
+                    logger.warning(f"[job ID {jobID}] Cannot Parse Max Memory Due to Missing Command String: {output}")
                 else:
-                    logger.debug("[job ID %s] Unable to collect maximum memory usage: %s",
-                                 str(jobID), bjobs)
-                return meminfo
-            except subprocess.CalledProcessError as err:
-                logger.debug("[job ID %s] Unable to collect maximum memory usage: %s",
-                             str(jobID), str(err))
+                    logger.info(f"[job ID {jobID}, Command {command.group(1)}] Max Memory Used: {max_mem.group(1)}")
+                return max_mem
+            except subprocess.CalledProcessError as e:
+                logger.warning(f"[job ID {jobID}] Unable to Collect Maximum Memory Usage: {e}")
 
     def getWaitDuration(self):
         """We give LSF a second to catch its breath (in seconds)"""
diff --git a/src/toil/batchSystems/lsfHelper.py b/src/toil/batchSystems/lsfHelper.py
--- a/src/toil/batchSystems/lsfHelper.py
+++ b/src/toil/batchSystems/lsfHelper.py
@@ -151,6 +151,17 @@ def get_lsf_units(resource: bool = False) -> str:
         return DEFAULT_LSF_UNITS
 
 
+def parse_mem_and_cmd_from_output(output: str):
+    """Use regex to find "MAX MEM" and "Command" inside of an output."""
+    # Handle hard wrapping in the middle of words and arbitrary
+    # indents. May drop spaces at the starts of lines that aren't
+    # meant to be part of the indent.
+    cleaned_up_output = ' '.join(re.sub(r"\n\s*", "", output).split(','))
+    max_mem = re.search(r"MAX ?MEM: ?(.*?);", cleaned_up_output)
+    command = re.search(r"Command ?<(.*?)>", cleaned_up_output)
+    return max_mem, command
+
+
 def get_lsf_version():
     """
     Get current LSF version


Test patch:
diff --git a/src/toil/test/batchSystems/test_lsf_helper.py b/src/toil/test/batchSystems/test_lsf_helper.py
new file mode 100644
--- /dev/null
+++ b/src/toil/test/batchSystems/test_lsf_helper.py
@@ -0,0 +1,74 @@
+"""lsfHelper.py shouldn't need a batch system and so the unit tests here should aim to run on any system."""
+from toil.batchSystems.lsfHelper import parse_mem_and_cmd_from_output
+from toil.test import ToilTest
+
+
+class LSFHelperTest(ToilTest):
+    def test_parse_mem_and_cmd_from_output(self):
+        # https://github.com/DataBiosphere/toil/pull/3475
+        output = ('\nJob <2924748>, Job Name <toil_job_64>, User <thiagogenez>, Project <default>, S'
+                  '\n                     tatus <RUN>, Queue <research-rh74>, Job Priority <50>, Com'
+                  '\n                     mand <_toil_worker CactusBarRecursion file:/hps/nobackup/p'
+                  '\n                     roduction/ensembl/thiagogenez/pairwises/arabidopsis/run/jo'
+                  '\n                     bstore/3 kind-CactusBarRecursion/instance-iu6wo56x --conte'
+                  '\n                     xt gAShortenedh32xqlE51Yi4=>, Share group charged </thiago'
+                  '\n                     genez>, Esub <esub>'
+                  '\nThu Mar 18 02:06:32: Submitted from host <noah-login-01>, CWD </hps/nobackup/pr'
+                  '\n                     oduction/ensembl/thiagogenez/pairwises/arabidopsis/run>, S'
+                  '\n                     pecified CWD </hps/nobackup/production/ensembl/thiagogenez'
+                  '\n                     /pairwises/arabidopsis/run/.>, Output File </tmp/toil_work'
+                  '\n                     flow_10e83102-2e4b-4093-9128-2a52f4bbc65a_job_64_batch_lsf'
+                  '\n                     _2924748_std_output.log>, Error File </tmp/toil_workflow_1'
+                  '\n                     0e83102-2e4b-4093-9128-2a52f4bbc65a_job_64_batch_lsf_29247'
+                  '\n                     48_std_error.log>, Requested Resources <select[(mem>4000)]'
+                  '\n                      rusage[mem=4000:duration=480, numcpus=1:duration=480]>;'
+                  '\nThu Mar 18 02:06:33: Started on <hx-noah-31-02>, Execution Home </homes/thiagog'
+                  '\n                     enez>, Execution CWD </hps/nobackup/production/ensembl/thi'
+                  '\n                     agogenez/pairwises/arabidopsis/run/.>;'
+                  '\nThu Mar 18 17:07:47: Resource usage collected.'
+                  '\n                     The CPU time used is 53936 seconds.'
+                  '\n                     MEM: 344 Mbytes;  SWAP: 1.3 Gbytes;  NTHREAD: 5'
+                  '\n                     PGID: 433168;  PIDs: 433168 433177 433179 444026 '
+                  '\n'
+                  '\n RUNLIMIT                '
+                  '\n 10085.0 min'
+                  '\n'
+                  '\n CORELIMIT MEMLIMIT'
+                  '\n      0 M     3.9 G '
+                  '\n'
+                  '\n MEMORY USAGE:'
+                  '\n MAX MEM: 2.5 Gbytes;  AVG MEM: 343 Mbytes'
+                  '\n'
+                  '\n SCHEDULING PARAMETERS:'
+                  '\n           r15s   r1m  r15m   ut      pg    io   ls    it    tmp    swp    mem'
+                  '\n loadSched   -     -     -     -    10.0     -    -     -   500M     -   1000M '
+                  '\n loadStop    -     -     -     -       -     -    -     -     -      -      -  '
+                  '\n'
+                  '\n          availcpus '
+                  '\n loadSched      1.0 '
+                  '\n loadStop        -  '
+                  '\n'
+                  '\n RESOURCE REQUIREMENT DETAILS:'
+                  '\n Combined: select[((mem>4000)) && (type == local)] order[r15s:pg] rusage[mem=40'
+                  '\n                     00.00:duration=8h:decay=0,numcpus=1.00:duration=8h:decay=0'
+                  '\n                     ] span[hosts=1]'
+                  '\n Effective: select[(((mem>4000))) && (type == local)] order[r15s:pg] rusage[mem'
+                  '\n                     =4000.00:duration=8h:decay=0,numcpus=1.00:duration=8h:deca'
+                  '\n                     y=0] span[hosts=1] '
+                  '\n'
+                  '\n')
+        max_mem, command = parse_mem_and_cmd_from_output(output=output)
+        assert len(max_mem.groups()) == 1
+        expected_mem = '2.5 Gbytes'
+        assert max_mem.group(1) == expected_mem, f'Actual: {max_mem.group(1)}, Expected: "{expected_mem}"'
+
+        assert len(command.groups()) == 1
+        expected_command = ('_toil_worker CactusBarRecursion file:/hps/nobackup/production/ensembl/thiagogenez/'
+                            'pairwises/arabidopsis/run/jobstore/3 kind-CactusBarRecursion/instance-iu6wo56x '
+                            '--context gAShortenedh32xqlE51Yi4=')
+        assert command.group(1) == expected_command, f'Actual: {command.group(1)}, Expected: "{expected_command}"'
+        print(command)
+
+        max_mem, command = parse_mem_and_cmd_from_output(output='')
+        assert max_mem == None
+        assert command == None