Repository: databiosphere/toil
Issue: --stats is missing certain job functions

_From @jvivian on December 14, 2015 19:29_

Stats output:
http://pastebin.com/piy7dWSz

Script being run:
https://github.com/BD2KGenomics/toil-scripts/blob/master/rnaseq_cgl/rnaseq_cgl_pipeline.py

So in this case, STAR and RSEM functions didn't show up in the output.  Given the times, you can tell certain jobs got "merged".

_Copied from original issue: BD2KGenomics/toil-scripts#70_


Changed files:
src/toil/test/utils/utilsTest.py, src/toil/utils/toilStats.py, src/toil/job.py, src/toil/worker.py

diff --git a/src/toil/job.py b/src/toil/job.py
index fe75ff40..52faf612 100644
--- a/src/toil/job.py
+++ b/src/toil/job.py
@@ -1301,11 +1301,15 @@ class Job(object):
             os.chdir(baseDir)
         #Finish up the stats
         if stats != None:
-            stats.jobs.time = str(time.time() - startTime)
             totalCpuTime, totalMemoryUsage = getTotalCpuTimeAndMemoryUsage()
-            stats.jobs.clock = str(totalCpuTime - startClock)
-            stats.jobs.class_name = self._jobName()
-            stats.jobs.memory = str(totalMemoryUsage)
+            stats.jobs.append(
+                Expando(
+                    time=str(time.time() - startTime),
+                    clock=str(totalCpuTime - startClock),
+                    class_name=self._jobName(),
+                    memory=str(totalMemoryUsage)
+                )
+            )
 
     def _jobName(self):
         """
diff --git a/src/toil/test/utils/utilsTest.py b/src/toil/test/utils/utilsTest.py
index 55003a15..a8119cc8 100644
--- a/src/toil/test/utils/utilsTest.py
+++ b/src/toil/test/utils/utilsTest.py
@@ -26,6 +26,8 @@ from toil.lib.bioio import getTempFile
 from toil.lib.bioio import system
 from toil.test import ToilTest
 from toil.test.sort.sortTest import makeFileToSort
+from toil.utils.toilStats import getStats, processData
+from toil.common import loadJobStore
 
 
 class UtilsTest(ToilTest):
@@ -167,9 +169,32 @@ class UtilsTest(ToilTest):
         options.logLevel = 'debug'
         Job.Runner.startToil(Job.wrapFn(printUnicodeCharacter), options)
 
+    def testMultipleJobsPerWorkerStats(self):
+        """
+        Tests case where multiple jobs are run on 1 worker to insure that all jobs report back their data
+        """
+        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
+        options.clean = 'never'
+        options.stats = True
+        Job.Runner.startToil(RunTwoJobsPerWorker(), options)
+
+        jobStore = loadJobStore(options.jobStore)
+        stats = getStats(options)
+        collatedStats =  processData(jobStore.config, stats, options)
+        self.assertTrue(len(collatedStats.job_types)==2,"Some jobs are not represented in the stats")
 
 def printUnicodeCharacter():
     # We want to get a unicode character to stdout but we can't print it directly because of
     # Python encoding issues. To work around this we print in a separate Python process. See
     # http://stackoverflow.com/questions/492483/setting-the-correct-encoding-when-piping-stdout-in-python
     check_call([sys.executable, '-c', "print '\\\\xc3\\\\xbc'"])
+
+class RunTwoJobsPerWorker(Job):
+    """
+    Runs child job with same resources as self in an attempt to chain the jobs on the same worker
+    """
+    def __init__(self):
+        Job.__init__(self)
+
+    def run(self, fileStore):
+        self.addChildFn(printUnicodeCharacter)
diff --git a/src/toil/utils/toilStats.py b/src/toil/utils/toilStats.py
index 285ed0b1..16f90bd3 100644
--- a/src/toil/utils/toilStats.py
+++ b/src/toil/utils/toilStats.py
@@ -557,6 +557,7 @@ def processData(config, stats, options):
     # Add worker info
     worker = filter(None, stats.workers)
     jobs = filter(None, stats.jobs)
+    jobs = [item for sublist in jobs for item in sublist]
 
     def fn4(job):
         try:
diff --git a/src/toil/worker.py b/src/toil/worker.py
index e80846dc..46dd1d63 100644
--- a/src/toil/worker.py
+++ b/src/toil/worker.py
@@ -199,6 +199,7 @@ def main():
 
     workerFailed = False
     statsDict = MagicExpando()
+    statsDict.jobs = []
     messages = []
     blockFn = lambda : True
     cleanCacheFn = lambda x : True