Repository: DataBiosphere/toil

Issue:
TestService failing



Discussion:
This test is disabled via
9942f87ff5b0469b25286fde17aea216c52d9fe4

Is this still failing?

Last time I checked (yesterday) it was failing.

On Thu, Aug 13, 2015 at 11:00 PM, Benedict Paten notifications@github.com
wrote:

> Is this still failing?
> 
> —
> Reply to this email directly or view it on GitHub
> https://github.com/BD2KGenomics/toil/issues/185#issuecomment-130982938.

## 

Hannes Schmidt
Software Engineer
Center for Biomolecular Science and Engineering
University of California, Santa Cruz

(206) 696-2316 (cell)
hannes@ucsc.edu


Resolving patch:
diff --git a/src/toil/common.py b/src/toil/common.py
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -28,7 +28,6 @@
 from argparse import ArgumentParser
 from optparse import OptionContainer, OptionGroup
 
-from bd2k.util.humanize import human2bytes
 from toil.lib.bioio import addLoggingOptions, getLogLevelString, system, absSymPath
 from toil.batchSystems.parasol import ParasolBatchSystem
 from toil.batchSystems.gridengine import GridengineBatchSystem
@@ -250,6 +249,7 @@ def createConfig(options):
     
     TODO: Make the config object a proper class
     """
+    from bd2k.util.humanize import human2bytes
     logger.info("Starting to create the toil setup for the first time")
     config = ET.Element("config")
     config.attrib["log_level"] = getLogLevelString()
diff --git a/src/toil/job.py b/src/toil/job.py
--- a/src/toil/job.py
+++ b/src/toil/job.py
@@ -137,6 +137,7 @@ def addService(self, service):
         """
         jobService = ServiceJob(service)
         self._services.append(jobService)
+        jobService._addPredecessor(self)
         return jobService.rv()
     
     def addFollowOn(self, followOnJob):
@@ -274,7 +275,7 @@ def checkJobGraphConnected(self):
         """
         rootJobs = self.getRootJobs()
         if len(rootJobs) != 1:
-            raise JobGraphDeadlockException("Graph does not contain exactly one root jobs: %s" % rootJobs)
+            raise JobGraphDeadlockException("Graph does not contain exactly one root job: %s" % rootJobs)
     
     def checkJobGraphAcylic(self):
         """


Test patch:
diff --git a/src/toil/test/sort/sortTest.py b/src/toil/test/sort/sortTest.py
--- a/src/toil/test/sort/sortTest.py
+++ b/src/toil/test/sort/sortTest.py
@@ -224,86 +224,5 @@ def makeFileToSort(fileName, lines=10, maxLineLength=10):
         for line in xrange(lines):
             fileHandle.write(getRandomLine(maxLineLength))
 
-
-###########################################
-# Job functions
-###########################################           
-
-success_ratio = 0.5
-
-
-def setup(job, inputFile, N):
-    """Sets up the sort.
-    """
-    tempOutputFileStoreID = job.fileStore.getEmptyFileStoreID()
-    job.addChildJobFn(down, inputFile, 0, os.path.getsize(inputFile), N, tempOutputFileStoreID)
-    job.addFollowOnJobFn(cleanup, tempOutputFileStoreID, inputFile)
-
-
-def down(job, inputFile, fileStart, fileEnd, N, outputFileStoreID):
-    """Input is a file and a range into that file to sort and an output location in which
-    to write the sorted file.
-    If the range is larger than a threshold N the range is divided recursively and
-    a follow on job is then created which merges back the results else
-    the file is sorted and placed in the output.
-    """
-    if random.random() > success_ratio:
-        raise RuntimeError()  # This error is a test error, it does not mean the tests have failed.
-    length = fileEnd - fileStart
-    assert length >= 0
-    if length > N:
-        job.fileStore.logToMaster("Splitting range (%i..%i) of file: %s"
-                                  % (fileStart, fileEnd, inputFile))
-        midPoint = getMidPoint(inputFile, fileStart, fileEnd)
-        assert midPoint >= fileStart
-        assert midPoint + 1 < fileEnd
-        # We will subdivide the file
-        tempFileStoreID1 = job.fileStore.getEmptyFileStoreID()
-        tempFileStoreID2 = job.fileStore.getEmptyFileStoreID()
-        # The use of rv here is for testing purposes
-        # The rv(0) of the first child job is tempFileStoreID1,
-        # similarly rv(0) of the second child is tempFileStoreID2
-        job.addFollowOnJobFn(up,
-                             job.addChildJobFn(down, inputFile, fileStart,
-                                               midPoint + 1, N, tempFileStoreID1).rv(0),
-                             job.addChildJobFn(down, inputFile, midPoint + 1,
-                                               fileEnd, N, tempFileStoreID2).rv(0),
-                             # Add one to avoid the newline
-                             outputFileStoreID)
-    else:
-        # We can sort this bit of the file
-        job.fileStore.logToMaster("Sorting range (%i..%i) of file: %s"
-                                  % (fileStart, fileEnd, inputFile))
-        with job.fileStore.updateGlobalFileStream(outputFileStoreID) as fileHandle:
-            copySubRangeOfFile(inputFile, fileStart, fileEnd, fileHandle)
-        # Make a local copy and sort the file
-        tempOutputFile = job.fileStore.readGlobalFile(outputFileStoreID)
-        sort(tempOutputFile)
-        job.fileStore.updateGlobalFile(outputFileStoreID, tempOutputFile)
-    return outputFileStoreID
-
-
-def up(job, inputFileID1, inputFileID2, outputFileStoreID):
-    """Merges the two files and places them in the output.
-    """
-    if random.random() > success_ratio:
-        raise RuntimeError()  # This error is a test error, it does not mean the tests have failed.
-    with job.fileStore.updateGlobalFileStream(outputFileStoreID) as fileHandle:
-        with job.fileStore.readGlobalFileStream(inputFileID1) as inputFileHandle1:
-            with job.fileStore.readGlobalFileStream(inputFileID2) as inputFileHandle2:
-                merge(inputFileHandle1, inputFileHandle2, fileHandle)
-    job.fileStore.logToMaster("Merging %s and %s to %s"
-                              % (inputFileID1, inputFileID2, outputFileStoreID))
-
-
-def cleanup(job, tempOutputFileStoreID, outputFile):
-    """Copies back the temporary file to input once we've successfully sorted the temporary file.
-    """
-    if random.random() > success_ratio:
-        raise RuntimeError()  # This is a test error and not a failure of the tests
-    job.fileStore.readGlobalFile(tempOutputFileStoreID, outputFile)
-    # sort(outputFile)
-
-
 if __name__ == '__main__':
     unittest.main()