Repository: DataBiosphere/toil

Issue:
Add sortTest for GridEngine and Parasol



Discussion:

Resolving patch:
diff --git a/src/toil/batchSystems/gridengine.py b/src/toil/batchSystems/gridengine.py
--- a/src/toil/batchSystems/gridengine.py
+++ b/src/toil/batchSystems/gridengine.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
 # Copyright (C) 2015 UCSC Computational Genomics Lab
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,13 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 from __future__ import absolute_import
 import logging
 import os
 import subprocess
 import time
 import math
-from Queue import Queue
+from Queue import Queue, Empty
 from threading import Thread
 
 from toil.batchSystems.abstractBatchSystem import AbstractBatchSystem
@@ -27,6 +26,8 @@
 
 logger = logging.getLogger(__name__)
 
+sleepSeconds = 1
+
 
 class MemoryString:
     def __init__(self, string):
@@ -70,34 +71,37 @@ def prepareQsub(cpu, mem, jobID):
 
     reqline = list()
     if mem is not None:
-        reqline.append("vf=" + str(mem / 1024) + "K")
-        reqline.append("h_vmem=" + str(mem / 1024) + "K")
+        memStr = str(mem / 1024) + "K"
+        reqline += [ "vf=" + memStr, "h_vmem=" + memStr ]
     if len(reqline) > 0:
-        qsubline.extend(["-hard","-l", ",".join(reqline)])
+        qsubline.extend(["-hard", "-l", ",".join(reqline)])
     if cpu is not None and math.ceil(cpu) > 1:
         qsubline.extend(["-pe", "smp", str(int(math.ceil(cpu)))])
     return qsubline
 
+
 def qsub(qsubline):
-    logger.debug("**" + " ".join(qsubline))
+    logger.debug("Running %r", qsubline)
     process = subprocess.Popen(qsubline, stdout=subprocess.PIPE)
     result = int(process.stdout.readline().strip().split('.')[0])
     return result
 
-def getjobexitcode(sgeJobID):
-        job, task = sgeJobID
-        args = ["qacct", "-j", str(job)]
-        if task is not None:
-             args.extend(["-t", str(task)])
-
-        process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        for line in process.stdout:
-            if line.startswith("failed") and int(line.split()[1]) == 1:
-                return 1
-            elif line.startswith("exit_status"):
-                logger.debug('Exit Status: {}'.format(line.split()[1]))
-                return int(line.split()[1])
-        return None
+
+def getJobExitCode(sgeJobID):
+    job, task = sgeJobID
+    args = ["qacct", "-j", str(job)]
+    if task is not None:
+        args.extend(["-t", str(task)])
+
+    process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    for line in process.stdout:
+        if line.startswith("failed") and int(line.split()[1]) == 1:
+            return 1
+        elif line.startswith("exit_status"):
+            logger.debug('Exit Status: %r', line.split()[1])
+            return int(line.split()[1])
+    return None
+
 
 class Worker(Thread):
     def __init__(self, newJobsQueue, updatedJobsQueue, killQueue, killedJobsQueue, boss):
@@ -117,26 +121,26 @@ def getRunningJobIDs(self):
         currentjobs = dict((str(self.sgeJobIDs[x][0]), x) for x in self.runningJobs)
         process = subprocess.Popen(["qstat"], stdout=subprocess.PIPE)
         stdout, stderr = process.communicate()
-        
+
         for currline in stdout.split('\n'):
             items = currline.strip().split()
             if items:
                 if items[0] in currentjobs and items[4] == 'r':
                     jobstart = " ".join(items[5:7])
-                    jobstart = time.mktime(time.strptime(jobstart,"%m/%d/%Y %H:%M:%S"))
+                    jobstart = time.mktime(time.strptime(jobstart, "%m/%d/%Y %H:%M:%S"))
                     times[currentjobs[items[0]]] = time.time() - jobstart
 
         return times
 
     def getSgeID(self, jobID):
         if not jobID in self.sgeJobIDs:
-             RuntimeError("Unknown jobID, could not be converted")
+            RuntimeError("Unknown jobID, could not be converted")
 
-        (job,task) = self.sgeJobIDs[jobID]
+        (job, task) = self.sgeJobIDs[jobID]
         if task is None:
-             return str(job)
+            return str(job)
         else:
-             return str(job) + "." + str(task)
+            return str(job) + "." + str(task)
 
     def forgetJob(self, jobID):
         self.runningJobs.remove(jobID)
@@ -146,14 +150,22 @@ def forgetJob(self, jobID):
     def killJobs(self):
         # Load hit list:
         killList = list()
-        while not self.killQueue.empty():
-            killList.append(self.killQueue.get())
+        while True:
+            try:
+                jobId = self.killQueue.get(block=False)
+            except Empty:
+                break
+            else:
+                killList.append(jobId)
+
+        if not killList:
+            return False
 
         # Do the dirty job
         for jobID in list(killList):
             if jobID in self.runningJobs:
-                logger.debug('Killing job: {}'.format(jobID))
-                process = subprocess.Popen(["qdel", self.getSgeID(jobID)])
+                logger.debug('Killing job: %s', jobID)
+                subprocess.check_call(['qdel', self.getSgeID(jobID)])
             else:
                 if jobID in self.waitingJobs:
                     self.waitingJobs.remove(jobID)
@@ -161,124 +173,139 @@ def killJobs(self):
                 killList.remove(jobID)
 
         # Wait to confirm the kill
-        while len(killList) > 0:
+        while killList:
             for jobID in list(killList):
-                if getjobexitcode(self.sgeJobIDs[jobID]) is not None:
-                    logger.debug('Adding {} jobID to killedJobsQueue'.format(jobID))
+                if getJobExitCode(self.sgeJobIDs[jobID]) is not None:
+                    logger.debug('Adding jobID %s to killedJobsQueue', jobID)
                     self.killedJobsQueue.put(jobID)
                     killList.remove(jobID)
                     self.forgetJob(jobID)
-
             if len(killList) > 0:
-                logger.warn("Tried to kill some jobs, but something happened and they are still going, "
-                            "so I'll try again")
-                time.sleep(5)
+                logger.warn("Some jobs weren't killed, trying again in %is.", sleepSeconds)
+                time.sleep(sleepSeconds)
 
-    def createJobs(self, new_job):
-        # Load new job id if present:
-        if new_job is not None:
-            self.waitingJobs.append(new_job)
+        return True
 
+    def createJobs(self, newJob):
+        activity = False
+        # Load new job id if present:
+        if newJob is not None:
+            self.waitingJobs.append(newJob)
         # Launch jobs as necessary:
-        while len(self.waitingJobs) > 0 and sum(self.allocatedCpus.values()) < int(self.boss.maxCores):
+        while len(self.waitingJobs) > 0 and sum(self.allocatedCpus.values()) < int(
+                self.boss.maxCores):
+            activity = True
             jobID, cpu, memory, command = self.waitingJobs.pop(0)
             qsubline = prepareQsub(cpu, memory, jobID) + [command]
             sgeJobID = qsub(qsubline)
             self.sgeJobIDs[jobID] = (sgeJobID, None)
             self.runningJobs.add(jobID)
             self.allocatedCpus[jobID] = cpu
+        return activity
 
     def checkOnJobs(self):
-        logger.debug('List of runningJobs: {}'.format(self.runningJobs))
+        activity = False
+        logger.debug('List of running jobs: %r', self.runningJobs)
         for jobID in list(self.runningJobs):
-            exit = getjobexitcode(self.sgeJobIDs[jobID])
+            exit = getJobExitCode(self.sgeJobIDs[jobID])
             if exit is not None:
+                activity = True
                 self.updatedJobsQueue.put((jobID, exit))
                 self.forgetJob(jobID)
-    
+        return activity
+
     def run(self):
         while True:
-            new_job = None
+            activity = False
+            newJob = None
             if not self.newJobsQueue.empty():
-                new_job = self.newJobsQueue.get()
-                if new_job is None:
+                activity = True
+                newJob = self.newJobsQueue.get()
+                if newJob is None:
                     logger.debug('Received queue sentinel.')
                     break
-            self.killJobs()
-            self.createJobs(new_job)
-            self.checkOnJobs()
-            time.sleep(10)
+            activity |= self.killJobs()
+            activity |= self.createJobs(newJob)
+            activity |= self.checkOnJobs()
+            if not activity:
+                logger.debug('No activity, sleeping for %is', sleepSeconds)
+                time.sleep(sleepSeconds)
+
 
 class GridengineBatchSystem(AbstractBatchSystem):
-    """The interface for gridengine.
     """
-    
+    The interface for SGE aka Sun GridEngine.
+    """
+
     def __init__(self, config, maxCores, maxMemory, maxDisk):
         AbstractBatchSystem.__init__(self, config, maxCores, maxMemory, maxDisk)
         self.gridengineResultsFile = getParasolResultsFileName(config.jobStore)
-        #Reset the job queue and results (initially, we do this again once we've killed the jobs)
+        # Reset the job queue and results (initially, we do this again once we've killed the jobs)
         self.gridengineResultsFileHandle = open(self.gridengineResultsFile, 'w')
-        self.gridengineResultsFileHandle.close() #We lose any previous state in this file, and ensure the files existence
-        self.currentjobs = set()
-        self.obtainSystemConstants()
+        # We lose any previous state in this file, and ensure the files existence
+        self.gridengineResultsFileHandle.close()
+        self.currentJobs = set()
+        self.maxCPU, self.maxMEM = self.obtainSystemConstants()
         self.nextJobID = 0
         self.newJobsQueue = Queue()
         self.updatedJobsQueue = Queue()
         self.killQueue = Queue()
         self.killedJobsQueue = Queue()
-        self.worker = Worker(self.newJobsQueue, self.updatedJobsQueue, self.killQueue, self.killedJobsQueue, self)
+        self.worker = Worker(self.newJobsQueue, self.updatedJobsQueue, self.killQueue,
+                             self.killedJobsQueue, self)
         self.worker.start()
-        
+
     def __des__(self):
-        #Closes the file handle associated with the results file.
-        self.gridengineResultsFileHandle.close() #Close the results file, cos were done.
+        # Closes the file handle associated with the results file.
+        self.gridengineResultsFileHandle.close()
 
     def issueBatchJob(self, command, memory, cores, disk):
         self.checkResourceRequest(memory, cores, disk)
         jobID = self.nextJobID
         self.nextJobID += 1
-        self.currentjobs.add(jobID)
+        self.currentJobs.add(jobID)
         self.newJobsQueue.put((jobID, cores, memory, command))
-        logger.debug("Issued the job command: %s with job id: %s " % (command, str(jobID)))
+        logger.debug("Issued the job command: %s with job id: %s ", command, str(jobID))
         return jobID
 
     def killBatchJobs(self, jobIDs):
-        """Kills the given jobs, represented as Job ids, then checks they are dead by checking
+        """
+        Kills the given jobs, represented as Job ids, then checks they are dead by checking
         they are not in the list of issued jobs.
         """
+        jobIDs = set(jobIDs)
+        logger.debug('Jobs to be killed: %r', jobIDs)
         for jobID in jobIDs:
             self.killQueue.put(jobID)
-
-        logger.debug('Jobs to be killed: {}'.format(jobIDs))
-        killList = set(jobIDs)
-        while len(killList) > 0:
-            i = self.killedJobsQueue.get()
-            if i is not None:
-                killList.remove(i)
-                if i in self.currentjobs:
-                    self.currentjobs.remove(i)
-            else:
+        while jobIDs:
+            killedJobId = self.killedJobsQueue.get()
+            if killedJobId is None:
                 break
-
-            if len(killList) > 0:
-                time.sleep(5)
+            jobIDs.remove(killedJobId)
+            if killedJobId in self.currentJobs:
+                self.currentJobs.remove(killedJobId)
+            if jobIDs:
+                logger.debug('Some kills (%s) still pending, sleeping %is', len(jobIDs),
+                             sleepSeconds)
+                time.sleep(sleepSeconds)
 
     def getIssuedBatchJobIDs(self):
-        """Gets the list of jobs issued to SGE.
         """
-        return list(self.currentjobs)
-    
+        Gets the list of jobs issued to SGE.
+        """
+        return list(self.currentJobs)
+
     def getRunningBatchJobIDs(self):
         return self.worker.getRunningJobIDs()
-    
+
     def getUpdatedBatchJob(self, maxWait):
         i = self.updatedJobsQueue.get()
-        logger.debug('UpdatedJobsQueue Item: {}'.format(i))
-        if i == None:
+        logger.debug('UpdatedJobsQueue Item: %s', i)
+        if i is None:
             return None
         jobID, retcode = i
         self.updatedJobsQueue.task_done()
-        self.currentjobs.remove(jobID)
+        self.currentJobs.remove(jobID)
         return i
 
     def shutdown(self):
@@ -292,58 +319,44 @@ def shutdown(self):
         self.worker.join()
 
     def getWaitDuration(self):
-        """We give parasol a second to catch its breath (in seconds)
+        """
+        We give parasol a second to catch its breath (in seconds)
         """
         return 0.0
 
     @classmethod
     def getRescueBatchJobFrequency(cls):
-        """Parasol leaks jobs, but rescuing jobs involves calls to parasol list jobs and pstat2,
+        """
+        Parasol leaks jobs, but rescuing jobs involves calls to parasol list jobs and pstat2,
         making it expensive. We allow this every 10 minutes..
         """
-        return 1800 #Half an hour
+        return 1800  # Half an hour
 
-    def obtainSystemConstants(self):
-        p = subprocess.Popen(["qhost"], stdout = subprocess.PIPE,stderr = subprocess.STDOUT)
-
-        line = p.stdout.readline()
+    @staticmethod
+    def obtainSystemConstants():
+        lines = filter(None, map(str.strip, subprocess.check_output(["qhost"]).split('\n')))
+        line = lines[0]
         items = line.strip().split()
         num_columns = len(items)
         cpu_index = None
-        mem_index = None        
+        mem_index = None
         for i in range(num_columns):
-                if items[i] == 'NCPU':
-                        cpu_index = i
-                elif items[i] == 'MEMTOT':
-                        mem_index = i
-
+            if items[i] == 'NCPU':
+                cpu_index = i
+            elif items[i] == 'MEMTOT':
+                mem_index = i
         if cpu_index is None or mem_index is None:
-                RuntimeError("qhost command does not return NCPU or MEMTOT columns")
-
-        p.stdout.readline()
-
-        self.maxCPU = 0
-        self.maxMEM = MemoryString("0")
-        for line in p.stdout:
-                items = line.strip().split()
-                if len(items) < num_columns:
-                        RuntimeError("qhost output has a varying number of columns")
-                if items[cpu_index] != '-' and items[cpu_index] > self.maxCPU:
-                        self.maxCPU = items[cpu_index]
-                if items[mem_index] != '-' and MemoryString(items[mem_index]) > self.maxMEM:
-                        self.maxMEM = MemoryString(items[mem_index])
-
-        if self.maxCPU is 0 or self.maxMEM is 0:
-                RuntimeError("qhost returns null NCPU or MEMTOT info")
-                
-        
-def main():
-    pass
-
-def _test():
-    import doctest      
-    return doctest.testmod()
-
-if __name__ == '__main__':
-    _test()
-    main()
+            RuntimeError('qhost command does not return NCPU or MEMTOT columns')
+        maxCPU = 0
+        maxMEM = MemoryString("0")
+        for line in lines[2:]:
+            items = line.strip().split()
+            if len(items) < num_columns:
+                RuntimeError('qhost output has a varying number of columns')
+            if items[cpu_index] != '-' and items[cpu_index] > maxCPU:
+                maxCPU = items[cpu_index]
+            if items[mem_index] != '-' and MemoryString(items[mem_index]) > maxMEM:
+                maxMEM = MemoryString(items[mem_index])
+        if maxCPU is 0 or maxMEM is 0:
+            RuntimeError('qhost returned null NCPU or MEMTOT info')
+        return maxCPU, maxMEM


Test patch:
diff --git a/src/toil/test/__init__.py b/src/toil/test/__init__.py
--- a/src/toil/test/__init__.py
+++ b/src/toil/test/__init__.py
@@ -161,7 +161,8 @@ def needs_gridengine(test_item):
     Use as a decorator before test classes or methods to only run them if GridEngine is installed.
     """
     try:
-        subprocess.Popen('qsub')
+        with open('/dev/null','a') as dev_null:
+            subprocess.Popen('qsub',stdout=dev_null,stderr=dev_null)
     except OSError:
         return unittest.skip("Skipping test. Install GridEngine to include this test.")(test_item)
     except:
@@ -191,7 +192,8 @@ def needs_parasol(test_item):
     Use as decorator so tests are only run if Parasol is installed.
     """
     try:
-        subprocess.Popen("parasol")
+        with open('/dev/null','a') as dev_null:
+            subprocess.Popen('parasol',stdout=dev_null,stderr=dev_null)
     except OSError:
         return unittest.skip("Skipping test. Install Parasol to include this test.")(test_item)
     except:
diff --git a/src/toil/test/sort/sort.py b/src/toil/test/sort/sort.py
--- a/src/toil/test/sort/sort.py
+++ b/src/toil/test/sort/sort.py
@@ -20,18 +20,22 @@
 from argparse import ArgumentParser
 import os
 import random
+from bd2k.util.humanize import human2bytes
 
 from toil.job import Job
 from toil.test.sort.lib import merge, sort, copySubRangeOfFile, getMidPoint
 
 success_ratio = 0.5
+sortMemory = human2bytes('1000M')
 
 def setup(job, inputFile, N):
     """Sets up the sort.
     """
     tempOutputFileStoreID = job.fileStore.getEmptyFileStoreID()
-    job.addChildJobFn(down, inputFile, 0, os.path.getsize(inputFile), N, tempOutputFileStoreID)
-    job.addFollowOnJobFn(cleanup, tempOutputFileStoreID, inputFile)
+    job.addChildJobFn(down, inputFile, 0, os.path.getsize(inputFile), N, tempOutputFileStoreID,
+                      memory=sortMemory)
+    job.addFollowOnJobFn(cleanup, tempOutputFileStoreID, inputFile,
+                         memory=sortMemory)
 
 def down(job, inputFile, fileStart, fileEnd, N, outputFileStoreID):
     """Input is a file and a range into that file to sort and an output location in which
@@ -58,10 +62,12 @@ def down(job, inputFile, fileStart, fileEnd, N, outputFileStoreID):
         #similarly rv() of the second child is tempFileStoreID2
         job.addFollowOnJobFn(up,
                                    job.addChildJobFn(down, inputFile, fileStart,
-                                                           midPoint+1, N, tempFileStoreID1).rv(),
+                                                           midPoint+1, N, tempFileStoreID1,
+                                                           memory=sortMemory).rv(),
                                    job.addChildJobFn(down, inputFile, midPoint+1,
-                                                           fileEnd, N, tempFileStoreID2).rv(), #Add one to avoid the newline
-                                   outputFileStoreID)                
+                                                           fileEnd, N, tempFileStoreID2,
+                                                           memory=sortMemory).rv(), # Add one to avoid the newline
+                                   outputFileStoreID,memory=sortMemory)
     else:
         #We can sort this bit of the file
         job.fileStore.logToMaster( "Sorting range (%i..%i) of file: %s"
@@ -97,28 +103,29 @@ def cleanup(job, tempOutputFileStoreID, outputFile):
 def main():
     parser = ArgumentParser()
     Job.Runner.addToilOptions(parser)
-    
+
     parser.add_argument("--fileToSort", dest="fileToSort",
                       help="The file you wish to sort")
-    
+
     parser.add_argument("--N", dest="N",
                       help="The threshold below which a serial sort function is"
-                      "used to sort file. All lines must of length less than or equal to N or program will fail", 
+                      "used to sort file. All lines must of length less than or equal to N or program will fail",
                       default=10000)
-    
+
     options = parser.parse_args()
-    
+
     if options.fileToSort is None:
         raise RuntimeError("No file to sort given")
 
     if not os.path.exists(options.fileToSort):
         raise RuntimeError("File to sort does not exist: %s" % options.fileToSort)
-    
+
     if int(options.N) <= 0:
         raise RuntimeError("Invalid value of N: %s" % options.N)
-    
+
     #Now we are ready to run
-    Job.Runner.startToil(Job.wrapJobFn(setup, options.fileToSort, int(options.N)), options)
+    Job.Runner.startToil(Job.wrapJobFn(setup, options.fileToSort, int(options.N),
+                                       memory=sortMemory), options)
 
 if __name__ == '__main__':
     main()
diff --git a/src/toil/test/sort/sortTest.py b/src/toil/test/sort/sortTest.py
--- a/src/toil/test/sort/sortTest.py
+++ b/src/toil/test/sort/sortTest.py
@@ -22,29 +22,54 @@
 import logging
 import subprocess
 
+from toil.batchSystems.parasolTestSupport import ParasolTestSupport
 from toil.job import Job, JobException
 from toil.lib.bioio import getLogLevelString
 from toil.batchSystems.mesos.test import MesosTestSupport
 from toil.test.sort.lib import merge, sort, copySubRangeOfFile, getMidPoint
-from toil.test.sort.sort import setup
-from toil.test import ToilTest, needs_aws, needs_mesos
+from toil.test.sort.sort import setup, sortMemory
+from toil.test import ToilTest, needs_aws, needs_mesos, needs_azure, needs_parasol, needs_gridengine
 from toil.jobStores.abstractJobStore import JobStoreCreationException
 from toil.leader import FailedJobsException
 
 log = logging.getLogger(__name__)
 
+defaultLineLen = int(os.environ.get('TOIL_TEST_SORT_LINE_LEN', '10'))
+defaultLines = int(os.environ.get('TOIL_TEST_SORT_LINES', '10'))
+defaultN = int(os.environ.get('TOIL_TEST_SORT_N', str(defaultLineLen * defaultLines / 5)))
+
+
+class SortTest(ToilTest, MesosTestSupport, ParasolTestSupport):
+    """
+    Tests Toil by sorting a file in parallel on various combinations of job stores and batch
+    systems.
+    """
 
-class SortTest(ToilTest, MesosTestSupport):
     def setUp(self):
         super(SortTest, self).setUp()
         self.tempDir = self._createTempDir(purpose='tempDir')
 
-    def toilSortTest(self, jobStore, batchSystem, lines, N, testNo=1, maxLineLength=10):
+    def _toilSort(self, jobStore, batchSystem,
+                  lines=defaultLines, N=defaultN, testNo=1, lineLen=defaultLineLen):
         """
-        Tests toil by sorting a file in parallel.
+        Generate a file consisting of the given number of random lines, each line of the given
+        length. Sort the file with Toil by splitting the file recursively until each part is less
+        than the given number of bytes, sorting each part and merging them back together. Then
+        verify the result.
+
+        :param jobStore: a job store string
+
+        :param batchSystem: the name of the batch system
+
+        :param lines: the number of random lines to generate
+
+        :param N: the size in bytes of each split
+
+        :param testNo: the number of repeats of this test
+
+        :param lineLen: the length of each random line in the file
         """
         for test in xrange(testNo):
-
             try:
                 # Specify options
                 options = Job.Runner.getDefaultOptions(jobStore)
@@ -55,7 +80,7 @@ def toilSortTest(self, jobStore, batchSystem, lines, N, testNo=1, maxLineLength=
 
                 # Make the file to sort
                 tempSortFile = os.path.join(self.tempDir, "fileToSort.txt")
-                makeFileToSort(tempSortFile, lines=lines, maxLineLength=maxLineLength)
+                makeFileToSort(tempSortFile, lines=lines, lineLen=lineLen)
 
                 # First make our own sorted version
                 with open(tempSortFile, 'r') as fileHandle:
@@ -63,7 +88,7 @@ def toilSortTest(self, jobStore, batchSystem, lines, N, testNo=1, maxLineLength=
                     l.sort()
 
                 # Make the first job
-                firstJob = Job.wrapJobFn(setup, tempSortFile, N, memory=5000)
+                firstJob = Job.wrapJobFn(setup, tempSortFile, N, memory=sortMemory)
 
                 # Check we get an exception if we try to restart a workflow that doesn't exist
                 options.restart = True
@@ -111,43 +136,64 @@ def toilSortTest(self, jobStore, batchSystem, lines, N, testNo=1, maxLineLength=
                 # Now check the file is properly sorted..
                 with open(tempSortFile, 'r') as fileHandle:
                     l2 = fileHandle.readlines()
-                    checkEqual(l, l2)
+                    self.assertEquals(l, l2)
             finally:
                 subprocess.check_call([self._getUtilScriptPath('toilMain'), 'clean', jobStore])
 
     @needs_aws
-    def testToilSortOnAWS(self):
-        self.toilSortTest(jobStore="aws:us-west-2:sort-test-%s" % uuid4(),
-                          batchSystem='singleMachine',
-                          lines=100, N=100)
+    def testAwsSingle(self):
+        self._toilSort(jobStore=self._awsJobStore(), batchSystem='singleMachine')
 
     @needs_aws
     @needs_mesos
-    def testToilSortOnAWSWithMesos(self):
+    def testAwsMesos(self):
         self._startMesos()
         try:
-            self.toilSortTest(jobStore="aws:us-west-2:sort-test-%s" % uuid4(),
-                              batchSystem="mesos",
-                              lines=100, N=100)
+            self._toilSort(jobStore=self._awsJobStore(), batchSystem="mesos")
         finally:
             self._stopMesos()
 
     @needs_mesos
-    def testToilSortWithMesos(self):
+    def testFileMesos(self):
         self._startMesos()
         try:
-            self.toilSortTest(jobStore=self._getTestJobStorePath(),
-                              batchSystem="mesos",
-                              lines=100, N=100)
+            self._toilSort(jobStore=self._getTestJobStorePath(), batchSystem="mesos")
         finally:
             self._stopMesos()
 
-    def testToilSort(self):
-        self.toilSortTest(jobStore=self._getTestJobStorePath(),
-                          batchSystem='singleMachine',
-                          lines=10000, N=10000)
+    @needs_azure
+    def testAzureSingle(self):
+        self._toilSort(jobStore=self._azureJobStore(), batchSystem='singleMachine')
 
-    # The following functions test the functions in the test!
+    @needs_azure
+    @needs_mesos
+    def testAzureMesos(self):
+        self._startMesos()
+        try:
+            self._toilSort(jobStore=self._azureJobStore(), batchSystem="mesos")
+        finally:
+            self._stopMesos()
+
+    def testFileSingle(self):
+        self._toilSort(jobStore=self._getTestJobStorePath(), batchSystem='singleMachine')
+
+    def testFileSingle10000(self):
+        self._toilSort(jobStore=self._getTestJobStorePath(), batchSystem='singleMachine',
+                       lines=10000, N=10000)
+
+    @needs_gridengine
+    def testFileGridEngine(self):
+        self._toilSort(jobStore=self._getTestJobStorePath(), batchSystem='gridengine')
+
+    @needs_parasol
+    def testFileParasol(self):
+        self._startParasol()
+        try:
+            self._toilSort(jobStore=self._getTestJobStorePath(), batchSystem='parasol')
+        finally:
+            self._stopParasol()
+
+    # The following functions test the functions in the test
 
     testNo = 5
 
@@ -155,12 +201,12 @@ def testSort(self):
         for test in xrange(self.testNo):
             tempFile1 = os.path.join(self.tempDir, "fileToSort.txt")
             makeFileToSort(tempFile1)
-            lines1 = loadFile(tempFile1)
+            lines1 = self._loadFile(tempFile1)
             lines1.sort()
             sort(tempFile1)
             with open(tempFile1, 'r') as f:
                 lines2 = f.readlines()
-            checkEqual(lines1, lines2)
+            self.assertEquals(lines1, lines2)
 
     def testMerge(self):
         for test in xrange(self.testNo):
@@ -175,17 +221,17 @@ def testMerge(self):
                 with open(tempFile1) as tempFileHandle1:
                     with open(tempFile2) as tempFileHandle2:
                         merge(tempFileHandle1, tempFileHandle2, fileHandle)
-            lines1 = loadFile(tempFile1) + loadFile(tempFile2)
+            lines1 = self._loadFile(tempFile1) + self._loadFile(tempFile2)
             lines1.sort()
             with open(tempFile3, 'r') as f:
                 lines2 = f.readlines()
-            checkEqual(lines1, lines2)
+            self.assertEquals(lines1, lines2)
 
     def testCopySubRangeOfFile(self):
         for test in xrange(self.testNo):
             tempFile = os.path.join(self.tempDir, "fileToSort1.txt")
             outputFile = os.path.join(self.tempDir, "outputFileToSort1.txt")
-            makeFileToSort(tempFile)
+            makeFileToSort(tempFile, lines=10, lineLen=defaultLineLen)
             fileSize = os.path.getsize(tempFile)
             assert fileSize > 0
             fileStart = random.choice(xrange(0, fileSize))
@@ -195,7 +241,7 @@ def testCopySubRangeOfFile(self):
             fileHandle.close()
             l = open(outputFile, 'r').read()
             l2 = open(tempFile, 'r').read()[fileStart:fileEnd]
-            checkEqual(l, l2)
+            self.assertEquals(l, l2)
 
     def testGetMidPoint(self):
         for test in xrange(self.testNo):
@@ -209,34 +255,24 @@ def testGetMidPoint(self):
             assert l[midPoint] == '\n'
             assert midPoint >= 0
 
+    # Support methods
 
-###########################################
-# Functions to generate file store and check result is okay
-###########################################  
-
-def checkEqual(i, j):
-    if i != j:
-        print "lengths", len(i), len(j)
-        print "true", i
-        print "false", j
-    assert i == j
-
-
-def loadFile(file):
-    with open(file, 'r') as fileHandle:
-        return fileHandle.readlines()
+    def _awsJobStore(self):
+        return "aws:us-west-2:sort-test-%s" % uuid4()
 
+    def _azureJobStore(self):
+        return "azure:toiltest:sort-test-%s" % uuid4()
 
-def getRandomLine(maxLineLength):
-    return "".join(
-        [random.choice(['a', 'c', 't', 'g', "A", "C", "G", "T", "N", "X", "Y", "Z"]) for i in
-         xrange(maxLineLength)]) + "\n"
+    def _loadFile(self, path):
+        with open(path, 'r') as f:
+            return f.readlines()
 
 
-def makeFileToSort(fileName, lines=10, maxLineLength=10):
+def makeFileToSort(fileName, lines=defaultLines, lineLen=defaultLineLen):
     with open(fileName, 'w') as fileHandle:
-        for line in xrange(lines):
-            fileHandle.write(getRandomLine(maxLineLength))
+        for _ in xrange(lines):
+            line = "".join(random.choice('actgACTGNXYZ') for _ in xrange(lineLen - 1)) + '\n'
+            fileHandle.write(line)
 
 
 if __name__ == '__main__':
diff --git a/src/toil/test/utils/utilsTest.py b/src/toil/test/utils/utilsTest.py
--- a/src/toil/test/utils/utilsTest.py
+++ b/src/toil/test/utils/utilsTest.py
@@ -18,9 +18,9 @@
 
 from toil.lib.bioio import system
 from toil.lib.bioio import getTempFile
-from toil.test.sort.sortTest import makeFileToSort, checkEqual
 from toil.common import toilPackageDirPath
 from toil.test import ToilTest
+from toil.test.sort.sortTest import makeFileToSort
 
 
 class UtilsTest(ToilTest):
@@ -37,9 +37,9 @@ def setUp(self):
         self.toilDir = os.path.join(self.tempDir, "jobstore")
         self.assertFalse(os.path.exists(self.toilDir))
         self.lines = 1000
-        self.maxLineLength = 10
+        self.lineLen = 10
         self.N = 1000
-        makeFileToSort(self.tempFile, self.lines, self.maxLineLength)
+        makeFileToSort(self.tempFile, self.lines, self.lineLen)
         # First make our own sorted version
         with open(self.tempFile, 'r') as fileHandle:
             self.correctSort = fileHandle.readlines()
@@ -115,7 +115,7 @@ def testUtilsSort(self):
         # Check the file is properly sorted
         with open(self.tempFile, 'r') as fileHandle:
             l2 = fileHandle.readlines()
-            checkEqual(self.correctSort, l2)
+            self.assertEquals(self.correctSort, l2)
 
         # Check we can run 'toil clean'
         toilCleanString = ("{self.toilMain} clean "
@@ -147,7 +147,7 @@ def testUtilsStatsSort(self):
         # Check the file is properly sorted
         with open(self.tempFile, 'r') as fileHandle:
             l2 = fileHandle.readlines()
-            checkEqual(self.correctSort, l2)
+            self.assertEquals(self.correctSort, l2)
 
 
 if __name__ == '__main__':