Repository: DataBiosphere/toil

Issue:
Add tracking of disk to single machine
Should be straight-forward to implement but it will increase the likelihood of deadlock due to the  nested, blocking nature of resource allocation in SingleMachine.

<!---
@huboard:{"milestone_order":253.94920761898413}
-->



Discussion:
Add a semaphore for `disk` and initialize it with `df` when batch system starts up.


Resolving patch:
diff --git a/src/toil/__init__.py b/src/toil/__init__.py
--- a/src/toil/__init__.py
+++ b/src/toil/__init__.py
@@ -77,3 +77,12 @@ def physicalMemory():
         return os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
     except ValueError:
         return int(check_output(['sysctl', '-n', 'hw.memsize']).strip())
+
+
+def physicalDisk(config, toilWorkflowDir=None):
+    if toilWorkflowDir is None:
+        from toil.common import Toil
+        toilWorkflowDir = Toil.getWorkflowDir(config.workflowID, config.workDir)
+    diskStats = os.statvfs(toilWorkflowDir)
+    return diskStats.f_frsize * diskStats.f_bavail
+
diff --git a/src/toil/batchSystems/abstractBatchSystem.py b/src/toil/batchSystems/abstractBatchSystem.py
--- a/src/toil/batchSystems/abstractBatchSystem.py
+++ b/src/toil/batchSystems/abstractBatchSystem.py
@@ -352,5 +352,6 @@ def __init__(self, resource, requested, available):
         self.resource = resource
 
     def __str__(self):
-        return 'Requesting more {} than available. Requested: {}, Available: {}' \
-               ''.format(self.resource, self.requested, self.available)
+        return 'Requesting more {} than either physically available, or enforced by --max{}. ' \
+               'Requested: {}, Available: {}'.format(self.resource, self.resource.capitalize(),
+                                                     self.requested, self.available)
diff --git a/src/toil/batchSystems/singleMachine.py b/src/toil/batchSystems/singleMachine.py
--- a/src/toil/batchSystems/singleMachine.py
+++ b/src/toil/batchSystems/singleMachine.py
@@ -25,7 +25,7 @@
 from Queue import Queue, Empty
 
 import toil
-from toil.batchSystems.abstractBatchSystem import BatchSystemSupport, AbstractBatchSystem
+from toil.batchSystems.abstractBatchSystem import BatchSystemSupport, InsufficientSystemResources
 
 log = logging.getLogger(__name__)
 
@@ -62,6 +62,10 @@ def __init__(self, config, maxCores, maxMemory, maxDisk):
         if maxMemory > self.physicalMemory:
             log.warn('Limiting maxMemory to physically available memory (%i).', self.physicalMemory)
             maxMemory = self.physicalMemory
+        self.physicalDisk = toil.physicalDisk(config)
+        if maxDisk > self.physicalDisk:
+            log.warn('Limiting maxDisk to physically available disk (%i).', self.physicalDisk)
+            maxDisk = self.physicalDisk
         super(SingleMachineBatchSystem, self).__init__(config, maxCores, maxMemory, maxDisk)
         assert self.maxCores >= self.minCores
         assert self.maxMemory >= 1
@@ -94,12 +98,20 @@ def __init__(self, config, maxCores, maxMemory, maxDisk):
         """
         :type list[Thread]
         """
+        # Variables involved with non-blocking resource acquisition
+        self.acquisitionTimeout = 5
+        self.acquisitionRetryDelay = 10
+        self.aquisitionCondition = Condition()
+
         # A pool representing available CPU in units of minCores
-        self.coreFractions = ResourcePool(self.numWorkers)
+        self.coreFractions = ResourcePool(self.numWorkers, 'cores', self.acquisitionTimeout)
         # A lock to work around the lack of thread-safety in Python's subprocess module
         self.popenLock = Lock()
         # A pool representing available memory in bytes
-        self.memory = ResourcePool(self.maxMemory)
+        self.memory = ResourcePool(self.maxMemory, 'memory', self.acquisitionTimeout)
+        # A pool representing the available space in bytes
+        self.disk = ResourcePool(self.maxDisk, 'disk', self.acquisitionTimeout)
+
         log.info('Setting up the thread pool with %i workers, '
                  'given a minimum CPU fraction of %f '
                  'and a maximum CPU value of %i.', self.numWorkers, self.minCores, maxCores)
@@ -118,37 +130,56 @@ def worker(self, inputQueue):
                 log.debug('Received queue sentinel.')
                 break
             jobCommand, jobID, jobCores, jobMemory, jobDisk, environment = args
-            try:
-                coreFractions = int(jobCores / self.minCores)
-                log.debug('Acquiring %i bytes of memory from a pool of %s.', jobMemory, self.memory)
-                with self.memory.acquisitionOf(jobMemory):
-                    log.debug('Acquiring %i fractional cores from a pool of %s to satisfy a '
-                              'request of %f cores', coreFractions, self.coreFractions, jobCores)
-                    with self.coreFractions.acquisitionOf(coreFractions):
-                        log.info("Executing command: '%s'.", jobCommand)
-                        startTime = time.time() #Time job is started
-                        with self.popenLock:
-                            popen = subprocess.Popen(jobCommand,
-                                                     shell=True,
-                                                     env=dict(os.environ, **environment))
-                        statusCode = None
-                        info = Info(time.time(), popen, killIntended=False)
-                        try:
-                            self.runningJobs[jobID] = info
-                            try:
-                                statusCode = popen.wait()
-                                if 0 != statusCode:
-                                    if statusCode != -9 or not info.killIntended:
-                                        log.error("Got exit code %i (indicating failure) from "
-                                                  "command '%s'.", statusCode, jobCommand)
-                            finally:
-                                self.runningJobs.pop(jobID)
-                        finally:
-                            if statusCode is not None and not info.killIntended:
-                                self.outputQueue.put((jobID, statusCode, time.time() - startTime))
-            finally:
-                log.debug('Finished job. self.coreFractions ~ %s and self.memory ~ %s',
-                          self.coreFractions.value, self.memory.value)
+            while True:
+                try:
+                    coreFractions = int(jobCores / self.minCores)
+                    log.debug('Acquiring %i bytes of memory from a pool of %s.', jobMemory,
+                              self.memory)
+                    with self.memory.acquisitionOf(jobMemory):
+                        log.debug('Acquiring %i fractional cores from a pool of %s to satisfy a '
+                                  'request of %f cores', coreFractions, self.coreFractions,
+                                  jobCores)
+                        with self.coreFractions.acquisitionOf(coreFractions):
+                            with self.disk.acquisitionOf(jobDisk):
+                                log.info("Executing command: '%s'.", jobCommand)
+                                startTime = time.time() #Time job is started
+                                with self.popenLock:
+                                    popen = subprocess.Popen(jobCommand,
+                                                             shell=True,
+                                                             env=dict(os.environ, **environment))
+                                statusCode = None
+                                info = Info(time.time(), popen, killIntended=False)
+                                try:
+                                    self.runningJobs[jobID] = info
+                                    try:
+                                        statusCode = popen.wait()
+                                        if 0 != statusCode:
+                                            if statusCode != -9 or not info.killIntended:
+                                                log.error("Got exit code %i (indicating failure) "
+                                                          "from command '%s'.", statusCode,
+                                                          jobCommand)
+                                    finally:
+                                        self.runningJobs.pop(jobID)
+                                finally:
+                                    if statusCode is not None and not info.killIntended:
+                                        self.outputQueue.put((jobID, statusCode,
+                                                              time.time() - startTime))
+                except ResourcePool.AcquisitionTimeoutException as e:
+                    log.debug('Could not acquire enough (%s) to run job. Requested: (%s), '
+                              'Avaliable: %s. Sleeping for 10s.', e.resource, e.requested,
+                              e.available)
+                    with self.aquisitionCondition:
+                        # Make threads sleep for the given delay, or until another job finishes.
+                        # Whichever is sooner.
+                        self.aquisitionCondition.wait(timeout=self.acquisitionRetryDelay)
+                    continue
+                else:
+                    log.debug('Finished job. self.coreFractions ~ %s and self.memory ~ %s',
+                              self.coreFractions.value, self.memory.value)
+                    with self.aquisitionCondition:
+                        # Wake up sleeping threads
+                        self.aquisitionCondition.notifyAll()
+                    break
         log.debug('Exiting worker thread normally.')
 
     def issueBatchJob(self, command, memory, cores, disk, preemptable):
@@ -241,15 +272,28 @@ def __init__(self, startTime, popen, killIntended):
 
 
 class ResourcePool(object):
-    def __init__(self, initial_value):
+    def __init__(self, initial_value, resourceType, timeout):
         super(ResourcePool, self).__init__()
         self.condition = Condition()
         self.value = initial_value
+        self.resourceType = resourceType
+        self.timeout = timeout
 
     def acquire(self, amount):
         with self.condition:
+            startTime = time.time()
             while amount > self.value:
-                self.condition.wait()
+                if time.time() - startTime >= self.timeout:
+                    # This means the thread timed out waiting for the resource.  We exit the nested
+                    # context managers in worker to prevent blocking of a resource due to
+                    # unavailability of a nested resource request.
+                    raise self.AcquisitionTimeoutException(resource=self.resourceType,
+                                                           requested=amount, available=self.value)
+                # Allow 5 seconds to get the resource, else quit through the above if condition.
+                # This wait + timeout is the last thing in the loop such that a request that takes
+                # longer than 5s due to multiple wakes under the 5 second threshold are still
+                # honored.
+                self.condition.wait(timeout=self.timeout)
             self.value -= amount
             self.__validate()
 
@@ -275,3 +319,26 @@ def acquisitionOf(self, amount):
             yield
         finally:
             self.release(amount)
+
+    class AcquisitionTimeoutException(Exception):
+        """
+        To be raised when a resource request times out.
+        """
+
+        def __init__(self, resource, requested, available):
+            """
+            Creates an instance of this exception that indicates which resource is insufficient for
+            current demands, as well as the amount requested and amount actually available.
+
+            :param str resource: string representing the resource type
+
+            :param int|float requested: the amount of the particular resource requested that resulted
+                   in this exception
+
+            :param int|float available: amount of the particular resource actually available
+            """
+            self.requested = requested
+            self.available = available
+            self.resource = resource
+
+


Test patch:
diff --git a/src/toil/test/batchSystems/batchSystemTest.py b/src/toil/test/batchSystems/batchSystemTest.py
--- a/src/toil/test/batchSystems/batchSystemTest.py
+++ b/src/toil/test/batchSystems/batchSystemTest.py
@@ -19,6 +19,7 @@
 import logging
 import os
 import fcntl
+import itertools
 import tempfile
 from textwrap import dedent
 import time
@@ -283,7 +284,7 @@ def supportsWallTime(self):
 
     def createBatchSystem(self):
         return SingleMachineBatchSystem(config=self.config,
-                                        maxCores=numCores, maxMemory=1e9, maxDisk=1001)
+                                        maxCores=numCores, maxMemory=1e9, maxDisk=2001)
 
 
 class MaxCoresSingleMachineBatchSystemTest(ToilTest):
@@ -577,6 +578,120 @@ class SingleMachineBatchSystemJobTest(hidden.AbstractBatchSystemJobTest):
     def getBatchSystemName(self):
         return "singleMachine"
 
+    def testConcurrencyWithDisk(self):
+        """
+        Tests that the batch system is allocating disk resources properly
+        """
+        tempDir = self._createTempDir('testFiles')
+
+        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
+        options.workDir = tempDir
+        from toil import physicalDisk
+        availableDisk = physicalDisk('', toilWorkflowDir=options.workDir)
+        options.batchSystem = self.batchSystemName
+
+        counterPath = os.path.join(tempDir, 'counter')
+        resetCounters(counterPath)
+        value, maxValue = getCounters(counterPath)
+        assert (value, maxValue) == (0, 0)
+
+        root = Job()
+        # Physically, we're asking for 50% of disk and 50% of disk + 500bytes in the two jobs. The
+        # batchsystem should not allow the 2 child jobs to run concurrently.
+        root.addChild(Job.wrapFn(measureConcurrency, counterPath, self.sleepTime, cores=1,
+                                    memory='1M', disk=availableDisk/2))
+        root.addChild(Job.wrapFn(measureConcurrency, counterPath, self.sleepTime, cores=1,
+                                 memory='1M', disk=(availableDisk / 2) + 500))
+        Job.Runner.startToil(root, options)
+        _, maxValue = getCounters(counterPath)
+        self.assertEqual(maxValue, 1)
+
+    @skipIf(SingleMachineBatchSystem.numCores < 4, 'Need at least four cores to run this test')
+    def testNestedResourcesDoNotBlock(self):
+        """
+        Resources are requested in the order Memory > Cpu > Disk.
+        Test that inavailability of cpus for one job that is scheduled does not block another job
+        that can run.
+        """
+        tempDir = self._createTempDir('testFiles')
+
+        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
+        options.workDir = tempDir
+        options.maxCores = 4
+        from toil import physicalMemory
+        availableMemory = physicalMemory()
+        options.batchSystem = self.batchSystemName
+
+        outFile = os.path.join(tempDir, 'counter')
+        open(outFile, 'w').close()
+
+        root = Job()
+
+        blocker = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=30, writeVal='b',
+                             cores=2, memory='1M', disk='1M')
+        firstJob = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5, writeVal='fJ',
+                              cores=1, memory='1M', disk='1M')
+        secondJob = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=10,
+                               writeVal='sJ', cores=1, memory='1M', disk='1M')
+
+        # Should block off 50% of memory while waiting for it's 3 cores
+        firstJobChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=0,
+                                   writeVal='fJC', cores=3, memory=(availableMemory/2), disk='1M')
+
+        # These two shouldn't be able to run before B because there should be only
+        # (50% of memory - 1M) available (firstJobChild should be blocking 50%)
+        secondJobChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5,
+                                    writeVal='sJC', cores=2, memory=(availableMemory/1.5),
+                                    disk='1M')
+        secondJobGrandChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5,
+                                         writeVal='sJGC', cores=2, memory=(availableMemory/1.5),
+                                         disk='1M')
+
+        root.addChild(blocker)
+        root.addChild(firstJob)
+        root.addChild(secondJob)
+
+        firstJob.addChild(firstJobChild)
+        secondJob.addChild(secondJobChild)
+
+        secondJobChild.addChild(secondJobGrandChild)
+        """
+        The tree is:
+                    root
+                  /   |   \
+                 b    fJ   sJ
+                      |    |
+                      fJC  sJC
+                           |
+                           sJGC
+        But the order of execution should be
+        root > b , fJ, sJ > sJC > sJGC > fJC
+        since fJC cannot run till bl finishes but sJC and sJGC can(fJC blocked by disk). If the
+        resource acquisition is written properly, then fJC which is scheduled before sJC and sJGC
+        should not block them, and should only run after they finish.
+        """
+        Job.Runner.startToil(root, options)
+        with open(outFile) as oFH:
+            outString = oFH.read()
+        # The ordering of b, fJ and sJ is non-deterministic since they are scheduled at the same
+        # time. We look for all possible permutations.
+        possibleStarts = tuple([''.join(x) for x in itertools.permutations(['b', 'fJ', 'sJ'])])
+        assert outString.startswith(possibleStarts)
+        assert outString.endswith('sJCsJGCfJC')
+
+
+def _resourceBlockTestAuxFn(outFile, sleepTime, writeVal):
+    """
+    Write a value to the out file and then sleep for requested seconds.
+    :param str outFile: File to write to
+    :param int sleepTime: Time to sleep for
+    :param str writeVal: Character to write
+    """
+    with open(outFile, 'a') as oFH:
+        fcntl.flock(oFH, fcntl.LOCK_EX)
+        oFH.write(writeVal)
+    time.sleep(sleepTime)
+
 
 @needs_mesos
 class MesosBatchSystemJobTest(hidden.AbstractBatchSystemJobTest, MesosTestSupport):