Repository: DataBiosphere/toil

Issue:
Add logging for each job's disk resources used.



Discussion:

Resolving patch:
diff --git a/src/toil/common.py b/src/toil/common.py
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -32,6 +32,10 @@
 
 logger = logging.getLogger(__name__)
 
+# This constant is set to the default value used on unix for block size (in bytes) when
+# os.stat(<file>).st_blocks is called.
+unixBlockSize = 512
+
 
 class Config(object):
     """
@@ -927,3 +931,34 @@ def cacheDirName(workflowID):
     :return: Name of the cache directory.
     """
     return 'cache-' + workflowID
+
+
+def getDirSizeRecursively(dirPath):
+    """
+    This method will walk through a directory and return the cumulative filesize in bytes of all
+    the files in the directory and its subdirectories.
+
+    :param dirPath: Path to a directory.
+    :return: cumulative size in bytes of all files in the directory.
+    :rtype: int
+    """
+    totalSize = 0
+    # The value from running stat on each linked file is equal. To prevent the same file
+    # from being counted multiple times, we save the inodes of files that have more than one
+    # nlink associated with them.
+    seenInodes = set()
+    for dirPath, dirNames, fileNames in os.walk(dirPath):
+        folderSize = 0
+        for f in fileNames:
+            fp = os.path.join(dirPath, f)
+            fileStats = os.stat(fp)
+            if fileStats.st_nlink > 1:
+                if fileStats.st_ino not in seenInodes:
+                    folderSize += fileStats.st_blocks * unixBlockSize
+                    seenInodes.add(fileStats.st_ino)
+                else:
+                    continue
+            else:
+                folderSize += fileStats.st_blocks * unixBlockSize
+        totalSize += folderSize
+    return totalSize
diff --git a/src/toil/fileStore.py b/src/toil/fileStore.py
--- a/src/toil/fileStore.py
+++ b/src/toil/fileStore.py
@@ -33,7 +33,8 @@
 from Queue import Queue, Empty
 from threading import Thread, Semaphore, Event
 
-from toil.common import cacheDirName
+from bd2k.util.humanize import bytes2human
+from toil.common import cacheDirName, getDirSizeRecursively
 from toil.lib.bioio import makePublicDir
 
 logger = logging.getLogger( __name__ )
@@ -132,6 +133,22 @@ def open(self, job):
             os.chdir(self.localTempDir)
             yield
         finally:
+            diskUsed = getDirSizeRecursively(self.localTempDir)
+            logString = ("Job {jobName} used {percent:.2f}% ({humanDisk}B [{disk}B] used, "
+                         "{humanRequestedDisk}B [{requestedDisk}B] requested) at the end of "
+                         "its run.".format(jobName=self.jobName,
+                                           percent=(float(diskUsed) / jobReqs * 100 if
+                                                    jobReqs > 0 else 0.0),
+                                           humanDisk=bytes2human(diskUsed),
+                                           disk=diskUsed,
+                                           humanRequestedDisk=bytes2human(jobReqs),
+                                           requestedDisk=jobReqs))
+            self.logToMaster(logString, level=logging.INFO)
+            if diskUsed > jobReqs:
+                self.logToMaster("Job used more disk than requested. Please reconsider modifying "
+                                 "the user script to avoid the chance  of failure due to "
+                                 "incorrectly requested resources. " + logString,
+                                 level=logging.WARNING)
             os.chdir(startingDir)
             self.cleanupInProgress = True
             # Delete all the job specific files and return sizes to jobReqs
@@ -1429,6 +1446,7 @@ class NonCachingFileStore(FileStore):
     def __init__(self, jobStore, jobWrapper, localTempDir, inputBlockFn):
         self.jobStore = jobStore
         self.jobWrapper = jobWrapper
+        self.jobName = self.jobWrapper.command.split()[1]
         self.localTempDir = os.path.abspath(localTempDir)
         self.inputBlockFn = inputBlockFn
         self.jobsToDelete = set()
@@ -1438,12 +1456,24 @@ def __init__(self, jobStore, jobWrapper, localTempDir, inputBlockFn):
 
     @contextmanager
     def open(self, job):
+        jobReqs = job.disk
         startingDir = os.getcwd()
         self.localTempDir = makePublicDir(os.path.join(self.localTempDir, str(uuid.uuid4())))
         try:
             os.chdir(self.localTempDir)
             yield
         finally:
+            diskUsed = getDirSizeRecursively(self.localTempDir)
+            logString = "Job %s used %.2f%% (%sB [%sB] used, %sB [%sB] requested) at the end of " \
+                        "its run." % (self.jobName,
+                                      float(diskUsed) / jobReqs * 100 if jobReqs > 0 else 0.0,
+                                      bytes2human(diskUsed), diskUsed, bytes2human(jobReqs),
+                                      jobReqs)
+            self.logToMaster(logString, level=logging.INFO)
+            if diskUsed > jobReqs:
+                self.logToMaster("Job used more disk than requested. Cconsider modifying the user "
+                                 "script to avoid the chance of failure due to incorrectly "
+                                 "requested resources. " + logString, level=logging.WARNING)
             os.chdir(startingDir)
 
     def writeGlobalFile(self, localFileName, cleanup=False):


Test patch:
diff --git a/src/toil/test/src/miscTests.py b/src/toil/test/src/miscTests.py
new file mode 100644
--- /dev/null
+++ b/src/toil/test/src/miscTests.py
@@ -0,0 +1,87 @@
+# Copyright (C) 2015-2016 Regents of the University of California
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import, print_function
+from toil.test import ToilTest
+from uuid import uuid4
+
+import math
+import os
+import random
+import tempfile
+
+
+class MiscTests(ToilTest):
+    """
+    This class contains miscellaneous tests that don't have enough content to be their own test
+    file, and that don't logically fit in with any of the other test suites.
+    """
+    def setUp(self):
+        super(MiscTests, self).setUp()
+        self.testDir = self._createTempDir()
+
+    def testGetSizeOfDirectoryWorks(self):
+        from toil.common import getDirSizeRecursively
+        # os.stat lists the number of 512-byte blocks used, but really, the file system is using
+        # a blocksize specific to itself.  For instance, my machine uses 4096-byte blocks.
+        #
+        #  >>> with open('/tmp/temp', 'w') as fileHandle:
+        #  ...     fileHandle.write(os.urandom(512*3))
+        #  ...
+        #  >>> os.stat('/tmp/temp').st_blksize
+        #  4096
+        #  >>> os.stat('/tmp/temp')
+        #  posix.stat_result(st_mode=33188, st_ino=2630547, st_dev=16777220, st_nlink=1, st_uid=501,
+        #                    st_gid=0, st_size=1536, st_atime=1475620048, st_mtime=1475628030,
+        #                    st_ctime=1475628030)
+        #  >>> os.stat('/tmp/temp').st_blocks
+        #  8
+        #
+        # Even though the file is just 3 * 512 bytes, the file system uses one 4096-byte block and
+        # os.stat says it is using eight 512-byte blocks.
+        systemBlockSize = os.stat('.').st_blksize
+        # a list of the directories used in the test
+        directories = [self.testDir]
+        # A dict of {FILENAME: FILESIZE} for all files used in the test
+        files = {}
+        # Create a random directory structure
+        for i in xrange(0,10):
+            directories.append(tempfile.mkdtemp(dir=random.choice(directories), prefix='test'))
+        # Create 50 random file entries in different locations in the directories. 75% of the time
+        # these are fresh files of sixe [1, 10] MB and 25% of the time they are hard links to old
+        # files.
+        while len(files) <= 50:
+            fileName = os.path.join(random.choice(directories), self._getRandomName())
+            if random.randint(0,100) < 75:
+                # Create a fresh file in the range of 1-10 MB
+                fileSize = int(round(random.random(), 2) * 10 * 1024 * 1024)
+                with open(fileName, 'w') as fileHandle:
+                    fileHandle.write(os.urandom(fileSize))
+                files[fileName] = int(math.ceil(fileSize * 1.0 / systemBlockSize) * systemBlockSize)
+            else:
+                # Link to one of the previous files
+                if len(files) == 0:
+                    continue
+                linkSrc = random.choice(files.keys())
+                os.link(linkSrc, fileName)
+                files[fileName] = 'Link to %s' % linkSrc
+
+        computedDirectorySize = getDirSizeRecursively(self.testDir)
+        totalExpectedSize = sum([x for x in files.values() if isinstance(x, int)])
+        self.assertEqual(computedDirectorySize, totalExpectedSize)
+
+    @staticmethod
+    def _getRandomName():
+        return uuid4().hex
+