Repository: DataBiosphere/toil

Issue:
Azure File Store Very Slow Download Speed
My jobs that run on the Azure file store are hanging when downloading large files (where "large" is about 30 or 40 megabytes).

Here is a log on a singleMachine batch system with the Azure job store. It can run 2 jobs at a time (since I've set max cores to 32 and set it to use 16 cores in all my jobs), and right now those two jobs are both stuck trying to pull files out of the global file store: the cactus.tar.gz file should be 33 MB, as it was when generated using the file job store, and the snp1000g.tar.gz file should be 36 MB. It is taking far longer to download these files from the file store than it took to upload them; it's been waiting 11 minutes so far, for an overall download speed (on Kolossus, from Azure) of 0.10 MB/s or less. The line it seems to be hung in in my code is:

```
local_path = job.fileStore.readGlobalFile(index_dir_id)
```

The log from my code is here:
[logAzureStoreDebug.txt](https://github.com/BD2KGenomics/toil/files/41182/logAzureStoreDebug.txt)

It seems to work fine on the indexes for my smaller graphs, about 1-2 megabytes in size.

Unless this can get fixed quickly (or is not present in the current Toil release; I'm using master from a couple of days ago), I don't think I will be able to do my scale-up that I have planned for tomorrow.



Discussion:

Resolving patch:
diff --git a/src/toil/jobStores/azureJobStore.py b/src/toil/jobStores/azureJobStore.py
--- a/src/toil/jobStores/azureJobStore.py
+++ b/src/toil/jobStores/azureJobStore.py
@@ -190,7 +190,10 @@ def readFile(self, jobStoreFileID, localFilePath):
         try:
             with self._downloadStream(jobStoreFileID, self.files) as read_fd:
                 with open(localFilePath, 'w') as write_fd:
-                    write_fd.write(read_fd.read(self._maxAzureBlockBytes))
+                    while True:
+                        buf = read_fd.read(self._maxAzureBlockBytes)
+                        write_fd.write(buf)
+                        if not buf: break
         except WindowsAzureMissingResourceError:
             raise NoSuchFileException(jobStoreFileID)
 
@@ -347,7 +350,8 @@ def _sanitizeTableName(self, tableName):
         return 'a' + filter(lambda x: x.isalnum(), tableName)
 
     # Maximum bytes that can be in any block of an Azure block blob
-    _maxAzureBlockBytes = 4194000
+    # https://github.com/Azure/azure-storage-python/blob/4c7666e05a9556c10154508335738ee44d7cb104/azure/storage/blob/blobservice.py#L106
+    _maxAzureBlockBytes = 4 * 1024 * 1024
 
     @contextmanager
     def _uploadStream(self, jobStoreFileID, container, checkForModification=False, encrypted=None):


Test patch:
diff --git a/src/toil/test/jobStores/jobStoreTest.py b/src/toil/test/jobStores/jobStoreTest.py
--- a/src/toil/test/jobStores/jobStoreTest.py
+++ b/src/toil/test/jobStores/jobStoreTest.py
@@ -394,6 +394,27 @@ def testZeroLengthFiles(self):
                 self.assertEquals(f.read(), "")
             self.master.delete(job.jobStoreID)
 
+        def testLargeFile(self):
+            dirPath = self._createTempDir()
+            filePath = os.path.join(dirPath, 'large')
+            hashIn = hashlib.md5()
+            with open(filePath, 'w') as f:
+                for i in xrange(0, 10):
+                    buf = os.urandom(self._partSize())
+                    f.write(buf)
+                    hashIn.update(buf)
+            job = self.master.create('1', 2, 3, 4, 0)
+            jobStoreFileID = self.master.writeFile(filePath, job.jobStoreID)
+            os.unlink(filePath)
+            self.master.readFile(jobStoreFileID, filePath)
+            hashOut = hashlib.md5()
+            with open(filePath, 'r') as f:
+                while True:
+                    buf = f.read(self._partSize())
+                    if not buf: break
+                    hashOut.update(buf)
+            self.assertEqual(hashIn.digest(), hashOut.digest())
+
         def assertUrl(self, url):
             prefix, path = url.split(':', 1)
             if prefix == 'file':
@@ -476,27 +497,6 @@ def testInlinedFiles(self):
                 with master.readSharedFileStream('foo') as f:
                     self.assertEqual(s, f.read())
 
-    def testLargeFile(self):
-        dirPath = self._createTempDir()
-        filePath = os.path.join(dirPath, 'large')
-        hashIn = hashlib.md5()
-        with open(filePath, 'w') as f:
-            for i in xrange(0, 10):
-                buf = os.urandom(self._partSize())
-                f.write(buf)
-                hashIn.update(buf)
-        job = self.master.create('1', 2, 3, 4, 0)
-        jobStoreFileID = self.master.writeFile(filePath, job.jobStoreID)
-        os.unlink(filePath)
-        self.master.readFile(jobStoreFileID, filePath)
-        hashOut = hashlib.md5()
-        with open(filePath, 'r') as f:
-            while True:
-                buf = f.read(self._partSize())
-                if not buf: break
-                hashOut.update(buf)
-        self.assertEqual(hashIn.digest(), hashOut.digest())
-
     def _largeLogEntrySize(self):
         from toil.jobStores.aws.jobStore import AWSJobStore
         # So we get into the else branch of reader() in uploadStream(multiPart=False):
@@ -528,6 +528,10 @@ def _createJobStore(self, config=None):
         from toil.jobStores.azureJobStore import AzureJobStore
         return AzureJobStore('toiltest', self.namePrefix, config=config, jobChunkSize=128)
 
+    def _partSize(self):
+        from toil.jobStores.azureJobStore import AzureJobStore
+        return AzureJobStore._maxAzureBlockBytes
+
 
 class EncryptedFileJobStoreTest(FileJobStoreTest, hidden.AbstractEncryptedJobStoreTest):
     pass