Repository: DataBiosphere/toil

Issue:
Bring Azure job store under CI



Discussion:

Resolving patch:
diff --git a/src/toil/jobStores/azureJobStore.py b/src/toil/jobStores/azureJobStore.py
--- a/src/toil/jobStores/azureJobStore.py
+++ b/src/toil/jobStores/azureJobStore.py
@@ -366,8 +366,10 @@ def updateFileStream(self, jobStoreFileID):
 
     def getEmptyFileStoreID(self, jobStoreID=None):
         jobStoreFileID = self._newFileID()
-        self.files.put_blob(blob_name=jobStoreFileID, blob='',
-                            x_ms_blob_type='BlockBlob')
+        for attempt in retry_azure(timeout=45):
+            with attempt:
+                self.files.put_blob(blob_name=jobStoreFileID, blob='',
+                                    x_ms_blob_type='BlockBlob')
         self._associateFileWithJob(jobStoreFileID, jobStoreID)
         return jobStoreFileID
 
@@ -409,22 +411,24 @@ def writeStatsAndLogging(self, statsAndLoggingString):
     def readStatsAndLogging(self, callback, readAll=False):
         suffix = '_old'
         numStatsFiles = 0
-        for entity in self.statsFileIDs.query_entities():
-            jobStoreFileID = entity.RowKey
-            hasBeenRead = len(jobStoreFileID) > self.jobIDLength
-            if not hasBeenRead:
-                with self._downloadStream(jobStoreFileID, self.statsFiles) as fd:
-                    callback(fd)
-                # Mark this entity as read by appending the suffix
-                self.statsFileIDs.insert_entity(entity={'RowKey': jobStoreFileID + suffix})
-                self.statsFileIDs.delete_entity(row_key=jobStoreFileID)
-                numStatsFiles += 1
-            elif readAll:
-                # Strip the suffix to get the original ID
-                jobStoreFileID = jobStoreFileID[:-len(suffix)]
-                with self._downloadStream(jobStoreFileID, self.statsFiles) as fd:
-                    callback(fd)
-                numStatsFiles += 1
+        for attempt in retry_azure():
+            with attempt:
+                for entity in self.statsFileIDs.query_entities():
+                    jobStoreFileID = entity.RowKey
+                    hasBeenRead = len(jobStoreFileID) > self.jobIDLength
+                    if not hasBeenRead:
+                        with self._downloadStream(jobStoreFileID, self.statsFiles) as fd:
+                            callback(fd)
+                        # Mark this entity as read by appending the suffix
+                        self.statsFileIDs.insert_entity(entity={'RowKey': jobStoreFileID + suffix})
+                        self.statsFileIDs.delete_entity(row_key=jobStoreFileID)
+                        numStatsFiles += 1
+                    elif readAll:
+                        # Strip the suffix to get the original ID
+                        jobStoreFileID = jobStoreFileID[:-len(suffix)]
+                        with self._downloadStream(jobStoreFileID, self.statsFiles) as fd:
+                            callback(fd)
+                        numStatsFiles += 1
         return numStatsFiles
 
     _azureTimeFormat = "%Y-%m-%dT%H:%M:%SZ"
@@ -815,7 +819,8 @@ def defaultRetryPredicate(exception):
     return (isinstance(exception, (socket.error,
                                    socket.gaierror,
                                    HTTPException,
-                                   requests.ConnectionError))
+                                   requests.ConnectionError,
+                                   requests.Timeout))
             or isinstance(exception, AzureException) and
             any(message in str(exception).lower() for message in (
                 "could not be completed within the specified time",


Test patch:
diff --git a/src/toil/test/jobStores/jobStoreTest.py b/src/toil/test/jobStores/jobStoreTest.py
--- a/src/toil/test/jobStores/jobStoreTest.py
+++ b/src/toil/test/jobStores/jobStoreTest.py
@@ -1041,7 +1041,6 @@ def testInvalidJobStoreName(self):
                           'us-west-2:a_b')
 
 
-@experimental
 @needs_azure
 class AzureJobStoreTest(AbstractJobStoreTest.Test):
     accountName = 'toiltest'
@@ -1116,7 +1115,6 @@ def _cleanUpExternalStore(self, containerName):
         blobService.delete_container(containerName)
 
 
-@experimental
 @needs_azure
 class InvalidAzureJobStoreTest(ToilTest):
     def testInvalidJobStoreName(self):
@@ -1142,7 +1140,6 @@ class EncryptedAWSJobStoreTest(AWSJobStoreTest, AbstractEncryptedJobStoreTest.Te
     pass
 
 
-@experimental
 @needs_azure
 @needs_encryption
 class EncryptedAzureJobStoreTest(AzureJobStoreTest, AbstractEncryptedJobStoreTest.Test):
diff --git a/src/toil/test/sort/sortTest.py b/src/toil/test/sort/sortTest.py
--- a/src/toil/test/sort/sortTest.py
+++ b/src/toil/test/sort/sortTest.py
@@ -186,12 +186,10 @@ def testFileMesos(self):
         finally:
             self._stopMesos()
 
-    @experimental
     @needs_azure
     def testAzureSingle(self):
         self._toilSort(jobStoreLocator=self._azureJobStore(), batchSystem='singleMachine')
 
-    @experimental
     @needs_azure
     @needs_mesos
     def testAzureMesos(self):
diff --git a/src/toil/test/src/fileStoreTest.py b/src/toil/test/src/fileStoreTest.py
--- a/src/toil/test/src/fileStoreTest.py
+++ b/src/toil/test/src/fileStoreTest.py
@@ -1288,13 +1288,11 @@ class CachingFileStoreTestWithAwsJobStore(hidden.AbstractCachingFileStoreTest):
 
 
 @needs_azure
-@experimental
 class NonCachingFileStoreTestWithAzureJobStore(hidden.AbstractNonCachingFileStoreTest):
     jobStoreType = 'azure'
 
 
 @needs_azure
-@experimental
 class CachingFileStoreTestWithAzureJobStore(hidden.AbstractCachingFileStoreTest):
     jobStoreType = 'azure'