Repository: DataBiosphere/toil

Issue:
HTTP import test is flappy
http://jenkins.cgcloud.info/job/toil-branches/branch/releases%252F3.2.x/7/testReport/junit/src.toil.test.jobStores.jobStoreTest/FileJobStoreTest/testImportHttpFile/



Discussion:
This should be fixed on the 3.2.x branch and master.

I want it fixed on the 3.2.x branch to avoid this test from getting in the way of future point releases on that branch.


Resolving patch:
diff --git a/src/toil/jobStores/abstractJobStore.py b/src/toil/jobStores/abstractJobStore.py
--- a/src/toil/jobStores/abstractJobStore.py
+++ b/src/toil/jobStores/abstractJobStore.py
@@ -16,14 +16,17 @@
 import urllib2
 import urlparse
 
+import shutil
+
 import re
 from abc import ABCMeta, abstractmethod
-from contextlib import contextmanager
+from contextlib import contextmanager, closing
 from datetime import timedelta
 from uuid import uuid4
 from toil.job import JobException
 from bd2k.util import memoize
 from bd2k.util.objects import abstractclassmethod
+from toil.jobStores.utils import retry_http
 
 try:
     import cPickle
@@ -924,7 +927,6 @@ def _requireValidSharedFileName(cls, sharedFileName):
 
 
 class JobStoreSupport(AbstractJobStore):
-
     __metaclass__ = ABCMeta
 
     @classmethod
@@ -933,10 +935,7 @@ def _supportsUrl(cls, url, export=False):
 
     @classmethod
     def _readFromUrl(cls, url, writable):
-        out = urllib2.urlopen(url.geturl())
-        while True:
-            toWrite = out.read(2**20)
-            if toWrite in ('', None):
-                break
-            else:
-                writable.write(toWrite)
+        for attempt in retry_http():
+            with attempt:
+                with closing(urllib2.urlopen(url.geturl())) as readable:
+                    shutil.copyfileobj(readable, writable)
diff --git a/src/toil/jobStores/utils.py b/src/toil/jobStores/utils.py
--- a/src/toil/jobStores/utils.py
+++ b/src/toil/jobStores/utils.py
@@ -1,4 +1,5 @@
 import logging
+import urllib2
 from contextlib import contextmanager
 
 import time
@@ -109,3 +110,26 @@ def single_attempt():
             yield
 
         yield single_attempt()
+
+default_delays = (0, 1, 1, 4, 16, 64)
+default_timeout = 300
+
+
+def retryable_http_error(e):
+    return isinstance(e, urllib2.HTTPError) and e.code in ('503', '408', '500')
+
+
+def retry_http(delays=default_delays, timeout=default_timeout, predicate=retryable_http_error):
+    """
+    >>> i = 0
+    >>> for attempt in retry_http(timeout=5):
+    ...     with attempt:
+    ...         i += 1
+    ...         raise urllib2.HTTPError('http://www.test.com', '408', 'some message', {}, None)
+    Traceback (most recent call last):
+    ...
+    HTTPError: HTTP Error 408: some message
+    >>> i > 1
+    True
+    """
+    return retry(delays=delays, timeout=timeout, predicate=predicate)


Test patch:
diff --git a/src/toil/test/jobStores/jobStoreTest.py b/src/toil/test/jobStores/jobStoreTest.py
--- a/src/toil/test/jobStores/jobStoreTest.py
+++ b/src/toil/test/jobStores/jobStoreTest.py
@@ -378,7 +378,7 @@ def cleanUpExternalStores(cls):
                 logger.info('Cleaning up external store for %s.', test)
                 test._cleanUpExternalStore(store)
 
-        mpTestPartSize = 5 * 2 ** 20
+        mpTestPartSize = 5 << 20
 
         @classmethod
         def makeImportExportTests(cls):
@@ -448,15 +448,11 @@ def testImportSharedFile(self, otherCls):
                        otherCls=activeTestClassesByName)
 
         def testImportHttpFile(self):
-            # Prepare random file for import
-            self.master.partSize = self.mpTestPartSize
-            srcUrl = 'https://raw.githubusercontent.com/BD2KGenomics/toil/master/Makefile'
-            srcHash = hashlib.md5(urllib2.urlopen(srcUrl).read()).hexdigest()
-            # test import
-            jobStoreFileID = self.master.importFile(srcUrl)
-            with self.master.readFileStream(jobStoreFileID) as f:
-                fileMD5 = hashlib.md5(f.read()).hexdigest()
-            self.assertEqual(fileMD5, srcHash)
+            srcUrl, srcHash = ('https://raw.githubusercontent.com/BD2KGenomics/toil'
+                               '/12c711ff91caa5d8fb37a072bad376f447faa797/Makefile',
+                               '16a38fd7ae233616083d43d81159d4e8')
+            with self.master.readFileStream(self.master.importFile(srcUrl)) as f:
+                self.assertEqual(hashlib.md5(f.read()).hexdigest(), srcHash)
 
         def testFileDeletion(self):
             """