Repository: DataBiosphere/toil

Issue:
HTTP import test is flappy
http://jenkins.cgcloud.info/job/toil-branches/branch/releases%252F3.2.x/7/testReport/junit/src.toil.test.jobStores.jobStoreTest/FileJobStoreTest/testImportHttpFile/

HTTP import test is flappy
http://jenkins.cgcloud.info/job/toil-branches/branch/releases%252F3.2.x/7/testReport/junit/src.toil.test.jobStores.jobStoreTest/FileJobStoreTest/testImportHttpFile/



Discussion:
This should be fixed on the 3.2.x branch and master.

I want it fixed on the 3.2.x branch to avoid this test from getting in the way of future point releases on that branch.

This should be fixed on the 3.2.x branch and master.

I want it fixed on the 3.2.x branch to avoid this test from getting in the way of future point releases on that branch.


Resolving patch:
diff --git a/src/toil/jobStores/abstractJobStore.py b/src/toil/jobStores/abstractJobStore.py
--- a/src/toil/jobStores/abstractJobStore.py
+++ b/src/toil/jobStores/abstractJobStore.py
@@ -16,14 +16,17 @@
 import urllib2
 import urlparse
 
+import shutil
+
 import re
 from abc import ABCMeta, abstractmethod
-from contextlib import contextmanager
+from contextlib import contextmanager, closing
 from datetime import timedelta
 from uuid import uuid4
 from toil.job import JobException
 from bd2k.util import memoize
 from bd2k.util.objects import abstractclassmethod
+from toil.jobStores.utils import retry_http
 
 try:
     import cPickle
@@ -924,7 +927,6 @@ def _requireValidSharedFileName(cls, sharedFileName):
 
 
 class JobStoreSupport(AbstractJobStore):
-
     __metaclass__ = ABCMeta
 
     @classmethod
@@ -933,10 +935,7 @@ def _supportsUrl(cls, url, export=False):
 
     @classmethod
     def _readFromUrl(cls, url, writable):
-        out = urllib2.urlopen(url.geturl())
-        while True:
-            toWrite = out.read(2**20)
-            if toWrite in ('', None):
-                break
-            else:
-                writable.write(toWrite)
+        for attempt in retry_http():
+            with attempt:
+                with closing(urllib2.urlopen(url.geturl())) as readable:
+                    shutil.copyfileobj(readable, writable)
diff --git a/src/toil/jobStores/utils.py b/src/toil/jobStores/utils.py
--- a/src/toil/jobStores/utils.py
+++ b/src/toil/jobStores/utils.py
@@ -1,4 +1,5 @@
 import logging
+import urllib2
 from contextlib import contextmanager
 
 import time
@@ -109,3 +110,26 @@ def single_attempt():
             yield
 
         yield single_attempt()
+
+default_delays = (0, 1, 1, 4, 16, 64)
+default_timeout = 300
+
+
+def retryable_http_error(e):
+    return isinstance(e, urllib2.HTTPError) and e.code in ('503', '408', '500')
+
+
+def retry_http(delays=default_delays, timeout=default_timeout, predicate=retryable_http_error):
+    """
+    >>> i = 0
+    >>> for attempt in retry_http(timeout=5):
+    ...     with attempt:
+    ...         i += 1
+    ...         raise urllib2.HTTPError('http://www.test.com', '408', 'some message', {}, None)
+    Traceback (most recent call last):
+    ...
+    HTTPError: HTTP Error 408: some message
+    >>> i > 1
+    True
+    """
+    return retry(delays=delays, timeout=timeout, predicate=predicate)


Test patch:
diff --git a/src/toil/test/jobStores/jobStoreTest.py b/src/toil/test/jobStores/jobStoreTest.py
--- a/src/toil/test/jobStores/jobStoreTest.py
+++ b/src/toil/test/jobStores/jobStoreTest.py
@@ -378,7 +378,7 @@ def cleanUpExternalStores(cls):
                 logger.info('Cleaning up external store for %s.', test)
                 test._cleanUpExternalStore(store)
 
-        mpTestPartSize = 5 * 2 ** 20
+        mpTestPartSize = 5 << 20
 
         @classmethod
         def makeImportExportTests(cls):
@@ -448,15 +448,11 @@ def testImportSharedFile(self, otherCls):
                        otherCls=activeTestClassesByName)
 
         def testImportHttpFile(self):
-            # Prepare random file for import
-            self.master.partSize = self.mpTestPartSize
-            srcUrl = 'https://raw.githubusercontent.com/BD2KGenomics/toil/master/Makefile'
-            srcHash = hashlib.md5(urllib2.urlopen(srcUrl).read()).hexdigest()
-            # test import
-            jobStoreFileID = self.master.importFile(srcUrl)
-            with self.master.readFileStream(jobStoreFileID) as f:
-                fileMD5 = hashlib.md5(f.read()).hexdigest()
-            self.assertEqual(fileMD5, srcHash)
+            srcUrl, srcHash = ('https://raw.githubusercontent.com/BD2KGenomics/toil'
+                               '/12c711ff91caa5d8fb37a072bad376f447faa797/Makefile',
+                               '16a38fd7ae233616083d43d81159d4e8')
+            with self.master.readFileStream(self.master.importFile(srcUrl)) as f:
+                self.assertEqual(hashlib.md5(f.read()).hexdigest(), srcHash)
 
         def testFileDeletion(self):
             """