Repository: DataBiosphere/toil

Issue:
Retry on Azure importFile timeout

Resource download isn't retried
I just got a burst of failed jobs with:

```
673c500c-95a9-42cd-a381-413901fb7d82    Traceback (most recent call last):
673c500c-95a9-42cd-a381-413901fb7d82      File "/usr/local/lib/python2.7/dist-packages/toil/worker.py", line 325, in main
673c500c-95a9-42cd-a381-413901fb7d82        job = Job._loadJob(jobGraph.command, jobStore)
673c500c-95a9-42cd-a381-413901fb7d82      File "/usr/local/lib/python2.7/dist-packages/toil/job.py", line 865, in _loadJob
673c500c-95a9-42cd-a381-413901fb7d82        userModule = cls._loadUserModule(userModule)
673c500c-95a9-42cd-a381-413901fb7d82      File "/usr/local/lib/python2.7/dist-packages/toil/job.py", line 842, in _loadUserModule
673c500c-95a9-42cd-a381-413901fb7d82        return userModule.load()
673c500c-95a9-42cd-a381-413901fb7d82      File "/usr/local/lib/python2.7/dist-packages/toil/resource.py", line 568, in load
673c500c-95a9-42cd-a381-413901fb7d82        module = self.makeLoadable()
673c500c-95a9-42cd-a381-413901fb7d82      File "/usr/local/lib/python2.7/dist-packages/toil/resource.py", line 562, in makeLoadable
673c500c-95a9-42cd-a381-413901fb7d82        module = self if self.belongsToToil else self.localize()
673c500c-95a9-42cd-a381-413901fb7d82      File "/usr/local/lib/python2.7/dist-packages/toil/resource.py", line 478, in localize
673c500c-95a9-42cd-a381-413901fb7d82        resource.download(callback=stash)
673c500c-95a9-42cd-a381-413901fb7d82      File "/usr/local/lib/python2.7/dist-packages/toil/resource.py", line 164, in download
673c500c-95a9-42cd-a381-413901fb7d82        self._save(tempDirPath)
673c500c-95a9-42cd-a381-413901fb7d82      File "/usr/local/lib/python2.7/dist-packages/toil/resource.py", line 283, in _save
673c500c-95a9-42cd-a381-413901fb7d82        self._download(bytesIO)
673c500c-95a9-42cd-a381-413901fb7d82      File "/usr/local/lib/python2.7/dist-packages/toil/resource.py", line 236, in _download
673c500c-95a9-42cd-a381-413901fb7d82        with closing(urlopen(self.url)) as content:
673c500c-95a9-42cd-a381-413901fb7d82      File "/usr/lib/python2.7/urllib2.py", line 127, in urlopen
673c500c-95a9-42cd-a381-413901fb7d82        return _opener.open(url, data, timeout)
673c500c-95a9-42cd-a381-413901fb7d82      File "/usr/lib/python2.7/urllib2.py", line 410, in open
673c500c-95a9-42cd-a381-413901fb7d82        response = meth(req, response)
673c500c-95a9-42cd-a381-413901fb7d82      File "/usr/lib/python2.7/urllib2.py", line 523, in http_response
673c500c-95a9-42cd-a381-413901fb7d82        'http', request, response, code, msg, hdrs)
673c500c-95a9-42cd-a381-413901fb7d82      File "/usr/lib/python2.7/urllib2.py", line 448, in error
673c500c-95a9-42cd-a381-413901fb7d82        return self._call_chain(*args)
673c500c-95a9-42cd-a381-413901fb7d82      File "/usr/lib/python2.7/urllib2.py", line 382, in _call_chain
673c500c-95a9-42cd-a381-413901fb7d82        result = func(*args)
673c500c-95a9-42cd-a381-413901fb7d82      File "/usr/lib/python2.7/urllib2.py", line 531, in http_error_default
673c500c-95a9-42cd-a381-413901fb7d82        raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
673c500c-95a9-42cd-a381-413901fb7d82    HTTPError: HTTP Error 400: Bad Request
```

I'm not sure why a 400 was raised here, but the request should probably be retried with exponential backoff. If I had to guess, this might happen in rare cases when the user script resource is being refreshed at the exact same time--although that would be more likely to raise a 404 than a 400 I would think.


Discussion:

Resolving patch:
diff --git a/src/toil/jobStores/azureJobStore.py b/src/toil/jobStores/azureJobStore.py
--- a/src/toil/jobStores/azureJobStore.py
+++ b/src/toil/jobStores/azureJobStore.py
@@ -278,9 +278,11 @@ def getSize(cls, url):
     @classmethod
     def _readFromUrl(cls, url, writable):
         blob = cls._parseWasbUrl(url)
-        blob.service.get_blob_to_file(container_name=blob.container,
-                                      blob_name=blob.name,
-                                      stream=writable)
+        for attempt in retry_azure():
+            with attempt:
+                blob.service.get_blob_to_file(container_name=blob.container,
+                                              blob_name=blob.name,
+                                              stream=writable)
 
     @classmethod
     def _writeToUrl(cls, readable, url):
diff --git a/src/toil/provisioners/aws/awsProvisioner.py b/src/toil/provisioners/aws/awsProvisioner.py
--- a/src/toil/provisioners/aws/awsProvisioner.py
+++ b/src/toil/provisioners/aws/awsProvisioner.py
@@ -82,6 +82,8 @@ def retryPredicate(e):
 
     @staticmethod
     def throttlePredicate(e):
+        if not isinstance(e, BotoServerError):
+            return False
         # boto/AWS gives multiple messages for the same error...
         if e.status == 503 and 'Request limit exceeded' in e.body:
             return True
diff --git a/src/toil/resource.py b/src/toil/resource.py
--- a/src/toil/resource.py
+++ b/src/toil/resource.py
@@ -27,9 +27,11 @@
 from io import BytesIO
 from pydoc import locate
 from tempfile import mkdtemp
+from urllib2 import HTTPError
 from zipfile import ZipFile, PyZipFile
 
 # Python 3 compatibility imports
+from bd2k.util.retry import retry
 from six.moves.urllib.request import urlopen
 
 from bd2k.util import strict_bool
@@ -233,8 +235,10 @@ def _download(self, dstFile):
 
         :type dstFile: io.BytesIO|io.FileIO
         """
-        with closing(urlopen(self.url)) as content:
-            buf = content.read()
+        for attempt in retry(predicate=lambda e: isinstance(e, HTTPError) and e.code == 400):
+            with attempt:
+                with closing(urlopen(self.url)) as content:
+                    buf = content.read()
         contentHash = hashlib.md5(buf)
         assert contentHash.hexdigest() == self.contentHash
         dstFile.write(buf)


Test patch: