Repository: DataBiosphere/toil

Issue:
Retry SDB requests on ECONNRESET
https://github.com/BD2KGenomics/toil/pull/959#issuecomment-225464031

Retry SDB requests on ECONNRESET
https://github.com/BD2KGenomics/toil/pull/959#issuecomment-225464031



Discussion:

Resolving patch:
diff --git a/src/toil/jobStores/aws/jobStore.py b/src/toil/jobStores/aws/jobStore.py
--- a/src/toil/jobStores/aws/jobStore.py
+++ b/src/toil/jobStores/aws/jobStore.py
@@ -42,7 +42,7 @@
 from toil.jobStores.abstractJobStore import (AbstractJobStore, NoSuchJobException,
                                              ConcurrentFileModificationException,
                                              NoSuchFileException)
-from toil.jobStores.aws.utils import (SDBHelper, retry_sdb, no_such_domain, sdb_unavailable,
+from toil.jobStores.aws.utils import (SDBHelper, retry_sdb, no_such_sdb_domain, sdb_unavailable,
                                       monkeyPatchSdbConnection, retry_s3)
 from toil.jobWrapper import JobWrapper
 import toil.lib.encryption as encryption
@@ -517,7 +517,7 @@ def _getOrCreateDomain(self, domain_name):
         try:
             return self.db.get_domain(domain_name)
         except SDBResponseError as e:
-            if no_such_domain(e):
+            if no_such_sdb_domain(e):
                 for attempt in retry_sdb(predicate=sdb_unavailable):
                     with attempt:
                         return self.db.create_domain(domain_name)
diff --git a/src/toil/jobStores/aws/utils.py b/src/toil/jobStores/aws/utils.py
--- a/src/toil/jobStores/aws/utils.py
+++ b/src/toil/jobStores/aws/utils.py
@@ -185,21 +185,28 @@ def monkeyPatchSdbConnection(sdb):
 default_timeout = 300
 
 
-def sdb_unavailable(e):
-    return isinstance(e, BotoServerError) and e.status == 503
+def connection_reset(e):
+    # For some reason we get 'error: [Errno 104] Connection reset by peer' where the
+    # English description suggests that errno is 54 (ECONNRESET) while the actual
+    # errno is listed as 104. To be safe, we check for both:
+    return isinstance(e, socket.error) and e.errno in (errno.ECONNRESET, 104)
 
 
-def retryable_sdb_errors(e):
-    return sdb_unavailable(e) or no_such_domain(e)
+def sdb_unavailable(e):
+    return isinstance(e, BotoServerError) and e.status == 503
 
 
-def no_such_domain(e):
+def no_such_sdb_domain(e):
     return (isinstance(e, SDBResponseError)
             and e.error_code
             and e.error_code.endswith('NoSuchDomain'))
 
 
-def retry_sdb(delays=(0, 1, 1, 4, 16, 64), timeout=300, predicate=retryable_sdb_errors):
+def retryable_sdb_errors(e):
+    return sdb_unavailable(e) or no_such_sdb_domain(e) or connection_reset(e)
+
+
+def retry_sdb(delays=default_delays, timeout=default_timeout, predicate=retryable_sdb_errors):
     return retry(delays=delays, timeout=timeout, predicate=predicate)
 
 
@@ -207,10 +214,7 @@ def retryable_s3_errors(e):
     return (isinstance(e, (S3CreateError, S3ResponseError))
             and e.status == 409
             and 'try again' in e.message
-            # For some reason we get 'error: [Errno 104] Connection reset by peer' where the
-            # English description suggests that errno is 54 (ECONNRESET) while the actual
-            # errno is listed as 104. To be safe, we check for both:
-            or isinstance(e, socket.error) and e.errno in (errno.ECONNRESET, 104)
+            or connection_reset(e)
             or isinstance(e, BotoServerError) and e.status == 500
             or isinstance(e, S3CopyError) and 'try again' in e.message)
 


Test patch: