Repository: DataBiosphere/toil

Issue:
Azure job store should retry operations
From http://jenkins.cgcloud.info/job/toil/367/testReport/junit/src.toil.test.jobStores.jobStoreTest/AzureJobStoreTest/testFileDeletion/

```
Error Message

timeout: timed out
Stacktrace

self = <toil.test.jobStores.jobStoreTest.AzureJobStoreTest testMethod=testFileDeletion>

    def testFileDeletion(self):
        """
                Intended to cover the batch deletion of items in the AWSJobStore, but it doesn't hurt running it on the
                other job stores.
                """
        job = self.master.create("1", 2, 3, 4, 0)
        file_list = []
        # FIXME: tie 30 to batch limit in AWSJobStore
        for file in xrange(0, 30):
            file_list.append(self.master.getEmptyFileStoreID(job.jobStoreID))
>       self.master.delete(job.jobStoreID)

src/toil/test/jobStores/jobStoreTest.py:271: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
src/toil/jobStores/azureJobStore.py:147: in delete
    self.deleteFile(jobStoreFileID)
src/toil/jobStores/azureJobStore.py:179: in deleteFile
    self._dissociateJobWithFile(jobStoreFileID)
src/toil/jobStores/azureJobStore.py:293: in _dissociateJobWithFile
    self.jobFileIDs.delete_entity(partition_key=jobStoreID, row_key=jobStoreFileID)
src/toil/jobStores/azureJobStore.py:445: in callable
    return function(**kwargs)
venv/local/lib/python2.7/site-packages/azure/storage/tableservice.py:562: in delete_entity
    self._perform_request(request)
venv/local/lib/python2.7/site-packages/azure/storage/storageclient.py:172: in _perform_request
    resp = self._filter(request)
venv/local/lib/python2.7/site-packages/azure/storage/tableservice.py:644: in _perform_request_worker
    return self._httpclient.perform_request(request)
venv/local/lib/python2.7/site-packages/azure/http/httpclient.py:218: in perform_request
    self.send_request_headers(connection, request.headers)
venv/local/lib/python2.7/site-packages/azure/http/httpclient.py:197: in send_request_headers
    connection.endheaders()
/usr/lib/python2.7/httplib.py:975: in endheaders
    self._send_output(message_body)
/usr/lib/python2.7/httplib.py:835: in _send_output
    self.send(msg)
/usr/lib/python2.7/httplib.py:797: in send
    self.connect()
/usr/lib/python2.7/httplib.py:1178: in connect
    self.timeout, self.source_address)
```



Discussion:
Oops. I can easily add a retry to my API shims, but it'd have to wait till after tomorrow.

This by no means urgent but we should talk about it during the planning meeting tomorrow. I don't even know if it is possible to safely detect the situations in which a retry should be performed. Amazon makes that very clear with the HTTP status code.

I think some exception is raised, though it may just be a generic socket timeout exception and not anything specific to Azure.

What time is the planning meeting?

3pm

I sent you an invitation.

On Tue, Sep 1, 2015 at 2:35 PM, Joel Armstrong notifications@github.com
wrote:

> I think some exception is raised, though it may just be a generic socket
> timeout exception and not anything specific to Azure.
> 
> What time is the planning meeting?
> 
> —
> Reply to this email directly or view it on GitHub
> https://github.com/BD2KGenomics/toil/issues/341#issuecomment-136868077.
> 
> ## 
> 
> You received this message because you are subscribed to the Google Groups
> "cgloud-jenkins" group.
> To unsubscribe from this group and stop receiving emails from it, send an
> email to cgloud-jenkins+unsubscribe@googlegroups.com.
> To post to this group, send email to cgloud-jenkins@googlegroups.com.
> To view this discussion on the web visit
> https://groups.google.com/d/msgid/cgloud-jenkins/BD2KGenomics/toil/issues/341/136868077%40github.com
> https://groups.google.com/d/msgid/cgloud-jenkins/BD2KGenomics/toil/issues/341/136868077%40github.com?utm_medium=email&utm_source=footer
> .
> 
> For more options, visit https://groups.google.com/d/optout.

## 

Hannes Schmidt
Software Engineer
Center for Biomolecular Science and Engineering
University of California, Santa Cruz

(206) 696-2316 (cell)
hannes@ucsc.edu


Resolving patch:
diff --git a/src/toil/jobStores/azureJobStore.py b/src/toil/jobStores/azureJobStore.py
--- a/src/toil/jobStores/azureJobStore.py
+++ b/src/toil/jobStores/azureJobStore.py
@@ -27,6 +27,8 @@
 import bz2
 import cPickle
 import base64
+import socket
+import httplib
 from datetime import datetime, timedelta
 from ConfigParser import RawConfigParser, NoOptionError
 
@@ -294,11 +296,15 @@ def _dissociateJobWithFile(self, jobStoreFileID):
 
     def _getOrCreateTable(self, tableName):
         # This will not fail if the table already exists.
-        self.tableService.create_table(tableName)
+        for attempt in retry_on_error():
+            with attempt:
+                self.tableService.create_table(tableName)
         return AzureTable(self.tableService, tableName)
 
     def _getOrCreateBlobContainer(self, containerName):
-        self.blobService.create_container(containerName)
+        for attempt in retry_on_error():
+            with attempt:
+                self.blobService.create_container(containerName)
         return AzureBlobContainer(self.blobService, containerName)
 
     def _sanitizeTableName(self, tableName):
@@ -442,7 +448,10 @@ def callable(*args, **kwargs):
             if 'entity' in kwargs:
                 if 'PartitionKey' not in kwargs['entity']:
                     kwargs['entity']['PartitionKey'] = self.defaultPartition
-            return function(**kwargs)
+
+            for attempt in retry_on_error():
+                with attempt:
+                    return function(**kwargs)
 
         return callable
 
@@ -470,7 +479,10 @@ def callable(*args, **kwargs):
             assert len(args) == 0
             function = getattr(self.blobService, name)
             kwargs['container_name'] = self.containerName
-            return function(**kwargs)
+
+            for attempt in retry_on_error():
+                with attempt:
+                    return function(**kwargs)
 
         return callable
 
@@ -523,3 +535,67 @@ def toItem(self, chunkSize=65535):
         for attributeOrder, chunk in enumerate(jobChunks):
             item['_' + str(attributeOrder).zfill(3)] = chunk
         return item
+
+def retry_on_error(num_tries=5, retriable_exceptions=(socket.error, socket.gaierror,
+                                                      httplib.HTTPException)):
+    """
+    Retries on a set of allowable exceptions, mimicking boto's behavior by default.
+
+    :param num_tries: number of times to try before giving up.
+
+    :return: a generator yielding contextmanagers
+
+    Retry the correct number of times and then give up and reraise
+    >>> i = 0
+    >>> for attempt in retry_on_error(retriable_exceptions=(RuntimeError)):
+    ...     with attempt:
+    ...         i += 1
+    ...         raise RuntimeError("foo")
+    Traceback (most recent call last):
+    ...
+    RuntimeError: foo
+    >>> i
+    5
+
+    Give up and reraise on any unexpected exceptions
+    >>> i = 0
+    >>> for attempt in retry_on_error(num_tries=5, retriable_exceptions=()):
+    ...     with attempt:
+    ...         i += 1
+    ...         raise RuntimeError("foo")
+    Traceback (most recent call last):
+    ...
+    RuntimeError: foo
+    >>> i
+    1
+
+    Do things only once if they succeed!
+    >>> i = 0
+    >>> for attempt in retry_on_error():
+    ...     with attempt:
+    ...         i += 1
+    >>> i
+    1
+    """
+    go = [None]
+    @contextmanager
+    def attempt(last=False):
+        try:
+            yield
+        except retriable_exceptions as e:
+            if last:
+                raise
+            else:
+                log.info("Got a retriable exception %s, trying again" % e.__class__.__name__)
+        else:
+            go.pop()
+
+    while go:
+        if num_tries == 1:
+            yield attempt(last=True)
+        else:
+            yield attempt()
+        # It's safe to do this, even with Python's weird default
+        # arguments behavior, since we are assigning to num_tries
+        # rather than mutating it.
+        num_tries -= 1


Test patch: