Repository: DataBiosphere/toil

Issue:
Delete sdb domains if unable to create job store bucket
refer [This log](http://jenkins.cgcloud.info/job/toil-pull-requests/1902/consoleFull). Take for example, the test with uuid `6cdc03f3-e1c1-4fab-97ea-19cc74fb1ea9`. The logs say

```
INFO:toil.jobStores.aws.jobStore:Binding to job store domain 'toil-registry'.
INFO:toil.jobStores.aws.jobStore:Binding to job store domain 'toil-registry'.
INFO:toil.jobStores.aws.jobStore:Binding to job store domain 'jobstore-test-6cdc03f3-e1c1-4fab-97ea-19cc74fb1ea9--jobs'.
INFO:toil.jobStores.aws.jobStore:Binding to job store domain 'jobstore-test-6cdc03f3-e1c1-4fab-97ea-19cc74fb1ea9--files'.
INFO:toil.jobStores.aws.jobStore:Binding to job store bucket 'jobstore-test-6cdc03f3-e1c1-4fab-97ea-19cc74fb1ea9--files'.
DEBUG:toil.jobStores.aws.jobStore:Bucket 'jobstore-test-6cdc03f3-e1c1-4fab-97ea-19cc74fb1ea9--files' does not exist.
DEBUG:toil.jobStores.aws.jobStore:Creating bucket 'jobstore-test-6cdc03f3-e1c1-4fab-97ea-19cc74fb1ea9--files'.
 AWSJobStoreTest.test_testImportExportFile__otherCls_awsjobstoretest__size_partsize 
src/toil/test/jobStores/jobStoreTest.py:104: in setUp
    self.master.initialize(self.config)
src/toil/jobStores/aws/jobStore.py:204: in initialize
    self._bind(create=True)
src/toil/jobStores/aws/jobStore.py:232: in _bind
    versioning=True)
src/toil/jobStores/aws/jobStore.py:674: in _bindBucket
    bucket = self.s3.create_bucket(bucket_name, location=location)
...
...
E           S3ResponseError: S3ResponseError: 400 Bad Request
E           <?xml version="1.0" encoding="UTF-8"?>
E           <Error><Code>TooManyBuckets</Code><Message>You have attempted to create more buckets than allowed</Message><CurrentNumberOfBuckets>101</CurrentNumberOfBuckets><AllowedNumberOfBuckets>101</AllowedNumberOfBuckets><RequestId>CA9B35247D82A7B9</RequestId><HostId>atOhiIDkPFkyEOdnBGfPkEzJbswJpWZpz1hLC4ofc7eSF6jQX0Zg/SyMNYhIfeaXlOaV1RAmWJ0=</HostId></Error>
```

We've apparently reached the limit on number of buckets, so Toil fails..... without deleting the bound jobstore domains -- as evident by

```
>>> [z for z in x.get_all_domains(max_domains=100) if '6cdc03f3-e1c1-4fab-97ea-19cc74fb1ea9' in z.name]
[Domain:jobstore-test-6cdc03f3-e1c1-4fab-97ea-19cc74fb1ea9--files,
 Domain:jobstore-test-6cdc03f3-e1c1-4fab-97ea-19cc74fb1ea9--jobs]
```



Discussion:
Good one. This should do it:

``` diff
--- a/src/toil/jobStores/aws/jobStore.py
+++ b/src/toil/jobStores/aws/jobStore.py
@@ -201,10 +201,16 @@ class AWSJobStore(AbstractJobStore):
         if self._registered:
             raise JobStoreExistsException(self.locator)
         self._registered = None
-        self._bind(create=True)
-        super(AWSJobStore, self).initialize(config)
-        # Only register after job store has been full initialized
-        self._registered = True
+        try:
+            self._bind(create=True)
+        except:
+            with panic(log):
+                self.destroy()
+            raise
+        else:
+            super(AWSJobStore, self).initialize(config)
+            # Only register after job store has been full initialized
+            self._registered = True

     @property
     def sseKeyPath(self):
```

Just need a good way to test it.

I made a slightly different PR that effectively handles all job stores and I added an AWS test that passes ;)


Resolving patch:
diff --git a/src/toil/jobStores/abstractJobStore.py b/src/toil/jobStores/abstractJobStore.py
--- a/src/toil/jobStores/abstractJobStore.py
+++ b/src/toil/jobStores/abstractJobStore.py
@@ -100,6 +100,13 @@ def __init__(self, locator):
             "the job store with 'toil clean' to start the workflow from scratch" % locator)
 
 
+class BucketLocationConflictException(Exception):
+    def __init__(self, bucketRegion):
+        message = ('A bucket with the same name as the jobstore was found in another region (%s). '
+                   'Cannot proceed as the unique bucket name is already in use.' % bucketRegion)
+        super(BucketLocationConflictException, self).__init__(message)
+
+
 class AbstractJobStore(object):
     """ 
     Represents the physical storage for the jobs and files in a Toil workflow.
diff --git a/src/toil/jobStores/aws/jobStore.py b/src/toil/jobStores/aws/jobStore.py
--- a/src/toil/jobStores/aws/jobStore.py
+++ b/src/toil/jobStores/aws/jobStore.py
@@ -48,6 +48,7 @@
                                              ConcurrentFileModificationException,
                                              NoSuchFileException,
                                              NoSuchJobStoreException,
+                                             BucketLocationConflictException,
                                              JobStoreExistsException)
 from toil.jobStores.aws.utils import (SDBHelper,
                                       retry_sdb,
@@ -201,10 +202,15 @@ def initialize(self, config):
         if self._registered:
             raise JobStoreExistsException(self.locator)
         self._registered = None
-        self._bind(create=True)
-        super(AWSJobStore, self).initialize(config)
-        # Only register after job store has been full initialized
-        self._registered = True
+        try:
+            self._bind(create=True)
+        except:
+            with panic(log):
+                self.destroy()
+        else:
+            super(AWSJobStore, self).initialize(config)
+            # Only register after job store has been full initialized
+            self._registered = True
 
     @property
     def sseKeyPath(self):
@@ -221,6 +227,9 @@ def qualify(name):
             assert len(name) <= self.maxNameLen
             return self.namePrefix + self.nameSeparator + name
 
+        # The order in which this sequence of events happens is important.  We can easily handle the
+        # inability to bind a domain, but it is a little harder to handle some cases of binding the
+        # jobstore bucket.  Maintaining this order allows for an easier `destroy` method.
         if self.jobsDomain is None:
             self.jobsDomain = self._bindDomain(qualify('jobs'), create=create, block=block)
         if self.filesDomain is None:
@@ -681,10 +690,17 @@ def bucket_creation_pending(e):
                             raise
                         else:
                             return None
+                    elif e.status == 301:
+                        # This is raised if the user attempts to get a bucket in a region outside
+                        # the specified one, if the specified one is not `us-east-1`.  The us-east-1
+                        # server allows a user to use buckets from any region.
+                        bucket = self.s3.get_bucket(bucket_name, validate=False)
+                        raise BucketLocationConflictException(self.__getBucketRegion(bucket))
                     else:
                         raise
                 else:
-                    assert self.__getBucketRegion(bucket) == self.region
+                    if self.__getBucketRegion(bucket) != self.region:
+                        raise BucketLocationConflictException(self.__getBucketRegion(bucket))
                     assert versioning is self.__getBucketVersioning(bucket)
                     log.debug("Using existing job store bucket '%s'.", bucket_name)
                     return bucket
@@ -1255,7 +1271,14 @@ def __getBucketRegion(self, bucket):
     def destroy(self):
         # FIXME: Destruction of encrypted stores only works after initialize() or .resume()
         # See https://github.com/BD2KGenomics/toil/issues/1041
-        self._bind(create=False, block=False)
+        try:
+            self._bind(create=False, block=False)
+        except BucketLocationConflictException:
+            # If the unique jobstore bucket name existed, _bind would have raised a
+            # BucketLocationConflictException before calling destroy.  Calling _bind here again
+            # would reraise the same exception so we need to catch and ignore that exception.
+            pass
+        # TODO: Add other failure cases to be ignored here.
         self._registered = None
         if self.filesBucket is not None:
             self._delete_bucket(self.filesBucket)


Test patch:
diff --git a/src/toil/test/jobStores/jobStoreTest.py b/src/toil/test/jobStores/jobStoreTest.py
--- a/src/toil/test/jobStores/jobStoreTest.py
+++ b/src/toil/test/jobStores/jobStoreTest.py
@@ -36,10 +36,12 @@
 # (installed by `make prepare`)
 from mock import patch
 
-from toil.common import Config
+from toil.common import Config, Toil
+from toil.job import Job
 from toil.jobStores.abstractJobStore import (AbstractJobStore,
                                              NoSuchJobException,
-                                             NoSuchFileException)
+                                             NoSuchFileException,
+                                             BucketLocationConflictException)
 from toil.jobStores.aws.utils import region_to_bucket_location
 from toil.jobStores.fileJobStore import FileJobStore
 from toil.test import (ToilTest,
@@ -839,6 +841,49 @@ def _corruptJobStore(self):
         assert isinstance(self.master, AWSJobStore)  # type hinting
         self.master.filesBucket.delete()
 
+    def testSDBDomainsDeletedOnFailedJobstoreBucketCreation(self):
+        """
+        This test ensures that SDB domains bound to a jobstore are deleted if the jobstore bucket
+        failed to be created.  We simulate a failed jobstore bucket creation by using a bucket in a
+        different region with the same name.
+        """
+        from boto.sdb import connect_to_region
+        from boto.s3.connection import Location, S3Connection
+        externalAWSLocation = Location.USWest
+        for testRegion in 'us-east-1', 'us-west-2':
+            # We run this test twice, once with the default s3 server us-east-1 as the test region
+            # and once with another server (us-west-2).  The external server is always us-west-1.
+            # This incidentally tests that the BucketLocationConflictException is thrown when using
+            # both the default, and a non-default server.
+            testJobStoreUUID = str(uuid.uuid4())
+            # Create the nucket at the external region
+            s3 = S3Connection()
+            bucket = s3.create_bucket('domain-test-' + testJobStoreUUID + '--files',
+                                      location=externalAWSLocation)
+            options = Job.Runner.getDefaultOptions('aws:' + testRegion + ':domain-test-' +
+                                                   testJobStoreUUID)
+            options.logLevel = 'DEBUG'
+            try:
+                with Toil(options) as toil:
+                    pass
+            except BucketLocationConflictException:
+                # Catch the expected BucketLocationConflictException and ensure that the bound
+                # domains don't exist in SDB.
+                sdb = connect_to_region(self.awsRegion())
+                next_token = None
+                allDomainNames = []
+                while True:
+                    domains = sdb.get_all_domains(max_domains=100, next_token=next_token)
+                    allDomainNames.extend([x.name for x in domains])
+                    next_token = domains.next_token
+                    if next_token is None:
+                        break
+                self.assertFalse([d for d in allDomainNames if testJobStoreUUID in d])
+            else:
+                self.fail()
+            finally:
+                s3.delete_bucket(bucket=bucket)
+
     def testInlinedFiles(self):
         from toil.jobStores.aws.jobStore import AWSJobStore
         master = self.master