Repository: DataBiosphere/toil

Issue:
AWS job store name is not being validated
Regression from e60e396d0ce6523bc153fe3de2f5a8876e12785d

AWS job store name is not being validated
Regression from e60e396d0ce6523bc153fe3de2f5a8876e12785d



Discussion:

Resolving patch:
diff --git a/src/toil/common.py b/src/toil/common.py
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -136,7 +136,7 @@ def f(x):
 
         setOption("environment", parseSetEnv)
 
-        # Resource requirements
+        #Resource requirements
         setOption("defaultMemory", h2b, iC(1))
         setOption("defaultCores", h2b, iC(1))
         setOption("defaultDisk", h2b, iC(1))
@@ -387,8 +387,7 @@ def loadJobStore( jobStoreString, config=None ):
         return FileJobStore( jobStoreArgs, config=config )
     elif jobStoreName == 'aws':
         from toil.jobStores.aws.jobStore import AWSJobStore
-        region, namePrefix = jobStoreArgs.split( ':', 1 )
-        return AWSJobStore( region, namePrefix, config=config )
+        return AWSJobStore.createJobStore( jobStoreArgs, config=config )
     elif jobStoreName == 'azure':
         from toil.jobStores.azureJobStore import AzureJobStore
         account, namePrefix = jobStoreArgs.split( ':', 1 )
diff --git a/src/toil/jobStores/aws/jobStore.py b/src/toil/jobStores/aws/jobStore.py
--- a/src/toil/jobStores/aws/jobStore.py
+++ b/src/toil/jobStores/aws/jobStore.py
@@ -57,6 +57,37 @@ class AWSJobStore(AbstractJobStore):
     item representing the job. UUIDs are used to identify jobs and files.
     """
 
+    @classmethod
+    def createJobStore(cls, jobStoreString, config=None):
+        region, namePrefix = jobStoreString.split(':')
+        if not cls.bucketNameRe.match(namePrefix):
+            raise ValueError("Invalid name prefix '%s'. Name prefixes must contain only digits, "
+                             "hyphens or lower-case letters and must not start or end in a "
+                             "hyphen." % namePrefix)
+        # Reserve 13 for separator and suffix
+        if len(namePrefix) > cls.maxBucketNameLen - cls.maxNameLen - len(cls.nameSeparator):
+            raise ValueError("Invalid name prefix '%s'. Name prefixes may not be longer than 50 "
+                             "characters." % namePrefix)
+        if '--' in namePrefix:
+            raise ValueError("Invalid name prefix '%s'. Name prefixes may not contain "
+                             "%s." % (namePrefix, cls.nameSeparator))
+        return cls(region, namePrefix, config=config)
+
+    # Dots in bucket names should be avoided because bucket names are used in HTTPS bucket
+    # URLs where the may interfere with the certificate common name. We use a double
+    # underscore as a separator instead.
+    #
+    bucketNameRe = re.compile(r'^[a-z0-9][a-z0-9-]+[a-z0-9]$')
+
+    # See http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
+    #
+    minBucketNameLen = 3
+    maxBucketNameLen = 63
+    maxNameLen = 10
+    nameSeparator = '--'
+
+    # Do not invoke the constructor, use the factory method above.
+
     def __init__(self, region, namePrefix, config=None):
         """
         Create a new job store in AWS or load an existing one from there.
@@ -91,6 +122,7 @@ def __init__(self, region, namePrefix, config=None):
                 self._checkJobStoreCreation(create, exists, region + ":" + namePrefix)
 
         def qualify(name):
+            assert len(name) <= self.maxNameLen
             return self.namePrefix + self.nameSeparator + name
 
         self.jobsDomain = self._getOrCreateDomain(qualify('jobs'))
@@ -303,32 +335,6 @@ def getSharedPublicUrl(self, sharedFileName):
         assert self._validateSharedFileName(sharedFileName)
         return self.getPublicUrl(self._sharedFileID(sharedFileName))
 
-    # Dots in bucket names should be avoided because bucket names are used in HTTPS bucket
-    # URLs where the may interfere with the certificate common name. We use a double
-    # underscore as a separator instead.
-    bucketNameRe = re.compile(r'^[a-z0-9][a-z0-9-]+[a-z0-9]$')
-
-    nameSeparator = '--'
-
-    @classmethod
-    def _parseArgs(cls, jobStoreString):
-        region, namePrefix = jobStoreString.split(':')
-        # See http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html,
-        # reserve 10 characters for separator and suffixes
-        if not cls.bucketNameRe.match(namePrefix):
-            raise ValueError("Invalid name prefix '%s'. Name prefixes must contain only digits, "
-                             "hyphens or lower-case letters and must not start or end in a "
-                             "hyphen." % namePrefix)
-        # reserve 13 for separator and suffix
-        if len(namePrefix) > 50:
-            raise ValueError("Invalid name prefix '%s'. Name prefixes may not be longer than 50 "
-                             "characters." % namePrefix)
-        if '--' in namePrefix:
-            raise ValueError("Invalid name prefix '%s'. Name prefixes may not contain "
-                             "%s." % (namePrefix, cls.nameSeparator))
-
-        return region, namePrefix
-
     def _connectSimpleDB(self):
         """
         :rtype: SDBConnection
@@ -355,8 +361,8 @@ def _getOrCreateBucket(self, bucket_name, versioning=False):
         """
         :rtype: Bucket
         """
+        assert self.minBucketNameLen <= len(bucket_name) <= self.maxBucketNameLen
         assert self.bucketNameRe.match(bucket_name)
-        assert 3 <= len(bucket_name) <= 63
         try:
             bucket = self.s3.get_bucket(bucket_name, validate=True)
             assert versioning is self.__getBucketVersioning(bucket)


Test patch:
diff --git a/src/toil/test/jobStores/jobStoreTest.py b/src/toil/test/jobStores/jobStoreTest.py
--- a/src/toil/test/jobStores/jobStoreTest.py
+++ b/src/toil/test/jobStores/jobStoreTest.py
@@ -461,7 +461,7 @@ def _createJobStore(self, config=None):
         for encrypted in (True, False):
             self.assertTrue(AWSJobStore.FileInfo.maxInlinedSize(encrypted) < partSize)
         AWSJobStore.FileInfo.s3PartSize = partSize
-        return AWSJobStore(self.testRegion, self.namePrefix, config=config)
+        return AWSJobStore.createJobStore(self.testRegion + ':' + self.namePrefix, config=config)
 
     def testInlinedFiles(self):
         from toil.jobStores.aws.jobStore import AWSJobStore
@@ -495,7 +495,7 @@ def testLargeFile(self):
                 buf = f.read(self._partSize())
                 if not buf: break
                 hashOut.update(buf)
-        self.assertEqual(hashIn.digest(),hashOut.digest())
+        self.assertEqual(hashIn.digest(), hashOut.digest())
 
     def _largeLogEntrySize(self):
         from toil.jobStores.aws.jobStore import AWSJobStore
@@ -507,6 +507,21 @@ def _batchDeletionSize(self):
         return AWSJobStore.itemsPerBatchDelete
 
 
+@needs_aws
+class InvalidAWSJobStoreTest(ToilTest):
+    def testInvalidJobStoreName(self):
+        from toil.jobStores.aws.jobStore import AWSJobStore
+        self.assertRaises(ValueError,
+                          AWSJobStore.createJobStore,
+                          'us-west-2:a--b')
+        self.assertRaises(ValueError,
+                          AWSJobStore.createJobStore,
+                          'us-west-2:' + ('a' * 100))
+        self.assertRaises(ValueError,
+                          AWSJobStore.createJobStore,
+                          'us-west-2:a_b')
+
+
 @needs_azure
 class AzureJobStoreTest(hidden.AbstractJobStoreTest):
     def _createJobStore(self, config=None):