Repository: DataBiosphere/toil

Issue:
Toil 3.19 release fails CWLTest.test_run_s3 trying to get_frozen_credentials on None
See the test log here:

https://ucsc-ci.com/databiosphere/toil/-/jobs/6397

```
=================================== FAILURES ===================================
_____________________________ CWLTest.test_run_s3 ______________________________
Traceback (most recent call last):
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/src/toil/test/cwl/cwlTest.py", line 111, in test_run_s3
    self.download('download_s3.json', self._tester)
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/src/toil/test/cwl/cwlTest.py", line 99, in download
    self._expected_download_output(self.outDir))
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/src/toil/test/cwl/cwlTest.py", line 71, in _tester
    cwltoil.main(main_args, stdout=st)
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/src/toil/cwl/cwltoil.py", line 1254, in main
    import_files(initialized_job_order)
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/src/toil/cwl/cwltoil.py", line 1232, in import_files
    skip_broken=True))
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/venv/local/lib/python2.7/site-packages/cwltool/pathmapper.py", line 47, in adjustFileObjs
    visit_class(rec, ("File",), op)
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/venv/local/lib/python2.7/site-packages/cwltool/utils.py", line 210, in visit_class
    visit_class(rec[d], cls, op)
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/venv/local/lib/python2.7/site-packages/cwltool/utils.py", line 208, in visit_class
    op(rec)
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/src/toil/cwl/cwltoil.py", line 365, in uploadFile
    uploadfunc, fileindex, existing, uf["location"])
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/src/toil/cwl/cwltoil.py", line 337, in write_file
    index[x] = "toilfs:" + writeFunc(rp)
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/src/toil/common.py", line 953, in importFile
    return self._jobStore.importFile(srcUrl, sharedFileName=sharedFileName)
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/src/toil/jobStores/abstractJobStore.py", line 299, in importFile
    return self._importFile(otherCls, srcUrl, sharedFileName=sharedFileName, hardlink=hardlink)
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/src/toil/jobStores/fileJobStore.py", line 272, in _importFile
    sharedFileName=sharedFileName)
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/src/toil/jobStores/abstractJobStore.py", line 320, in _importFile
    otherCls._readFromUrl(url, writable)
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/src/toil/jobStores/aws/jobStore.py", line 435, in _readFromUrl
    srcKey = cls._getKeyForUrl(url, existing=True)
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/src/toil/jobStores/aws/jobStore.py", line 476, in _getKeyForUrl
    with closing(boto.connect_s3()) as s3:
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/venv/local/lib/python2.7/site-packages/boto/__init__.py", line 141, in connect_s3
    return S3Connection(aws_access_key_id, aws_secret_access_key, **kwargs)
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/venv/local/lib/python2.7/site-packages/boto/s3/connection.py", line 194, in __init__
    validate_certs=validate_certs, profile_name=profile_name)
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/venv/local/lib/python2.7/site-packages/boto/connection.py", line 555, in __init__
    profile_name)
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/src/toil/__init__.py", line 491, in __init__
    profile_name=profile_name, **kwargs)
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/venv/local/lib/python2.7/site-packages/boto/provider.py", line 201, in __init__
    self.get_credentials(access_key, secret_key, security_token, profile_name)
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/src/toil/__init__.py", line 503, in get_credentials
    self._obtain_credentials_from_cache_or_boto3()
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/src/toil/__init__.py", line 620, in _obtain_credentials_from_cache_or_boto3
    self._obtain_credentials_from_boto3()
  File "/home/gitlab-runner/builds/WgJTr25x/0/databiosphere/toil/src/toil/__init__.py", line 545, in _obtain_credentials_from_boto3
    creds.get_frozen_credentials()
AttributeError: 'NoneType' object has no attribute 'get_frozen_credentials'
```

This clearly has something to do with my AWS credentials hacking.


Discussion:

Resolving patch:
diff --git a/src/toil/__init__.py b/src/toil/__init__.py
--- a/src/toil/__init__.py
+++ b/src/toil/__init__.py
@@ -25,6 +25,7 @@
 from docker.errors import ImageNotFound
 from toil.lib.memoize import memoize
 from toil.lib.misc import mkdir_p
+from toil.lib.retry import retry
 from toil.version import currentCommit
 
 # subprocess32 is a backport of python3's subprocess module for use on Python2,
@@ -538,8 +539,18 @@ def _obtain_credentials_from_boto3(self):
             
             # We get a Credentials object
             # <https://github.com/boto/botocore/blob/8d3ea0e61473fba43774eb3c74e1b22995ee7370/botocore/credentials.py#L227>
-            # or a RefreshableCredentials
-            creds = self._boto3_resolver.load_credentials()
+            # or a RefreshableCredentials, or None on failure.
+            creds = None
+            for attempt in retry(timeout=10, predicate=lambda _: True):
+                with attempt:
+                    creds = self._boto3_resolver.load_credentials()
+                    
+                    if creds is None:
+                        try:
+                            resolvers = str(self._boto3_resolver.providers)
+                        except:
+                            resolvers = "(Resolvers unavailable)"
+                        raise RuntimeError("Could not obtain AWS credentials from Boto3. Resolvers tried: " + resolvers)
             
             # Make sure the credentials actually has some credentials if it is lazy
             creds.get_frozen_credentials()
diff --git a/src/toil/jobStores/aws/jobStore.py b/src/toil/jobStores/aws/jobStore.py
--- a/src/toil/jobStores/aws/jobStore.py
+++ b/src/toil/jobStores/aws/jobStore.py
@@ -489,7 +489,7 @@ def _getKeyForUrl(url, existing=None):
             keyName = url.path[1:]
             bucketName = url.netloc
             bucket = s3.get_bucket(bucketName)
-            key = bucket.get_key(bytes(keyName))
+            key = bucket.get_key(keyName.encode('utf-8'))
             if existing is True:
                 if key is None:
                     raise RuntimeError("Key '%s' does not exist in bucket '%s'." %


Test patch: