Repository: databiosphere/toil
Issue: Public URL to hot-deployed resources in S3 expires when IAM role creds expire

If the leader is using temporary IAM role credentials, there are two expirations at play, they explicit one specified when the public URL is generated, and an implicit that happens when the IAM role credentials expire, usually after six hours. If workers are added or restarted after those six hours, they will likely get the following exception:

```
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:    ---TOIL WORKER OUTPUT LOG---
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:    Traceback (most recent call last):
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:      File "/usr/local/lib/python2.7/dist-packages/toil/worker.py", line 266, in main
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:        job = Job._loadJob(jobWrapper.command, jobStore)
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:      File "/usr/local/lib/python2.7/dist-packages/toil/job.py", line 937, in _loadJob
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:        userModule = cls._loadUserModule(userModule)
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:      File "/usr/local/lib/python2.7/dist-packages/toil/job.py", line 908, in _loadUserModule
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:        userModule = userModule.localize()
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:      File "/usr/local/lib/python2.7/dist-packages/toil/resource.py", line 416, in localize
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:        resource.download(callback=stash)
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:      File "/usr/local/lib/python2.7/dist-packages/toil/resource.py", line 152, in download
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:        self._save(tempDirPath)
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:      File "/usr/local/lib/python2.7/dist-packages/toil/resource.py", line 269, in _save
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:        self._download(bytesIO)
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:      File "/usr/local/lib/python2.7/dist-packages/toil/resource.py", line 223, in _download
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:        with closing(urlopen(self.url)) as content:
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:      File "/usr/lib/python2.7/urllib2.py", line 154, in urlopen
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:        return opener.open(url, data, timeout)
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:      File "/usr/lib/python2.7/urllib2.py", line 437, in open
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:        response = meth(req, response)
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:      File "/usr/lib/python2.7/urllib2.py", line 550, in http_response
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:        'http', request, response, code, msg, hdrs)
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:      File "/usr/lib/python2.7/urllib2.py", line 475, in error
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:        return self._call_chain(*args)
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:      File "/usr/lib/python2.7/urllib2.py", line 409, in _call_chain
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:        result = func(*args)
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:      File "/usr/lib/python2.7/urllib2.py", line 558, in http_error_default
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:        raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:    HTTPError: HTTP Error 400: Bad Request
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:    Exiting the worker because of a failed jobWrapper on host ip-172-31-45-12
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:    ERROR:toil.worker:Exiting the worker because of a failed jobWrapper on host ip-172-31-45-12
WARNING:toil.leader:e8f55793-15ca-471a-947f-38936f917bba:    WARNING:toil.jobWrapper:Due to failure we are reducing the remaining retry count of job e8f55793-15ca-471a-947f-38936f917bba to 2
```

Existing nodes will not show this behavior because the resource was downloaded when the node first started. To be precise, the resources are downloaded once per Mesos executor process. If the executor fails after six hours and is subsequently restarted, this will also happen.

Changed files:
src/toil/common.py, src/toil/resource.py

diff --git a/src/toil/common.py b/src/toil/common.py
index c6de763c..7904463b 100644
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -22,6 +22,7 @@ import sys
 import tempfile
 import time
 from argparse import ArgumentParser
+from threading import Thread
 
 from bd2k.util.exceptions import require
 from bd2k.util.humanize import bytes2human
@@ -832,9 +833,36 @@ class Toil(object):
         if userScript is None:
             logger.info('No user script to hot-deploy.')
         else:
-            logger.info('Saving user script %s as a resource', userScript)
+            logger.debug('Saving user script %s as a resource', userScript)
             userScriptResource = userScript.saveAsResourceTo(self._jobStore)
-            logger.info('Hot-deploying user script resource %s.', userScriptResource)
+            logger.debug('Injecting user script %s into batch system.', userScriptResource)
+            self._batchSystem.setUserScript(userScriptResource)
+            thread = Thread(target=self._refreshUserScript,
+                            name='refreshUserScript',
+                            kwargs=dict(userScriptResource=userScriptResource))
+            thread.daemon = True
+            thread.start()
+
+    def _refreshUserScript(self, userScriptResource):
+        """
+        Periodically refresh the user script in the job store to prevent credential
+        expiration from causing the public URL to the user script to expire.
+        """
+        while True:
+            # Boto refreshes IAM credentials if they will be expiring within the next five
+            # minutes, but it will only check the expiry if and when credentials are needed to
+            # sign an actual AWS request. This means that we should be refreshing the user script
+            # at least every 5 minutes. Note that refreshing the user script in the job store
+            # involves an S3 request requiring credentials and therefore also triggers refreshing
+            # the IAM role credentials. In the worst case, refresh() is called 5 minutes plus
+            # epsilon before IAM credential expiration. The resource is refreshed three minutes
+            # after that, leaving two minutes plus epsilon generating a new signed URL, this time
+            # with refreshed IAM role credentials. This consideration only applies to AWS and
+            # Boto2, of course. See https://github.com/BD2KGenomics/toil/issues/1372.
+            time.sleep(3 * 60)
+            logger.debug('Refreshing user script resource %s.', userScriptResource)
+            userScriptResource = userScriptResource.refresh(self._jobStore)
+            logger.debug('Injecting refreshed user script %s into batch system.', userScriptResource)
             self._batchSystem.setUserScript(userScriptResource)
 
     def importFile(self, srcUrl, sharedFileName=None):
diff --git a/src/toil/resource.py b/src/toil/resource.py
index c9260454..a526eec6 100644
--- a/src/toil/resource.py
+++ b/src/toil/resource.py
@@ -69,6 +69,10 @@ class Resource(namedtuple('Resource', ('name', 'pathHash', 'url', 'contentHash')
         and returns a resource object representing that content for the purpose of obtaining it
         again at a generic, public URL. This method should be invoked on the leader node.
 
+        :param toil.jobStores.abstractJobStore.AbstractJobStore jobStore:
+
+        :param str leaderPath:
+
         :rtype: Resource
         """
         pathHash = cls._pathHash(leaderPath)
@@ -84,6 +88,12 @@ class Resource(namedtuple('Resource', ('name', 'pathHash', 'url', 'contentHash')
                    url=jobStore.getSharedPublicUrl(sharedFileName=pathHash),
                    contentHash=contentHash.hexdigest())
 
+    def refresh(self, jobStore):
+        return type(self)(name=self.name,
+                          pathHash=self.pathHash,
+                          url=jobStore.getSharedPublicUrl(sharedFileName=self.pathHash),
+                          contentHash=self.contentHash)
+
     @classmethod
     def prepareSystem(cls):
         """
@@ -540,7 +550,7 @@ class ModuleDescriptor(namedtuple('ModuleDescriptor', ('dirPath', 'name', 'fromV
     @classmethod
     def fromCommand(cls, command):
         assert len(command) == 3
-        return cls( dirPath=command[0], name=command[1], fromVirtualEnv=strict_bool(command[2]))
+        return cls(dirPath=command[0], name=command[1], fromVirtualEnv=strict_bool(command[2]))
 
     def makeLoadable(self):
         module = self if self.belongsToToil else self.localize()
@@ -557,6 +567,5 @@ class ModuleDescriptor(namedtuple('ModuleDescriptor', ('dirPath', 'name', 'fromV
             raise
 
 
-
 class ResourceException(Exception):
     pass