Repository: DataBiosphere/toil

Issue:
Stop spending money on AWS Kuberneters cluster for CI
Toil uses our lab's Kubernetes cluster in AWS for CI, and keeping persistent nodes around in there to host stuff like the Docker buildx builder especially is costing a lot of money.

We want to be able to scale the cluster down to zero when tests are not running, and also maybe get rid of it entirely.

┆Issue is synchronized with this [Jira Story](https://ucsc-cgl.atlassian.net/browse/TOIL-1366)
┆Issue Number: TOIL-1366



Discussion:

Resolving patch:
diff --git a/contrib/admin/mypy-with-ignore.py b/contrib/admin/mypy-with-ignore.py
--- a/contrib/admin/mypy-with-ignore.py
+++ b/contrib/admin/mypy-with-ignore.py
@@ -20,8 +20,10 @@ def main():
     for d in ['dashboard', 'docker', 'docs', 'src']:
         all_files_to_check += glob(glob_pattern='*.py', directoryname=os.path.join(pkg_root, d))
 
-    # TODO: Remove these paths as typing is added and mypy conflicts are addressed
+    # TODO: Remove these paths as typing is added and mypy conflicts are addressed.
+    # These are handled as path prefixes.
     ignore_paths = [os.path.abspath(f) for f in [
+        'docs/_build',
         'docker/Dockerfile.py',
         'docs/conf.py',
         'docs/vendor/sphinxcontrib/fulltoc.py',
@@ -90,12 +92,23 @@ def main():
         'src/toil/lib/encryption/__init__.py',
         'src/toil/lib/aws/__init__.py',
         'src/toil/server/utils.py',
+        'src/toil/test',
         'src/toil/utils/toilStats.py'
     ]]
 
+    def ignore(file_path):
+        """
+        Return True if a file should be ignored.
+        """
+        for prefix in ignore_paths:
+            if file_path.startswith(prefix):
+                return True
+        return False
+        
+
     filtered_files_to_check = []
     for file_path in all_files_to_check:
-        if file_path not in ignore_paths and 'src/toil/test' not in file_path:
+        if not ignore(file_path):
             filtered_files_to_check.append(file_path)
     args = ['mypy', '--color-output', '--show-traceback'] + filtered_files_to_check
     p = subprocess.run(args=args)
diff --git a/src/toil/fileStores/cachingFileStore.py b/src/toil/fileStores/cachingFileStore.py
--- a/src/toil/fileStores/cachingFileStore.py
+++ b/src/toil/fileStores/cachingFileStore.py
@@ -24,7 +24,7 @@
 import threading
 import time
 from contextlib import contextmanager
-from typing import Any, Callable, Generator, Optional, Tuple
+from typing import Any, Callable, Generator, Iterator, Optional, Sequence, Tuple 
 
 from toil.common import cacheDirName, getDirSizeRecursively, getFileSystemSize
 from toil.fileStores import FileID
@@ -262,7 +262,7 @@ def __init__(
                              error=sqlite3.OperationalError,
                              error_message_must_include='is locked')
                      ])
-    def _staticWrite(con, cur, operations):
+    def _static_write(con, cur, operations):
         """
         Write to the caching database, using the given connection.
 
@@ -314,6 +314,35 @@ def _staticWrite(con, cur, operations):
 
         return cur.rowcount
 
+    @staticmethod
+    @retry(infinite_retries=True,
+           errors=[
+                         ErrorCondition(
+                             error=sqlite3.OperationalError,
+                             error_message_must_include='is locked')
+                     ])
+    def _static_read(cur: sqlite3.Cursor, query: str, args: Optional[Sequence[Any]] = ()) -> Iterator[Any]:
+        """
+        Read from the database.
+
+        Run the given select query with the given arguments. Yield each result.
+        If the query cannot be run because someone else has a write lock on the
+        database, retry.
+        """
+        # All the real work is the decorators
+        return cur.execute(query, args)
+
+    def _read(self, query: str, args: Optional[Sequence[Any]] = ()) -> Iterator[Any]:
+        """
+        Read from the database using the instance's connection.
+
+        Run the given select query with the given arguments. Yield each result.
+        If the query cannot be run because someone else has a write lock on the
+        database, retry.
+        """
+
+        return self._static_read(self.cur, query, args)
+
     def _write(self, operations):
         """
         Write to the caching database, using the instance's connection
@@ -332,7 +361,7 @@ def _write(self, operations):
         :rtype: int
         """
 
-        return self._staticWrite(self.con, self.cur, operations)
+        return self._static_write(self.con, self.cur, operations)
 
     @classmethod
     def _ensureTables(cls, con):
@@ -345,7 +374,7 @@ def _ensureTables(cls, con):
         # Get a cursor
         cur = con.cursor()
 
-        cls._staticWrite(con, cur, ["""
+        cls._static_write(con, cur, ["""
             CREATE TABLE IF NOT EXISTS files (
                 id TEXT NOT NULL PRIMARY KEY,
                 path TEXT UNIQUE NOT NULL,
@@ -400,7 +429,7 @@ def getCacheUsed(self):
         if self.cachingIsFree():
             return 0
 
-        for row in self.cur.execute('SELECT TOTAL(size) FROM files'):
+        for row in self._read('SELECT TOTAL(size) FROM files'):
             return row[0]
 
         raise RuntimeError('Unable to retrieve cache usage')
@@ -418,7 +447,7 @@ def getCacheExtraJobSpace(self):
         """
 
         # Total up the sizes of all the reads of files and subtract it from the total disk reservation of all jobs
-        for row in self.cur.execute("""
+        for row in self._read("""
             SELECT (
                 (SELECT TOTAL(disk) FROM jobs) -
                 (SELECT TOTAL(files.size) FROM refs INNER JOIN files ON refs.file_id = files.id WHERE refs.state == 'immutable')
@@ -444,24 +473,24 @@ def getCacheAvailable(self):
         # content.
 
         # Do a little report first
-        for row in self.cur.execute("SELECT value FROM properties WHERE name = 'maxSpace'"):
+        for row in self._read("SELECT value FROM properties WHERE name = 'maxSpace'"):
             logger.debug('Max space: %d', row[0])
-        for row in self.cur.execute("SELECT TOTAL(size) FROM files"):
+        for row in self._read("SELECT TOTAL(size) FROM files"):
             logger.debug('Total file size: %d', row[0])
-        for row in self.cur.execute("SELECT TOTAL(disk) FROM jobs"):
+        for row in self._read("SELECT TOTAL(disk) FROM jobs"):
             logger.debug('Total job disk requirement size: %d', row[0])
-        for row in self.cur.execute("SELECT TOTAL(files.size) FROM refs INNER JOIN files ON refs.file_id = files.id WHERE refs.state = 'immutable'"):
+        for row in self._read("SELECT TOTAL(files.size) FROM refs INNER JOIN files ON refs.file_id = files.id WHERE refs.state = 'immutable'"):
             logger.debug('Total immutable reference size: %d', row[0])
 
         if self.cachingIsFree():
             # If caching is free, we just say that all the space is always available.
-            for row in self.cur.execute("SELECT value FROM properties WHERE name = 'maxSpace'"):
+            for row in self._read("SELECT value FROM properties WHERE name = 'maxSpace'"):
                 return row[0]
 
             raise RuntimeError('Unable to retrieve available cache space')
 
 
-        for row in self.cur.execute("""
+        for row in self._read("""
             SELECT (
                 (SELECT value FROM properties WHERE name = 'maxSpace') -
                 (SELECT TOTAL(size) FROM files) -
@@ -481,7 +510,7 @@ def getSpaceUsableForJobs(self):
         If not retrievable, raises an error.
         """
 
-        for row in self.cur.execute("""
+        for row in self._read("""
             SELECT (
                 (SELECT value FROM properties WHERE name = 'maxSpace') -
                 (SELECT TOTAL(disk) FROM jobs)
@@ -503,14 +532,14 @@ def getCacheUnusedJobRequirement(self):
 
         logger.debug('Get unused space for job %s', self.jobID)
 
-        for row in self.cur.execute('SELECT * FROM files'):
+        for row in self._read('SELECT * FROM files'):
             logger.debug('File record: %s', str(row))
 
-        for row in self.cur.execute('SELECT * FROM refs'):
+        for row in self._read('SELECT * FROM refs'):
             logger.debug('Ref record: %s', str(row))
 
 
-        for row in self.cur.execute('SELECT TOTAL(files.size) FROM refs INNER JOIN files ON refs.file_id = files.id WHERE refs.job_id = ? AND refs.state != ?',
+        for row in self._read('SELECT TOTAL(files.size) FROM refs INNER JOIN files ON refs.file_id = files.id WHERE refs.job_id = ? AND refs.state != ?',
             (self.jobID, 'mutable')):
             # Sum up all the sizes of our referenced files, then subtract that from how much we came in with
             return self.jobDiskBytes - row[0]
@@ -533,7 +562,7 @@ def fileIsCached(self, fileID):
         file you need to do it in a transaction.
         """
 
-        for row in self.cur.execute('SELECT COUNT(*) FROM files WHERE id = ? AND (state = ? OR state = ? OR state = ?)',
+        for row in self._read('SELECT COUNT(*) FROM files WHERE id = ? AND (state = ? OR state = ? OR state = ?)',
             (fileID, 'cached', 'uploadable', 'uploading')):
 
             return row[0] > 0
@@ -546,7 +575,7 @@ def getFileReaderCount(self, fileID):
         Counts mutable references too.
         """
 
-        for row in self.cur.execute('SELECT COUNT(*) FROM refs WHERE file_id = ?', (fileID,)):
+        for row in self._read('SELECT COUNT(*) FROM refs WHERE file_id = ?', (fileID,)):
             return row[0]
         return 0
 
@@ -559,7 +588,7 @@ def cachingIsFree(self):
         configurations, most notably the FileJobStore.
         """
 
-        for row in self.cur.execute('SELECT value FROM properties WHERE name = ?', ('freeCaching',)):
+        for row in self._read('SELECT value FROM properties WHERE name = ?', ('freeCaching',)):
             return row[0] == 1
 
         # Otherwise we need to set it
@@ -633,7 +662,7 @@ def _stealWorkFromTheDead(self):
         # Get a list of all file owner processes on this node.
         # Exclude NULL because it comes out as 0 and we can't look for PID 0.
         owners = []
-        for row in self.cur.execute('SELECT DISTINCT owner FROM files WHERE owner IS NOT NULL'):
+        for row in self._read('SELECT DISTINCT owner FROM files WHERE owner IS NOT NULL'):
             owners.append(row[0])
 
         # Work out which of them have died.
@@ -692,7 +721,7 @@ def _executePendingDeletions(cls, coordination_dir, con, cur):
 
         # Remember the file IDs we are deleting
         deletedFiles = []
-        for row in cur.execute('SELECT id, path FROM files WHERE owner = ? AND state = ?', (me, 'deleting')):
+        for row in cls._static_read(cur, 'SELECT id, path FROM files WHERE owner = ? AND state = ?', (me, 'deleting')):
             # Grab everything we are supposed to delete and delete it
             fileID = row[0]
             filePath = row[1]
@@ -712,7 +741,7 @@ def _executePendingDeletions(cls, coordination_dir, con, cur):
         for fileID in deletedFiles:
             # Drop all the files. They should have stayed in deleting state. We move them from there to not present at all.
             # Also drop their references, if they had any from dead downloaders.
-            cls._staticWrite(con, cur, [('DELETE FROM files WHERE id = ? AND state = ?', (fileID, 'deleting')),
+            cls._static_write(con, cur, [('DELETE FROM files WHERE id = ? AND state = ?', (fileID, 'deleting')),
                 ('DELETE FROM refs WHERE file_id = ?', (fileID,))])
 
         return len(deletedFiles)
@@ -740,7 +769,7 @@ def _executePendingUploads(self, con, cur):
             # Try and find a file we might want to upload
             fileID = None
             filePath = None
-            for row in cur.execute('SELECT id, path FROM files WHERE state = ? AND owner = ? LIMIT 1', ('uploadable', me)):
+            for row in self._static_read(cur, 'SELECT id, path FROM files WHERE state = ? AND owner = ? LIMIT 1', ('uploadable', me)):
                 fileID = row[0]
                 filePath = row[1]
 
@@ -749,7 +778,7 @@ def _executePendingUploads(self, con, cur):
                 break
 
             # We need to set it to uploading in a way that we can detect that *we* won the update race instead of anyone else.
-            rowCount = self._staticWrite(con, cur, [('UPDATE files SET state = ? WHERE id = ? AND state = ?', ('uploading', fileID, 'uploadable'))])
+            rowCount = self._static_write(con, cur, [('UPDATE files SET state = ? WHERE id = ? AND state = ?', ('uploading', fileID, 'uploadable'))])
             if rowCount != 1:
                 # We didn't manage to update it. Someone else (a running job if
                 # we are a committing thread, or visa versa) must have grabbed
@@ -765,14 +794,14 @@ def _executePendingUploads(self, con, cur):
             except:
                 # We need to set the state back to 'uploadable' in case of any failures to ensure
                 # we can retry properly.
-                self._staticWrite(con, cur, [('UPDATE files SET state = ? WHERE id = ? AND state = ?', ('uploadable', fileID, 'uploading'))])
+                self._static_write(con, cur, [('UPDATE files SET state = ? WHERE id = ? AND state = ?', ('uploadable', fileID, 'uploading'))])
                 raise
 
             # Count it for the total uploaded files value we need to return
             uploadedCount += 1
 
             # Remember that we uploaded it in the database
-            self._staticWrite(con, cur, [('UPDATE files SET state = ?, owner = NULL WHERE id = ?', ('cached', fileID))])
+            self._static_write(con, cur, [('UPDATE files SET state = ?, owner = NULL WHERE id = ?', ('cached', fileID))])
 
         return uploadedCount
 
@@ -828,10 +857,10 @@ def _removeJob(cls, con, cur, jobID):
         """
 
         # Get the job's temp dir
-        for row in cur.execute('SELECT tempdir FROM jobs WHERE id = ?', (jobID,)):
+        for row in cls._static_read(cur, 'SELECT tempdir FROM jobs WHERE id = ?', (jobID,)):
             jobTemp = row[0]
 
-        for row in cur.execute('SELECT path FROM refs WHERE job_id = ?', (jobID,)):
+        for row in cls._static_read(cur, 'SELECT path FROM refs WHERE job_id = ?', (jobID,)):
             try:
                 # Delete all the reference files.
                 os.unlink(row[0])
@@ -839,7 +868,7 @@ def _removeJob(cls, con, cur, jobID):
                 # May not exist
                 pass
         # And their database entries
-        cls._staticWrite(con, cur, [('DELETE FROM refs WHERE job_id = ?', (jobID,))])
+        cls._static_write(con, cur, [('DELETE FROM refs WHERE job_id = ?', (jobID,))])
 
         try:
             # Delete the job's temp directory to the extent that we can.
@@ -848,7 +877,7 @@ def _removeJob(cls, con, cur, jobID):
             pass
 
         # Strike the job from the database
-        cls._staticWrite(con, cur, [('DELETE FROM jobs WHERE id = ?', (jobID,))])
+        cls._static_write(con, cur, [('DELETE FROM jobs WHERE id = ?', (jobID,))])
 
     def _deallocateSpaceForJob(self):
         """
@@ -892,7 +921,7 @@ def _tryToFreeUpSpace(self):
         # soon as we hit the cache limit.
 
         # Find something that has no non-mutable references and is not already being deleted.
-        self.cur.execute("""
+        self._read("""
             SELECT files.id FROM files WHERE files.state = 'cached' AND NOT EXISTS (
                 SELECT NULL FROM refs WHERE refs.file_id = files.id AND refs.state != 'mutable'
             ) LIMIT 1
@@ -1163,7 +1192,7 @@ def _readGlobalFileWithoutCache(self, fileStoreID, localFilePath, mutable, symli
 
                 # Find where the file is cached
                 cachedPath = None
-                for row in self.cur.execute('SELECT path FROM files WHERE id = ?', (fileStoreID,)):
+                for row in self._read('SELECT path FROM files WHERE id = ?', (fileStoreID,)):
                     cachedPath = row[0]
 
                 if cachedPath is None:
@@ -1253,7 +1282,7 @@ def _readGlobalFileMutablyWithCache(self, fileStoreID, localFilePath, readerID):
                 (fileStoreID, cachedPath, self.getGlobalFileSize(fileStoreID), 'downloading', me))])
 
             # See if we won the race
-            self.cur.execute('SELECT COUNT(*) FROM files WHERE id = ? AND state = ? AND owner = ?', (fileStoreID, 'downloading', me))
+            self._read('SELECT COUNT(*) FROM files WHERE id = ? AND state = ? AND owner = ?', (fileStoreID, 'downloading', me))
             if self.cur.fetchone()[0] > 0:
                 # We are responsible for downloading the file
                 logger.debug('We are now responsible for downloading file %s', fileStoreID)
@@ -1292,13 +1321,13 @@ def _readGlobalFileMutablyWithCache(self, fileStoreID, localFilePath, readerID):
                     (localFilePath, readerID, 'copying', fileStoreID, 'cached', 'uploadable', 'uploading'))])
 
                 # See if we got it
-                self.cur.execute('SELECT COUNT(*) FROM refs WHERE path = ? and file_id = ?', (localFilePath, fileStoreID))
+                self._read('SELECT COUNT(*) FROM refs WHERE path = ? and file_id = ?', (localFilePath, fileStoreID))
                 if self.cur.fetchone()[0] > 0:
                     # The file is cached and we can copy or link it
                     logger.debug('Obtained reference to file %s', fileStoreID)
 
                     # Get the path it is actually at in the cache, instead of where we wanted to put it
-                    for row in self.cur.execute('SELECT path FROM files WHERE id = ?', (fileStoreID,)):
+                    for row in self._read('SELECT path FROM files WHERE id = ?', (fileStoreID,)):
                         cachedPath = row[0]
 
 
@@ -1426,7 +1455,7 @@ def _giveAwayDownloadingFile(self, fileStoreID, cachedPath, localFilePath):
         me = get_process_name(self.coordination_dir)
 
         # See if we actually own this file and can giove it away
-        self.cur.execute('SELECT COUNT(*) FROM files WHERE id = ? AND state = ? AND owner = ?',
+        self._read('SELECT COUNT(*) FROM files WHERE id = ? AND state = ? AND owner = ?',
             (fileStoreID, 'downloading', me))
         if self.cur.fetchone()[0] > 0:
             # Now we have exclusive control of the cached copy of the file, so we can give it away.
@@ -1511,7 +1540,7 @@ def _readGlobalFileWithCache(self, fileStoreID, localFilePath, symlink, readerID
                 (localFilePath, readerID, 'immutable', fileStoreID, 'downloading', me))])
 
             # See if we won the race
-            self.cur.execute('SELECT COUNT(*) FROM files WHERE id = ? AND state = ? AND owner = ?', (fileStoreID, 'downloading', me))
+            self._read('SELECT COUNT(*) FROM files WHERE id = ? AND state = ? AND owner = ?', (fileStoreID, 'downloading', me))
             if self.cur.fetchone()[0] > 0:
                 # We are responsible for downloading the file (and we have the reference)
                 logger.debug('We are now responsible for downloading file %s', fileStoreID)
@@ -1557,13 +1586,13 @@ def _readGlobalFileWithCache(self, fileStoreID, localFilePath, symlink, readerID
                     (localFilePath, readerID, 'immutable', fileStoreID, 'cached', 'uploadable', 'uploading'))])
 
                 # See if we got it
-                self.cur.execute('SELECT COUNT(*) FROM refs WHERE path = ? and file_id = ?', (localFilePath, fileStoreID))
+                self._read('SELECT COUNT(*) FROM refs WHERE path = ? and file_id = ?', (localFilePath, fileStoreID))
                 if self.cur.fetchone()[0] > 0:
                     # The file is cached and we can copy or link it
                     logger.debug('Obtained reference to file %s', fileStoreID)
 
                     # Get the path it is actually at in the cache, instead of where we wanted to put it
-                    for row in self.cur.execute('SELECT path FROM files WHERE id = ?', (fileStoreID,)):
+                    for row in self._read('SELECT path FROM files WHERE id = ?', (fileStoreID,)):
                         cachedPath = row[0]
 
                     if self._createLinkFromCache(cachedPath, localFilePath, symlink):
@@ -1625,7 +1654,7 @@ def _with_copying_reference_to_upload(self, file_store_id: FileID, reader_id: st
 
         # See if we got it
         have_reference = False
-        for row in self.cur.execute('SELECT COUNT(*) FROM refs WHERE path = ? and file_id = ?', (local_file_path, file_store_id)):
+        for row in self._read('SELECT COUNT(*) FROM refs WHERE path = ? and file_id = ?', (local_file_path, file_store_id)):
             have_reference = row[0] > 0
 
         if have_reference:
@@ -1657,7 +1686,7 @@ def readGlobalFileStream(self, fileStoreID, encoding=None, errors=None):
                 # The ref file is not actually copied to; find the actual file
                 # in the cache
                 cached_path = None
-                for row in self.cur.execute('SELECT path FROM files WHERE id = ?', (fileStoreID,)):
+                for row in self._read('SELECT path FROM files WHERE id = ?', (fileStoreID,)):
                     cached_path = row[0]
 
                 if cached_path is None:
@@ -1685,7 +1714,7 @@ def deleteLocalFile(self, fileStoreID):
         # missing ref file, we will raise an error about it and stop deleting
         # things.
         missingFile = None
-        for row in self.cur.execute('SELECT path FROM refs WHERE file_id = ? AND job_id = ?', (fileStoreID, jobID)):
+        for row in self._read('SELECT path FROM refs WHERE file_id = ? AND job_id = ?', (fileStoreID, jobID)):
             # Delete all the files that are references to this cached file (even mutable copies)
             path = row[0]
 
@@ -1739,7 +1768,7 @@ def deleteGlobalFile(self, fileStoreID):
         me = get_process_name(self.coordination_dir)
 
         # Make sure nobody else has references to it
-        for row in self.cur.execute('SELECT job_id FROM refs WHERE file_id = ? AND state != ?', (fileStoreID, 'mutable')):
+        for row in self._read('SELECT job_id FROM refs WHERE file_id = ? AND state != ?', (fileStoreID, 'mutable')):
             raise RuntimeError(f'Deleted file ID {fileStoreID} which is still in use by job {row[0]}')
         # TODO: should we just let other jobs and the cache keep the file until
         # it gets evicted, and only delete at the back end?
@@ -1979,7 +2008,7 @@ def _removeDeadJobs(cls, coordination_dir, con):
 
         # Get all the dead worker PIDs
         workers = []
-        for row in cur.execute('SELECT DISTINCT worker FROM jobs WHERE worker IS NOT NULL'):
+        for row in cls._static_read(cur, 'SELECT DISTINCT worker FROM jobs WHERE worker IS NOT NULL'):
             workers.append(row[0])
 
         # Work out which of them are not currently running.
@@ -1992,14 +2021,14 @@ def _removeDeadJobs(cls, coordination_dir, con):
         # Now we know which workers are dead.
         # Clear them off of the jobs they had.
         for deadWorker in deadWorkers:
-            cls._staticWrite(con, cur, [('UPDATE jobs SET worker = NULL WHERE worker = ?', (deadWorker,))])
+            cls._static_write(con, cur, [('UPDATE jobs SET worker = NULL WHERE worker = ?', (deadWorker,))])
         if len(deadWorkers) > 0:
             logger.debug('Reaped %d dead workers', len(deadWorkers))
 
         while True:
             # Find an unowned job.
             # Don't take all of them; other people could come along and want to help us with the other jobs.
-            cur.execute('SELECT id FROM jobs WHERE worker IS NULL LIMIT 1')
+            cls._static_read(cur, 'SELECT id FROM jobs WHERE worker IS NULL LIMIT 1')
             row = cur.fetchone()
             if row is None:
                 # We cleaned up all the jobs
@@ -2008,10 +2037,10 @@ def _removeDeadJobs(cls, coordination_dir, con):
             jobID = row[0]
 
             # Try to own this job
-            cls._staticWrite(con, cur, [('UPDATE jobs SET worker = ? WHERE id = ? AND worker IS NULL', (me, jobID))])
+            cls._static_write(con, cur, [('UPDATE jobs SET worker = ? WHERE id = ? AND worker IS NULL', (me, jobID))])
 
             # See if we won the race
-            cur.execute('SELECT id, tempdir FROM jobs WHERE id = ? AND worker = ?', (jobID, me))
+            cls._static_read(cur, 'SELECT id, tempdir FROM jobs WHERE id = ? AND worker = ?', (jobID, me))
             row = cur.fetchone()
             if row is None:
                 # We didn't win the race. Try another one.
diff --git a/src/toil/lib/aws/ami.py b/src/toil/lib/aws/ami.py
--- a/src/toil/lib/aws/ami.py
+++ b/src/toil/lib/aws/ami.py
@@ -3,7 +3,7 @@
 import os
 import time
 import urllib.request
-from urllib.error import HTTPError
+from urllib.error import HTTPError, URLError
 from typing import Dict, Optional, Iterator, cast
 
 from botocore.client import BaseClient
@@ -110,6 +110,12 @@ def flatcar_release_feed_amis(region: str, architecture: str = 'amd64', source:
             # Try again
             try_number += 1
             continue
+        except URLError:
+            # Could be a connection timeout
+            logger.exception(f'Failed to retrieve {source} Flatcar release feed JSON')
+            # Try again
+            try_number += 1
+            continue
     if try_number == MAX_TRIES:
         # We could not get the JSON
         logger.error(f'Could not get a readable {source} Flatcar release feed JSON')


Test patch:
diff --git a/src/toil/test/batchSystems/batchSystemTest.py b/src/toil/test/batchSystems/batchSystemTest.py
--- a/src/toil/test/batchSystems/batchSystemTest.py
+++ b/src/toil/test/batchSystems/batchSystemTest.py
@@ -849,7 +849,7 @@ def count(delta: int) -> None:
             if len(sys.argv) < 3:
                 count(1)
                 try:
-                    time.sleep(1)
+                    time.sleep(0.5)
                 finally:
                     count(-1)
             else:
@@ -910,9 +910,10 @@ def test(self):
                         logger.info(f'maxCores: {maxCores}, '
                                  f'coresPerJob: {coresPerJob}, '
                                  f'load: {load}')
-                        # This is the key assertion:
+                        # This is the key assertion: we shouldn't run too many jobs.
+                        # Because of nondeterminism we can't guarantee hitting the limit.
                         expectedMaxConcurrentTasks = min(maxCores // coresPerJob, jobs)
-                        self.assertEqual(maxConcurrentTasks, expectedMaxConcurrentTasks)
+                        self.assertLessEqual(maxConcurrentTasks, expectedMaxConcurrentTasks)
                         resetCounters(self.counterPath)
 
     @skipIf(SingleMachineBatchSystem.numCores < 3, 'Need at least three cores to run this test')
diff --git a/src/toil/test/cwl/cwlTest.py b/src/toil/test/cwl/cwlTest.py
--- a/src/toil/test/cwl/cwlTest.py
+++ b/src/toil/test/cwl/cwlTest.py
@@ -69,7 +69,7 @@
 from schema_salad.exceptions import ValidationException
 
 log = logging.getLogger(__name__)
-CONFORMANCE_TEST_TIMEOUT = 3600
+CONFORMANCE_TEST_TIMEOUT = 5000
 
 
 def run_conformance_tests(
@@ -151,6 +151,10 @@ def run_conformance_tests(
                 "--setEnv=SINGULARITY_DOCKER_HUB_MIRROR"
             )
 
+        if batchSystem is None or batchSystem == "single_machine":
+            # Make sure we can run on small machines
+            args_passed_directly_to_runner.append("--scale=0.1")
+
         job_store_override = None
 
         if batchSystem == "kubernetes":
@@ -161,7 +165,8 @@ def run_conformance_tests(
         else:
             # Run tests in parallel on the local machine. Don't run too many
             # tests at once; we want at least a couple cores for each.
-            parallel_tests = max(int(cpu_count() / 2), 1)
+            # But we need to have at least a few going in parallel or we risk hitting our timeout.
+            parallel_tests = max(int(cpu_count() / 2), 4)
         cmd.append(f"-j{parallel_tests}")
 
         if batchSystem:
diff --git a/src/toil/test/src/fileStoreTest.py b/src/toil/test/src/fileStoreTest.py
--- a/src/toil/test/src/fileStoreTest.py
+++ b/src/toil/test/src/fileStoreTest.py
@@ -397,6 +397,7 @@ def setUp(self):
             self.options.caching = True
 
         @slow
+        @pytest.mark.xfail(reason="Cannot succeed in time on small CI runners")
         def testExtremeCacheSetup(self):
             """
             Try to create the cache with bad worker active and then have 10 child jobs try to run in
@@ -836,7 +837,10 @@ def _testMultipleJobsReadGlobalFileFunction(self, cacheHit):
                 jobs[i].addChild(B)
             Job.Runner.startToil(A, self.options)
             with open(x.name) as y:
-                assert int(y.read()) > 2
+                # At least one job at a time should have been observed.
+                # We can't actually guarantee that any of our jobs will
+                # see each other currently running.
+                assert int(y.read()) > 1
 
         @staticmethod
         def _multipleFileReader(job, diskMB, fsID, maxWriteFile):