Repository: DataBiosphere/toil

Issue:
Identify time-consuming tests.
The idea is to identify tests that can be run more quickly as a quick for code changes.


Discussion:

Resolving patch:
diff --git a/src/toil/jobStores/googleJobStore.py b/src/toil/jobStores/googleJobStore.py
--- a/src/toil/jobStores/googleJobStore.py
+++ b/src/toil/jobStores/googleJobStore.py
@@ -1,13 +1,28 @@
+# Copyright (C) 2015-2016 Regents of the University of California
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
 from future import standard_library
 standard_library.install_aliases()
 from builtins import str
 import base64
 from contextlib import contextmanager
 import hashlib
-import os
 import uuid
-from bd2k.util.threading import ExceptionalThread
 import boto
+from boto.exception import GSResponseError, GSDataError
 import logging
 import time
 
@@ -69,7 +84,12 @@ def __init__(self, namePrefix, projectID=None, config=None):
         log.debug("Instantiating google jobStore with name: %s", self.bucketName)
         self.gsBucketURL = "gs://"+self.bucketName
 
-        self._headerValues = {"x-goog-project-id": projectID} if projectID else {}
+        self._headerValues = {"x-goog-project-id": bytes(projectID)} if projectID else {}
+
+        # Not reading .boto correctly, this works to create bucket after 'pip install gsutils', which updates osauth2
+        #import gcs_oauth2_boto_plugin
+        #gcs_oauth2_boto_plugin.SetFallbackClientIdAndSecret(gs_access_key_id, gs_secret_access_key)
+
         self._encryptedHeaders = self.headerValues
 
         self.uri = boto.storage_uri(self.gsBucketURL, GOOGLE_STORAGE)
@@ -78,12 +98,9 @@ def __init__(self, namePrefix, projectID=None, config=None):
         exists = True
         try:
             self.files = self.uri.get_bucket(headers=self.headerValues, validate=True)
-        except boto.exception.GSResponseError:
+        except GSResponseError:
             exists = False
 
-        create = config is not None
-        self._checkJobStoreCreation(create, exists, projectID+':'+namePrefix)
-
         if not exists:
             self.files = self._retryCreateBucket(self.uri, self.headerValues)
 
@@ -103,7 +120,7 @@ def destroy(self):
         while True:
             try:
                 self.uri.delete_bucket()
-            except boto.exception.GSResponseError as e:
+            except GSResponseError as e:
                 if e.status == 404:
                     return  # the bucket doesn't exist so we are done
                 else:
@@ -117,7 +134,7 @@ def destroy(self):
             for obj in self.files.list():
                 try:
                     obj.delete()
-                except boto.exception.GSResponseError:
+                except GSResponseError:
                     pass
 
     def create(self, jobNode):
@@ -204,7 +221,7 @@ def deleteFile(self, jobStoreFileID):
         headers = self.encryptedHeaders
         try:
             self._getKey(jobStoreFileID, headers).delete(headers)
-        except boto.exception.GSDataError as e:
+        except GSDataError as e:
             # we tried to delete unencrypted file with encryption headers. unfortunately,
             # we can't determine whether the file passed in is encrypted or not beforehand.
             if e.status == 400:
@@ -217,7 +234,7 @@ def fileExists(self, jobStoreFileID):
         try:
             self._getKey(jobStoreFileID)
             return True
-        except (NoSuchFileException, boto.exception.GSResponseError) as e:
+        except (NoSuchFileException, GSResponseError) as e:
             if isinstance(e, NoSuchFileException):
                 return False
             elif e.status == 400:
@@ -337,8 +354,9 @@ def _retryCreateBucket(uri, headers):
         while True:
             try:
                 # FIMXE: this leaks a connection on exceptions
+                print "====HEADERS", headers
                 return uri.create_bucket(headers=headers)
-            except boto.exception.GSResponseError as e:
+            except GSResponseError as e:
                 if e.status == 429:
                     time.sleep(10)
                 else:
@@ -377,7 +395,7 @@ def _delete(self, jobStoreID, encrypt=True):
         else:
             try:
                 key.delete()
-            except boto.exception.GSResponseError as e:
+            except GSResponseError as e:
                 if e.status == 404:
                     pass
         # best effort delete associated files
@@ -392,7 +410,7 @@ def _getKey(self, jobStoreID=None, headers=None):
         key = None
         try:
             key = self.files.get_key(jobStoreID, headers=headers)
-        except boto.exception.GSDataError:
+        except GSDataError:
             if headers == self.encryptedHeaders:
                 # https://github.com/boto/boto/issues/3518
                 # see self._writeFile for more
@@ -422,7 +440,7 @@ def _writeFile(self, jobStoreID, fileObj, update=False, encrypt=True):
         headers = self.encryptedHeaders if encrypt else self.headerValues
         try:
             key.set_contents_from_file(fileObj, headers=headers)
-        except boto.exception.GSDataError:
+        except GSDataError:
             if encrypt:
                 # Per https://cloud.google.com/storage/docs/encryption#customer-supplied_encryption_keys
                 # the etag and md5 will not match with customer supplied
@@ -445,7 +463,7 @@ def readFrom(self, readable):
                 if update:
                     try:
                         key.set_contents_from_stream(readable, headers=headers)
-                    except boto.exception.GSDataError:
+                    except GSDataError:
                         if encrypt:
                             # https://github.com/boto/boto/issues/3518
                             # see self._writeFile for more
@@ -458,8 +476,8 @@ def readFrom(self, readable):
                         # given generation, i.e. version, before modifying anything. Passing a
                         # generation of 0 insures that the key does not exist remotely.
                         key.set_contents_from_stream(readable, headers=headers, if_generation=0)
-                    except (boto.exception.GSResponseError, boto.exception.GSDataError) as e:
-                        if isinstance(e, boto.exception.GSResponseError):
+                    except (GSResponseError, GSDataError) as e:
+                        if isinstance(e, GSResponseError):
                             if e.status == 412:
                                 raise ConcurrentFileModificationException(key.name)
                             else:


Test patch:
diff --git a/src/toil/test/__init__.py b/src/toil/test/__init__.py
--- a/src/toil/test/__init__.py
+++ b/src/toil/test/__init__.py
@@ -277,10 +277,8 @@ def needs_aws(test_item):
     """
     test_item = _mark_test('aws', test_item)
     keyName = os.getenv('TOIL_AWS_KEYNAME')
-    log.info('Checking keyname: %s', keyName)
     if not keyName or keyName is None:
         return unittest.skip("Set TOIL_AWS_KEYNAME to include this test.")(test_item)
-    log.info("NOPE %s" % keyName)
 
     try:
         # noinspection PyUnresolvedReferences
@@ -317,6 +315,9 @@ def needs_google(test_item):
     Use as a decorator before test classes or methods to only run them if Google Storage usable.
     """
     test_item = _mark_test('google', test_item)
+    projectID = os.getenv('TOIL_GOOGLE_PROJECTID')
+    if not projectID or projectID is None:
+        return unittest.skip("Set TOIL_GOOGLE_PROJECTID to include this test.")(test_item)
     try:
         # noinspection PyUnresolvedReferences
         from boto import config
@@ -337,6 +338,10 @@ def needs_azure(test_item):
     Use as a decorator before test classes or methods to only run them if Azure is usable.
     """
     test_item = _mark_test('azure', test_item)
+    keyName = os.getenv('TOIL_AZURE_KEYNAME')
+    if not keyName or keyName is None:
+        return unittest.skip("Set TOIL_AZURE_KEYNAME to include this test.")(test_item)
+
     try:
         # noinspection PyUnresolvedReferences
         import azure.storage
@@ -383,6 +388,7 @@ def needs_mesos(test_item):
     try:
         # noinspection PyUnresolvedReferences
         import mesos.native
+        import psutil
     except ImportError:
         return unittest.skip(
             "Install Mesos (and Toil with the 'mesos' extra) to include this test.")(test_item)
@@ -506,6 +512,17 @@ def integrative(test_item):
             'Set TOIL_TEST_INTEGRATIVE="True" to include this integration test, '
             'or run `make integration_test_local` to run all integration tests.')(test_item)
 
+def slow(test_item):
+    """
+    Use this decorator to identify tests that are slow and not critical.
+    Skip them if TOIL_TEST_QUICK is true.
+    """
+    test_item = _mark_test('slow', test_item)
+    if not less_strict_bool(os.getenv('TOIL_TEST_QUICK')):
+        return test_item
+    else:
+        return unittest.skip(
+            'Skipped because TOIL_TEST_QUICK is "True"')(test_item)
 
 methodNamePartRegex = re.compile('^[a-zA-Z_0-9]+$')
 
diff --git a/src/toil/test/batchSystems/batchSystemTest.py b/src/toil/test/batchSystems/batchSystemTest.py
--- a/src/toil/test/batchSystems/batchSystemTest.py
+++ b/src/toil/test/batchSystems/batchSystemTest.py
@@ -49,6 +49,7 @@
                        needs_gridengine,
                        needs_slurm,
                        needs_torque,
+                       slow,
                        tempFileContaining)
 from future.utils import with_metaclass
 
@@ -280,6 +281,7 @@ def setUp(self):
         def tearDown(self):
             super(hidden.AbstractBatchSystemJobTest, self).tearDown()
 
+        @slow
         def testJobConcurrency(self):
             """
             Tests that the batch system is allocating core resources properly for concurrent tasks.
@@ -324,7 +326,7 @@ def testResultFile(self):
             self.assertTrue(locator.startswith('file:'))
             self.assertEqual(locator[len('file:'):], filePath)
 
-
+@slow
 @needs_mesos
 class MesosBatchSystemTest(hidden.AbstractBatchSystemTest, MesosTestSupport):
     """
@@ -365,7 +367,8 @@ def supportsWallTime(self):
     def createBatchSystem(self):
         return SingleMachineBatchSystem(config=self.config,
                                         maxCores=numCores, maxMemory=1e9, maxDisk=2001)
-        
+
+@slow
 class MaxCoresSingleMachineBatchSystemTest(ToilTest):
     """
     This test ensures that single machine batch system doesn't exceed the configured number of
@@ -538,7 +541,7 @@ def check(self):
     def stop(self, fileStore):
         subprocess.check_call(self.cmd + ' -1', shell=True)
 
-
+@slow
 @needs_parasol
 class ParasolBatchSystemTest(hidden.AbstractBatchSystemTest, ParasolTestSupport):
     """
@@ -615,6 +618,7 @@ def _getBatchList(self):
         return [self._parseBatchString(line) for line in batchLines[1:] if line]
 
 
+@slow
 @needs_gridengine
 class GridEngineBatchSystemTest(hidden.AbstractGridEngineBatchSystemTest):
     """
@@ -633,6 +637,7 @@ def tearDown(self):
         for f in glob('toil_job*.o*'):
             os.unlink(f)
 
+@slow
 @needs_slurm
 class SlurmBatchSystemTest(hidden.AbstractGridEngineBatchSystemTest):
     """
@@ -651,6 +656,7 @@ def tearDown(self):
         for f in glob('slurm-*.out'):
             os.unlink(f)
 
+@slow
 @needs_torque
 class TorqueBatchSystemTest(hidden.AbstractGridEngineBatchSystemTest):
     """
@@ -683,6 +689,7 @@ class SingleMachineBatchSystemJobTest(hidden.AbstractBatchSystemJobTest):
     def getBatchSystemName(self):
         return "singleMachine"
 
+    @slow
     def testConcurrencyWithDisk(self):
         """
         Tests that the batch system is allocating disk resources properly
@@ -712,6 +719,7 @@ def testConcurrencyWithDisk(self):
         self.assertEqual(maxValue, 1)
 
     @skipIf(SingleMachineBatchSystem.numCores < 4, 'Need at least four cores to run this test')
+    @slow
     def testNestedResourcesDoNotBlock(self):
         """
         Resources are requested in the order Memory > Cpu > Disk.
@@ -798,6 +806,7 @@ def _resourceBlockTestAuxFn(outFile, sleepTime, writeVal):
     time.sleep(sleepTime)
 
 
+@slow
 @needs_mesos
 class MesosBatchSystemJobTest(hidden.AbstractBatchSystemJobTest, MesosTestSupport):
     """
diff --git a/src/toil/test/cwl/cwlTest.py b/src/toil/test/cwl/cwlTest.py
--- a/src/toil/test/cwl/cwlTest.py
+++ b/src/toil/test/cwl/cwlTest.py
@@ -28,7 +28,7 @@
 from six.moves import StringIO
 from six import u as str
 
-from toil.test import ToilTest, needs_cwl
+from toil.test import ToilTest, needs_cwl, slow
 
 @needs_cwl
 class CWLTest(ToilTest):
@@ -61,6 +61,7 @@ def test_run_revsort(self):
                 u'class': u'File',
                 u'checksum': u'sha1$b9214658cc453331b62c2282b772a5c063dbd284'}})
 
+    @slow
     def test_restart(self):
         """Enable restarts with CWLtoil -- run failing test, re-run correct test.
         """
@@ -92,6 +93,7 @@ def path_without_rev():
         except NoSuchJobStoreException:
             pass
 
+    @slow
     def test_run_conformance(self):
         rootDir = self._projectRootPath()
         cwlSpec = os.path.join(rootDir, 'src/toil/test/cwl/spec')
diff --git a/src/toil/test/jobStores/jobStoreTest.py b/src/toil/test/jobStores/jobStoreTest.py
--- a/src/toil/test/jobStores/jobStoreTest.py
+++ b/src/toil/test/jobStores/jobStoreTest.py
@@ -64,6 +64,7 @@
                        needs_encryption,
                        make_tests,
                        needs_google,
+                       slow,
                        experimental)
 from future.utils import with_metaclass
 
@@ -534,6 +535,7 @@ def testImportFtpFile(self):
             finally:
                 ftp.stop()
 
+        @slow
         def testFileDeletion(self):
             """
             Intended to cover the batch deletion of items in the AWSJobStore, but it doesn't hurt
@@ -549,6 +551,7 @@ def testFileDeletion(self):
                     # NB: the fooStream() methods return context managers
                     self.assertRaises(NoSuchFileException, master.readFileStream(fileID).__enter__)
 
+        @slow
         def testMultipartUploads(self):
             """
             This test is meant to cover multi-part uploads in the AWSJobStore but it doesn't hurt
@@ -645,6 +648,7 @@ def testZeroLengthFiles(self):
                 self.assertEquals(f.read(), "")
             self.master.delete(job.jobStoreID)
 
+        @slow
         def testLargeFile(self):
             dirPath = self._createTempDir()
             filePath = os.path.join(dirPath, 'large')
@@ -677,6 +681,7 @@ def assertUrl(self, url):
                 except:
                     self.fail()
 
+        @slow
         def testCleanCache(self):
             # Make a bunch of jobs
             master = self.master
@@ -759,6 +764,7 @@ def _corruptJobStore(self):
             """
             raise NotImplementedError()
 
+        @slow
         def testDestructionOfCorruptedJobStore(self):
             self._corruptJobStore()
             worker = self._createJobStore()
@@ -870,7 +876,7 @@ def _cleanUpExternalStore(self, dirPath):
 @experimental
 @needs_google
 class GoogleJobStoreTest(AbstractJobStoreTest.Test):
-    projectID = 'cgc-05-0006'
+    projectID = os.getenv('TOIL_GOOGLE_PROJECTID')
     headers = {"x-goog-project-id": projectID}
 
     def _createJobStore(self):
@@ -990,6 +996,7 @@ def testSDBDomainsDeletedOnFailedJobstoreBucketCreation(self):
                     with attempt:
                         s3.delete_bucket(bucket=bucket)
 
+    @slow
     def testInlinedFiles(self):
         from toil.jobStores.aws.jobStore import AWSJobStore
         master = self.master
@@ -1011,6 +1018,7 @@ def testInaccessableLocation(self):
             args, kwargs = mock_log.warn.call_args
             self.assertTrue('Could not determine location' in args[0])
 
+    @slow
     def testMultiPartImportFailures(self):
         # This should be less than the number of threads in the pool used by the MP copy.
         num_parts = 10
@@ -1118,7 +1126,7 @@ def testInvalidJobStoreName(self):
 
 @needs_azure
 class AzureJobStoreTest(AbstractJobStoreTest.Test):
-    accountName = 'toiltest'
+    accountName = os.getenv('TOIL_AZURE_KEYNAME')
 
     def _createJobStore(self):
         from toil.jobStores.azureJobStore import AzureJobStore
@@ -1213,12 +1221,14 @@ class EncryptedFileJobStoreTest(FileJobStoreTest, AbstractEncryptedJobStoreTest.
 
 @needs_aws
 @needs_encryption
+@slow
 class EncryptedAWSJobStoreTest(AWSJobStoreTest, AbstractEncryptedJobStoreTest.Test):
     pass
 
 
 @needs_azure
 @needs_encryption
+@slow
 class EncryptedAzureJobStoreTest(AzureJobStoreTest, AbstractEncryptedJobStoreTest.Test):
     pass
 
diff --git a/src/toil/test/lib/dockerTest.py b/src/toil/test/lib/dockerTest.py
--- a/src/toil/test/lib/dockerTest.py
+++ b/src/toil/test/lib/dockerTest.py
@@ -11,7 +11,7 @@
 from toil.job import Job
 from toil.leader import FailedJobsException
 from toil.lib.docker import dockerCall, dockerCheckOutput, _containerIsRunning, _dockerKill, STOP, FORGO, RM
-from toil.test import ToilTest
+from toil.test import ToilTest, slow
 
 _log = logging.getLogger(__name__)
 
@@ -32,6 +32,7 @@ class DockerTest(ToilTest):
     def setUp(self):
         self.tempDir = self._createTempDir(purpose='tempDir')
 
+    @slow
     def testDockerClean(self, caching=True):
         """
         Run the test container that creates a file in the work dir, and sleeps for 5 minutes.  Ensure
@@ -144,6 +145,7 @@ def testDockerPermissions(self, caching=True):
     def testDockerPermissionsNonCaching(self):
         self.testDockerPermissions(caching=False)
 
+    @slow
     def testNonCachingDockerChain(self):
         self.testDockerPipeChain(caching=False)
 
diff --git a/src/toil/test/mesos/MesosDataStructuresTest.py b/src/toil/test/mesos/MesosDataStructuresTest.py
--- a/src/toil/test/mesos/MesosDataStructuresTest.py
+++ b/src/toil/test/mesos/MesosDataStructuresTest.py
@@ -17,9 +17,10 @@
 import uuid
 import random
 
-from toil.test import ToilTest, needs_mesos
+from toil.test import ToilTest, needs_mesos, slow
 
 
+@slow
 @needs_mesos
 class DataStructuresTest(ToilTest):
 
diff --git a/src/toil/test/mesos/mesosTest.py b/src/toil/test/mesos/mesosTest.py
--- a/src/toil/test/mesos/mesosTest.py
+++ b/src/toil/test/mesos/mesosTest.py
@@ -22,7 +22,7 @@
 
 from toil.lib.bioio import getLogLevelString, system
 from toil.test.mesos.stress import main as stressMain
-from toil.test import ToilTest, needs_mesos
+from toil.test import ToilTest, needs_mesos, slow
 from toil.batchSystems.mesos.test import MesosTestSupport
 from toil.test.mesos import helloWorld
 
@@ -32,6 +32,7 @@
 log = logging.getLogger(__name__)
 
 
+@slow
 @needs_mesos
 class MesosTest(ToilTest, MesosTestSupport):
     """
diff --git a/src/toil/test/provisioners/aws/awsProvisionerTest.py b/src/toil/test/provisioners/aws/awsProvisionerTest.py
--- a/src/toil/test/provisioners/aws/awsProvisionerTest.py
+++ b/src/toil/test/provisioners/aws/awsProvisionerTest.py
@@ -33,7 +33,7 @@
 from uuid import uuid4
 
 
-from toil.test import needs_aws, integrative, ToilTest, needs_appliance, timeLimit
+from toil.test import needs_aws, integrative, ToilTest, needs_appliance, timeLimit, slow
 
 log = logging.getLogger(__name__)
 
@@ -41,6 +41,7 @@
 @needs_aws
 @integrative
 @needs_appliance
+@slow
 class AbstractAWSAutoscaleTest(ToilTest):
 
     def sshUtil(self, command):
diff --git a/src/toil/test/provisioners/clusterScalerTest.py b/src/toil/test/provisioners/clusterScalerTest.py
--- a/src/toil/test/provisioners/clusterScalerTest.py
+++ b/src/toil/test/provisioners/clusterScalerTest.py
@@ -33,7 +33,7 @@
 
 from toil.job import JobNode
 from toil.provisioners import Node
-from toil.test import ToilTest
+from toil.test import ToilTest, slow
 from toil.batchSystems.abstractBatchSystem import (AbstractScalableBatchSystem,
                                                    NodeInfo,
                                                    AbstractBatchSystem)
@@ -140,6 +140,7 @@ def _testClusterScaling(self, config, numJobs, numPreemptableJobs):
                          bs.totalWorkerTime,
                          old_div(bs.totalWorkerTime, bs.totalJobs) if bs.totalJobs > 0 else 0.0))
 
+    @slow
     def testClusterScaling(self):
         """
         Test scaling for a batch of non-preemptable jobs and no preemptable jobs (makes debugging
@@ -167,6 +168,7 @@ def testClusterScaling(self):
 
         self._testClusterScaling(config, numJobs=100, numPreemptableJobs=0)
 
+    @slow
     def testClusterScalingWithPreemptableJobs(self):
         """
         Test scaling simultaneously for a batch of preemptable and non-preemptable jobs.
diff --git a/src/toil/test/sort/sortTest.py b/src/toil/test/sort/sortTest.py
--- a/src/toil/test/sort/sortTest.py
+++ b/src/toil/test/sort/sortTest.py
@@ -42,6 +42,7 @@
                        needs_gridengine,
                        needs_torque,
                        needs_google,
+                       slow,
                        experimental)
 from toil.jobStores.abstractJobStore import NoSuchJobStoreException, JobStoreExistsException
 from toil.leader import FailedJobsException
@@ -69,6 +70,7 @@ def runMain(options):
             raise
 
 
+@slow
 class SortTest(ToilTest, MesosTestSupport, ParasolTestSupport):
     """
     Tests Toil by sorting a file in parallel on various combinations of job stores and batch
@@ -328,7 +330,8 @@ def _awsJobStore(self):
         return 'aws:%s:sort-test-%s' % (self.awsRegion(), uuid4())
 
     def _azureJobStore(self):
-        return "azure:toiltest:sort-test-%s" % uuid4()
+        accountName = os.getenv('TOIL_AZURE_KEYNAME')
+        return "azure:%s:sort-test-%s" % (accountName, uuid4())
 
     def _googleJobStore(self):
         return "google:cgc-05-0006:sort-test-%s" % uuid4()
diff --git a/src/toil/test/src/checkpointTest.py b/src/toil/test/src/checkpointTest.py
--- a/src/toil/test/src/checkpointTest.py
+++ b/src/toil/test/src/checkpointTest.py
@@ -14,7 +14,7 @@
 
 from __future__ import absolute_import
 from toil.job import Job
-from toil.test import ToilTest
+from toil.test import ToilTest, slow
 from toil.leader import FailedJobsException
 from toil.jobStores.abstractJobStore import NoSuchFileException
 
@@ -32,6 +32,7 @@ def testCheckpointNotRetried(self):
         with self.assertRaises(FailedJobsException):
             Job.Runner.startToil(CheckRetryCount(numFailuresBeforeSuccess=1), options)
 
+    @slow
     def testCheckpointRetriedOnce(self):
         """A checkpoint job should be retried exactly once if the workflow has a retryCount of 1."""
         options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
@@ -47,6 +48,7 @@ def testCheckpointRetriedOnce(self):
         with self.assertRaises(FailedJobsException):
             Job.Runner.startToil(CheckRetryCount(numFailuresBeforeSuccess=2), options)
 
+    @slow
     def testCheckpointedRestartSucceeds(self):
         """A checkpointed job should succeed on restart of a failed run if its child job succeeds."""
         options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
diff --git a/src/toil/test/src/fileStoreTest.py b/src/toil/test/src/fileStoreTest.py
--- a/src/toil/test/src/fileStoreTest.py
+++ b/src/toil/test/src/fileStoreTest.py
@@ -25,7 +25,7 @@
 
 from toil.job import Job
 from toil.fileStore import IllegalDeletionCacheError, CachingFileStore
-from toil.test import ToilTest, needs_aws, needs_azure, needs_google, experimental
+from toil.test import ToilTest, needs_aws, needs_azure, needs_google, experimental, slow
 from toil.leader import FailedJobsException
 from toil.jobStores.abstractJobStore import NoSuchFileException
 from toil.fileStore import CacheUnbalancedError
@@ -66,7 +66,8 @@ def _getTestJobStore(self):
             elif self.jobStoreType == 'aws':
                 return 'aws:%s:cache-tests-%s' % (self.awsRegion(), uuid4())
             elif self.jobStoreType == 'azure':
-                return 'azure:toiltest:cache-tests-' + str(uuid4())
+                accountName = os.getenv('TOIL_AZURE_KEYNAME')
+                return 'azure:%s:cache-tests-%s' % (accountName, str(uuid4()))
             elif self.jobStoreType == 'google':
                 projectID = 'cgc-05-0006'
                 return 'google:' + projectID + ':cache-tests-' + str(uuid4())
@@ -106,6 +107,7 @@ def testToilIsNotBroken(self):
 
         # Test filestore operations.  This is a slightly less intense version of the cache specific
         # test `testReturnFileSizes`
+        @slow
         def testFileStoreOperations(self):
             """
             Write a couple of files to the jobstore.  Delete a couple of them.  Read back written
@@ -253,6 +255,7 @@ def _writeNonLocalFilesLambda(job, files):
             job.defer(lmd, files[0], nlf=files[1])
             return None
 
+        @slow
         def testDeferredFunctionRunsWithFailures(self):
             """
             Create 2 non local filesto use as flags.  Create a job that registers a function that
@@ -311,6 +314,7 @@ def _deleteFile(nonLocalFile, nlf=None):
             if nlf is not None:
                 os.remove(nlf)
 
+        @slow
         def testNewJobsCanHandleOtherJobDeaths(self):
             """
             Create 2 non-local files and then create 2 jobs. The first job registers a deferred job
@@ -461,6 +465,7 @@ def setUp(self):
             super(hidden.AbstractCachingFileStoreTest, self).setUp()
             self.options.disableCaching = False
 
+        @slow
         def testExtremeCacheSetup(self):
             """
             Try to create the cache with bad worker active and then have 10 child jobs try to run in
@@ -482,6 +487,7 @@ def testExtremeCacheSetup(self):
                     jobs[i].addChild(F)
                 Job.Runner.startToil(E, self.options)
 
+        @slow
         def testCacheLockRace(self):
             """
             Make 3 jobs compete for the same cache lock file.  If they have the lock at the same
@@ -540,6 +546,7 @@ def _raceTestSuccess(job):
                 assert cacheInfo.nlink == 0
                 assert cacheInfo.cached > 1
 
+        @slow
         def testCacheEvictionPartialEvict(self):
             """
             Ensure the cache eviction happens as expected.  Two files (20MB and 30MB) are written
@@ -555,6 +562,7 @@ def testCacheEvictionPartialEvict(self):
 
             self._testCacheEviction(file1MB=20, file2MB=30, diskRequestMB=10)
 
+        @slow
         def testCacheEvictionTotalEvict(self):
             """
             Ensure the cache eviction happens as expected.  Two files (20MB and 30MB) are written
@@ -570,6 +578,7 @@ def testCacheEvictionTotalEvict(self):
 
             self._testCacheEviction(file1MB=20, file2MB=30, diskRequestMB=30)
 
+        @slow
         def testCacheEvictionFailCase(self):
             """
             Ensure the cache eviction happens as expected.  Two files (20MB and 30MB) are written
@@ -707,6 +716,7 @@ def _probeJobReqs(job, total=None, cached=None, sigmaJob=None):
                     assert cacheInfoMB == expectedMB, 'Testing %s: Expected ' % value + \
                                                       '%s but got %s.' % (expectedMB, cacheInfoMB)
 
+        @slow
         def testAsyncWriteWithCaching(self):
             """
             Ensure the Async Writing of files happens as expected.  The first Job forcefully
@@ -866,6 +876,7 @@ def testReadCachHitFileFromJobStore(self):
             A.addChild(B)
             Job.Runner.startToil(A, self.options)
 
+        @slow
         def testMultipleJobsReadSameCacheHitGlobalFile(self):
             """
             Write a local file to the job store (hence adding a copy to cache), then have 10 jobs
@@ -876,6 +887,7 @@ def testMultipleJobsReadSameCacheHitGlobalFile(self):
             """
             self._testMultipleJobsReadGlobalFileFunction(cacheHit=True)
 
+        @slow
         def testMultipleJobsReadSameCacheMissGlobalFile(self):
             """
             Write a non-local file to the job store(hence no cached copy), then have 10 jobs read
@@ -958,6 +970,7 @@ def _writeExportGlobalFile(job):
             job.fileStore.exportFile(job.fileStore.writeGlobalFile(fileName), 'File://' + outputFile)
             assert filecmp.cmp(fileName, outputFile)
 
+        @slow
         def testFileStoreExportFile(self):
             # Tests that files written to job store can be immediately exported
             # motivated by https://github.com/BD2KGenomics/toil/issues/1469
@@ -965,6 +978,7 @@ def testFileStoreExportFile(self):
             Job.Runner.startToil(root, self.options)
 
         # Testing for the return of file sizes to the sigma job pool.
+        @slow
         def testReturnFileSizes(self):
             """
             Write a couple of files to the jobstore.  Delete a couple of them.  Read back written
@@ -979,6 +993,7 @@ def testReturnFileSizes(self):
                               disk='2G')
             Job.Runner.startToil(F, self.options)
 
+        @slow
         def testReturnFileSizesWithBadWorker(self):
             """
             Write a couple of files to the jobstore.  Delete a couple of them.  Read back written
@@ -1101,6 +1116,7 @@ def _requirementsConcur(job, jobDisk, cached):
             assert jobState['jobReqs'] == jobDisk
 
         # Testing the resumability of a failed worker
+        @slow
         def testControlledFailedWorkerRetry(self):
             """
             Conduct a couple of job store operations.  Then die.  Ensure that the restarted job is
@@ -1137,12 +1153,14 @@ def _controlledFailTestFn(job, jobDisk, testDir):
                     fH.write(pack('d', cached))
                 os.kill(os.getpid(), signal.SIGKILL)
 
+        @slow
         def testRemoveLocalMutablyReadFile(self):
             """
             If a mutably read file is deleted by the user, it is ok.
             """
             self._deleteLocallyReadFilesFn(readAsMutable=True)
 
+        @slow
         def testRemoveLocalImmutablyReadFile(self):
             """
             If an immutably read file is deleted by the user, it is not ok.
@@ -1283,6 +1301,7 @@ class NonCachingFileStoreTestWithAwsJobStore(hidden.AbstractNonCachingFileStoreT
     jobStoreType = 'aws'
 
 
+@slow
 @needs_aws
 @pytest.mark.timeout(1000)
 class CachingFileStoreTestWithAwsJobStore(hidden.AbstractCachingFileStoreTest):
@@ -1294,6 +1313,7 @@ class NonCachingFileStoreTestWithAzureJobStore(hidden.AbstractNonCachingFileStor
     jobStoreType = 'azure'
 
 
+@slow
 @needs_azure
 @pytest.mark.timeout(1000)
 class CachingFileStoreTestWithAzureJobStore(hidden.AbstractCachingFileStoreTest):
@@ -1306,6 +1326,7 @@ class NonCachingFileStoreTestWithGoogleJobStore(hidden.AbstractNonCachingFileSto
     jobStoreType = 'google'
 
 
+@slow
 @experimental
 @needs_google
 @pytest.mark.timeout(1000)
diff --git a/src/toil/test/src/hotDeploymentTest.py b/src/toil/test/src/hotDeploymentTest.py
--- a/src/toil/test/src/hotDeploymentTest.py
+++ b/src/toil/test/src/hotDeploymentTest.py
@@ -7,13 +7,14 @@
 
 from bd2k.util.iterables import concat
 
-from toil.test import needs_mesos, ApplianceTestSupport, needs_appliance
+from toil.test import needs_mesos, ApplianceTestSupport, needs_appliance, slow
 
 log = logging.getLogger(__name__)
 
 
 @needs_mesos
 @needs_appliance
+@slow
 class HotDeploymentTest(ApplianceTestSupport):
     """
     Tests various hot-deployment scenarios. Using the appliance, i.e. a docker container,
diff --git a/src/toil/test/src/importExportFileTest.py b/src/toil/test/src/importExportFileTest.py
--- a/src/toil/test/src/importExportFileTest.py
+++ b/src/toil/test/src/importExportFileTest.py
@@ -21,7 +21,7 @@
 from toil.common import Toil
 from toil.job import Job
 from toil.leader import FailedJobsException
-from toil.test import ToilTest
+from toil.test import ToilTest, slow
 from toil.fileStore import FileID
 
 
@@ -72,6 +72,7 @@ def _importExport(self, restart):
 
         self._importExportFile(options, fail=False)
 
+    @slow
     def testImportExportRestartTrue(self):
         self._importExport(restart=True)
 
diff --git a/src/toil/test/src/jobFileStoreTest.py b/src/toil/test/src/jobFileStoreTest.py
--- a/src/toil/test/src/jobFileStoreTest.py
+++ b/src/toil/test/src/jobFileStoreTest.py
@@ -22,7 +22,7 @@
 
 from toil.common import Toil
 from toil.job import Job
-from toil.test import ToilTest
+from toil.test import ToilTest, slow
 
 PREFIX_LENGTH=200
 
@@ -51,7 +51,8 @@ def testImportLinking(self):
         """
         options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
         options.linkImports = True
-        fileName = 'dummyFile.txt'
+        tempDir = self._createTempDir('testImportLinking')
+        fileName = os.path.join(tempDir, 'dummyFile.txt')
         with open(fileName, 'w') as fh:
             fh.write('Subtle literature reference.')
         with Toil(options) as workflow:
@@ -91,7 +92,8 @@ def testJobFileStore(self):
         Tests case that about half the files are cached
         """
         self._testJobFileStore(retryCount=0, badWorker=0.0,  stringNo=5, stringLength=1000000)
-    
+
+    @slow
     def testJobFileStoreWithBadWorker(self):
         """
         Tests case that about half the files are cached and the worker is randomly
diff --git a/src/toil/test/src/jobServiceTest.py b/src/toil/test/src/jobServiceTest.py
--- a/src/toil/test/src/jobServiceTest.py
+++ b/src/toil/test/src/jobServiceTest.py
@@ -24,7 +24,7 @@
 
 from toil.lib.bioio import getTempFile
 from toil.job import Job
-from toil.test import ToilTest
+from toil.test import ToilTest, slow
 import time
 import sys
 from threading import Thread, Event
@@ -38,7 +38,8 @@ class JobServiceTest(ToilTest):
     """
     Tests testing the Job.Service class
     """
-    
+
+    @slow
     def testServiceSerialization(self):
         """
         Tests that a service can receive a promise without producing a serialization
@@ -53,7 +54,8 @@ def testServiceSerialization(self):
         # serialization on services is working correctly.
         
         self.runToil(job)
-        
+
+    @slow
     def testService(self, checkpoint=False):
         """
         Tests the creation of a Job.Service with random failures of the worker.
@@ -72,7 +74,8 @@ def testService(self, checkpoint=False):
                 self.assertEquals(int(open(outFile, 'r').readline()), messageInt)
             finally:
                 os.remove(outFile)
-                
+
+    @slow
     def testServiceDeadlock(self):
         """
         Creates a job with more services than maxServices, checks that deadlock is detected.
@@ -109,6 +112,7 @@ def testServiceWithCheckpoints(self):
         """
         self.testService(checkpoint=True)
 
+    @slow
     @skipIf(SingleMachineBatchSystem.numCores < 4, 'Need at least four cores to run this test')
     def testServiceRecursive(self, checkpoint=True):
         """
@@ -131,6 +135,7 @@ def testServiceRecursive(self, checkpoint=True):
             finally:
                 os.remove(outFile)
 
+    @slow
     @skipIf(SingleMachineBatchSystem.numCores < 4, 'Need at least four cores to run this test')
     def testServiceParallelRecursive(self, checkpoint=True):
         """
@@ -331,7 +336,7 @@ def serviceAccessor(job, communicationFiles, outFile, randInt):
             return
 
     assert 0 # Job failed to get info from the service
-    
+
 class TestServiceSerialization(Job.Service):
     def __init__(self, messageInt, *args, **kwargs):
         """
diff --git a/src/toil/test/src/jobTest.py b/src/toil/test/src/jobTest.py
--- a/src/toil/test/src/jobTest.py
+++ b/src/toil/test/src/jobTest.py
@@ -30,7 +30,7 @@
 from toil.leader import FailedJobsException
 from toil.lib.bioio import getTempFile
 from toil.job import Job, JobGraphDeadlockException, JobFunctionWrappingJob
-from toil.test import ToilTest
+from toil.test import ToilTest, slow
 
 logger = logging.getLogger(__name__)
 
@@ -56,6 +56,7 @@ def testResourceRequirements(self):
         with Toil(options) as toil:
             toil.start(Job.wrapJobFn(checkRequirements, memory='1000M'))
 
+    @slow
     def testStatic(self):
         """
         Create a DAG of jobs non-dynamically and run it. DAG is:
@@ -139,7 +140,8 @@ def testStatic2(self):
             self.assertEquals(open(outFile, 'r').readline(), "ABCDE")
         finally:
             os.remove(outFile)
-            
+
+    @slow
     def testTrivialDAGConsistency(self):
         options = Job.Runner.getDefaultOptions(self._createTempDir() + '/jobStore')
         options.clean = 'always'
@@ -167,6 +169,7 @@ def testDAGConsistency(self):
             else:
                 self.fail()
 
+    @slow
     def testSiblingDAGConsistency(self):
         """
         Slightly more complex case. The stranded job's predecessors are siblings instead of
@@ -279,6 +282,7 @@ def checkFollowOnEdgeCycleDetection(fNode, tNode):
                 and (fNode, tNode) not in childEdges and (fNode, tNode) not in followOnEdges):
                 checkFollowOnEdgeCycleDetection(fNode, tNode)
 
+    @slow
     def testNewCheckpointIsLeafVertexNonRootCase(self):
         """
         Test for issue #1465: Detection of checkpoint jobs that are not leaf vertices
@@ -300,6 +304,7 @@ def createWorkflow():
 
         self.runNewCheckpointIsLeafVertexTest(createWorkflow)
 
+    @slow
     def testNewCheckpointIsLeafVertexRootCase(self):
         """
         Test for issue #1466: Detection of checkpoint jobs that are not leaf vertices
@@ -385,6 +390,7 @@ def runCheckpointVertexTest(self,
             except expectedException as ex:
                 logger.info("The expected exception was thrown: %s", repr(ex))
 
+    @slow
     def testEvaluatingRandomDAG(self):
         """
         Randomly generate test input then check that the job graph can be 
diff --git a/src/toil/test/src/miscTests.py b/src/toil/test/src/miscTests.py
--- a/src/toil/test/src/miscTests.py
+++ b/src/toil/test/src/miscTests.py
@@ -14,7 +14,7 @@
 
 from __future__ import absolute_import, print_function
 from builtins import range
-from toil.test import ToilTest
+from toil.test import ToilTest, slow
 from uuid import uuid4
 
 import math
@@ -34,6 +34,7 @@ def setUp(self):
         super(MiscTests, self).setUp()
         self.testDir = self._createTempDir()
 
+    @slow
     def testGetSizeOfDirectoryWorks(self):
         from toil.common import getDirSizeRecursively
         # os.stat lists the number of 512-byte blocks used, but really, the file system is using
diff --git a/src/toil/test/src/multipartTransferTest.py b/src/toil/test/src/multipartTransferTest.py
--- a/src/toil/test/src/multipartTransferTest.py
+++ b/src/toil/test/src/multipartTransferTest.py
@@ -25,7 +25,7 @@
 
 from toil.jobStores.aws.jobStore import copyKeyMultipart
 from toil.jobStores.aws.utils import region_to_bucket_location
-from toil.test import ToilTest, make_tests
+from toil.test import ToilTest, make_tests, slow
 
 partSize = 2 ** 20 * 5
 logger = logging.getLogger(__name__)
@@ -61,6 +61,7 @@ def openS3(keySize=None):
             bucket.delete()
 
 
+@slow
 class AWSMultipartCopyTest(ToilTest):
 
     @classmethod
diff --git a/src/toil/test/src/promisedRequirementTest.py b/src/toil/test/src/promisedRequirementTest.py
--- a/src/toil/test/src/promisedRequirementTest.py
+++ b/src/toil/test/src/promisedRequirementTest.py
@@ -24,7 +24,7 @@
 
 from toil.job import Job
 from toil.job import PromisedRequirement
-from toil.test import needs_mesos
+from toil.test import needs_mesos, slow
 from toil.batchSystems.mesos.test import MesosTestSupport
 
 log = logging.getLogger(__name__)
@@ -42,6 +42,7 @@ class AbstractPromisedRequirementsTest(batchSystemTest.hidden.AbstractBatchSyste
         An abstract base class for testing Toil workflows with promised requirements.
         """
 
+        @slow
         def testConcurrencyDynamic(self):
             """
             Asserts that promised core resources are allocated properly using a dynamic Toil workflow
@@ -56,6 +57,7 @@ def testConcurrencyDynamic(self):
                 maxValue = max(values)
                 self.assertEqual(maxValue, old_div(self.cpuCount, coresPerJob))
 
+        @slow
         def testConcurrencyStatic(self):
             """
             Asserts that promised core resources are allocated properly using a static DAG
@@ -126,6 +128,7 @@ def testPromisesWithJobStoreFileObjects(self, caching=True):
         def testPromisesWithNonCachingFileStore(self):
             self.testPromisesWithJobStoreFileObjects(caching=False)
 
+        @slow
         def testPromiseRequirementRaceStatic(self):
             """
             Checks for a race condition when using promised requirements and child job functions.
diff --git a/src/toil/test/src/regularLogTest.py b/src/toil/test/src/regularLogTest.py
--- a/src/toil/test/src/regularLogTest.py
+++ b/src/toil/test/src/regularLogTest.py
@@ -17,7 +17,7 @@
 import sys
 import os
 
-from toil.test import ToilTest
+from toil.test import ToilTest, slow
 from toil.test.mesos import helloWorld
 
 
@@ -46,6 +46,7 @@ def _assertFileTypeExists(self, dir, extension, encoding=None):
                     self.assertEqual(mime[1], encoding)
 
 
+    @slow
     def testLogToMaster(self):
         toilOutput = subprocess.check_output([sys.executable,
                                               '-m', helloWorld.__name__,
@@ -64,6 +65,7 @@ def testWriteLogs(self):
                                              stderr=subprocess.STDOUT)
         self._assertFileTypeExists(self.tempDir, '.log')
 
+    @slow
     def testWriteGzipLogs(self):
         toilOutput = subprocess.check_output([sys.executable,
                                               '-m', helloWorld.__name__,
@@ -74,6 +76,7 @@ def testWriteGzipLogs(self):
                                               stderr=subprocess.STDOUT)
         self._assertFileTypeExists(self.tempDir, '.log.gz', 'gzip')
 
+    @slow
     def testMultipleLogToMaster(self):
         toilOutput = subprocess.check_output([sys.executable,
                                               '-m', helloWorld.__name__,
diff --git a/src/toil/test/src/restartDAGTest.py b/src/toil/test/src/restartDAGTest.py
--- a/src/toil/test/src/restartDAGTest.py
+++ b/src/toil/test/src/restartDAGTest.py
@@ -19,7 +19,7 @@
 from toil.common import Toil
 from toil.job import Job
 from toil.leader import FailedJobsException
-from toil.test import ToilTest
+from toil.test import ToilTest, slow
 
 import inspect
 import os
@@ -44,9 +44,11 @@ def tearDown(self):
         super(RestartDAGTest, self).tearDown()
         shutil.rmtree(self.testJobStore)
 
+    @slow
     def testRestartedWorkflowSchedulesCorrectJobsOnFailedParent(self):
         self._testRestartedWorkflowSchedulesCorrectJobs('raise')
 
+    @slow
     def testRestartedWorkflowSchedulesCorrectJobsOnKilledParent(self):
         self._testRestartedWorkflowSchedulesCorrectJobs('kill')
 
diff --git a/src/toil/test/src/resumabilityTest.py b/src/toil/test/src/resumabilityTest.py
--- a/src/toil/test/src/resumabilityTest.py
+++ b/src/toil/test/src/resumabilityTest.py
@@ -20,10 +20,11 @@
 from six.moves import xrange
 
 from toil.job import Job
-from toil.test import ToilTest
+from toil.test import ToilTest, slow
 from toil.jobStores.abstractJobStore import NoSuchFileException
 from toil.leader import FailedJobsException
 
+@slow
 class ResumabilityTest(ToilTest):
     """
     https://github.com/BD2KGenomics/toil/issues/808
diff --git a/src/toil/test/src/toilContextManagerTest.py b/src/toil/test/src/toilContextManagerTest.py
--- a/src/toil/test/src/toilContextManagerTest.py
+++ b/src/toil/test/src/toilContextManagerTest.py
@@ -15,9 +15,10 @@
 from __future__ import absolute_import
 from toil.common import Toil, ToilContextManagerException
 from toil.job import Job
-from toil.test import ToilTest
+from toil.test import ToilTest, slow
 
 
+@slow
 class ToilContextManagerTest(ToilTest):
     def testContextManger(self):
         options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
diff --git a/src/toil/test/src/userDefinedJobArgTypeTest.py b/src/toil/test/src/userDefinedJobArgTypeTest.py
--- a/src/toil/test/src/userDefinedJobArgTypeTest.py
+++ b/src/toil/test/src/userDefinedJobArgTypeTest.py
@@ -17,7 +17,7 @@
 import sys
 from subprocess import check_call
 from toil.job import Job
-from toil.test import ToilTest
+from toil.test import ToilTest, slow
 
 
 class UserDefinedJobArgTypeTest(ToilTest):
@@ -39,6 +39,7 @@ def testJobFunction(self):
         """Test with first job being a function"""
         Job.Runner.startToil(Job.wrapJobFn(jobFunction, 0, Foo()), self.options)
 
+    @slow
     def testJobClass(self):
         """Test with first job being an instance of a class"""
         Job.Runner.startToil(JobClass(0, Foo()), self.options)
diff --git a/src/toil/test/utils/utilsTest.py b/src/toil/test/utils/utilsTest.py
--- a/src/toil/test/utils/utilsTest.py
+++ b/src/toil/test/utils/utilsTest.py
@@ -30,7 +30,7 @@
 from toil import resolveEntryPoint
 from toil.job import Job
 from toil.lib.bioio import getTempFile, system
-from toil.test import ToilTest, needs_aws, needs_rsync3, integrative
+from toil.test import ToilTest, needs_aws, needs_rsync3, integrative, slow
 from toil.test.sort.sortTest import makeFileToSort
 from toil.utils.toilStats import getStats, processData
 from toil.common import Toil, Config
@@ -87,6 +87,7 @@ def statusCommand(self, failIfNotComplete=False):
     @pytest.mark.timeout(1200)
     @needs_aws
     @integrative
+    @slow
     def testAWSProvisionerUtils(self):
         clusterName = 'cluster-utils-test' + str(uuid.uuid4())
         keyName = os.getenv('TOIL_AWS_KEYNAME')
@@ -185,6 +186,7 @@ def testAWSProvisionerUtils(self):
             except NameError:
                 pass
 
+    @slow
     def testUtilsSort(self):
         """
         Tests the status and stats commands of the toil command line utility using the
@@ -255,6 +257,7 @@ def testUtilsSort(self):
         # Check we can run 'toil clean'
         system(self.cleanCommand)
 
+    @slow
     def testUtilsStatsSort(self):
         """
         Tests the stats commands on a complete run of the stats test.
@@ -293,6 +296,7 @@ def testUnicodeSupport(self):
         options.logLevel = 'debug'
         Job.Runner.startToil(Job.wrapFn(printUnicodeCharacter), options)
 
+    @slow
     def testMultipleJobsPerWorkerStats(self):
         """
         Tests case where multiple jobs are run on 1 worker to insure that all jobs report back their data