Repository: DataBiosphere/toil

Issue:
Create integration tests
Front-to-back execution of a Toil workflow in AWS. Run a real-world pipeline (probably @jvivian's rnaseq pipeline from BD2KGenomics/toil-scripts) on a set of minimal, fake samples. Install cgcloud, use cgcloud to create toil-box image, start a leader, install toil-scripts on it, rely on autoscaling to spawn the instance.

Create integration tests
Front-to-back execution of a Toil workflow in AWS. Run a real-world pipeline (probably @jvivian's rnaseq pipeline from BD2KGenomics/toil-scripts) on a set of minimal, fake samples. Install cgcloud, use cgcloud to create toil-box image, start a leader, install toil-scripts on it, rely on autoscaling to spawn the instance.



Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -12,27 +12,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from version import version
+from version import version as toil_version, cgcloud_version
 from setuptools import find_packages, setup
 
 botoRequirement = 'boto==2.38.0'
 
 setup(
     name='toil',
-    version=version,
+    version=toil_version,
     description='Pipeline management software for clusters.',
     author='Benedict Paten',
     author_email='benedict@soe.usc.edu',
     url="https://github.com/BD2KGenomics/toil",
     install_requires=[
-        'bd2k-python-lib==1.14a1.dev29',
+        'bd2k-python-lib>=1.14a1.dev29',
         'dill==0.2.5'],
     extras_require={
         'mesos': [
             'psutil==3.0.1'],
         'aws': [
             botoRequirement,
-            'cgcloud-lib==1.4a1.dev195',
+            'cgcloud-lib==' + cgcloud_version,
             'futures==3.0.5'],
         'azure': [
             'azure==1.0.3'],
@@ -44,7 +44,10 @@
         'cwl': [
             'cwltool==1.0.20160714182449']},
     package_dir={'': 'src'},
-    packages=find_packages('src', exclude=['*.test']),
+    packages=find_packages(where='src',
+                           # Note that we intentionally include the top-level `test` package for
+                           # functionality like the @experimental and @integrative decoratorss:
+                           exclude=['*.test.*']),
     entry_points={
         'console_scripts': [
             'toil = toil.utils.toilMain:main',
diff --git a/src/toil/provisioners/aws/provisioner.py b/src/toil/provisioners/aws/provisioner.py
--- a/src/toil/provisioners/aws/provisioner.py
+++ b/src/toil/provisioners/aws/provisioner.py
@@ -8,7 +8,7 @@
 from bd2k.util import memoize
 from bd2k.util.iterables import concat
 from boto.ec2.instance import Instance
-from cgcloud.lib.ec2 import wait_for_spot_instances, ec2_instance_types, wait_transition
+from cgcloud.lib.ec2 import ec2_instance_types, wait_transition
 from cgcloud.lib.util import papply
 
 from toil.batchSystems.abstractBatchSystem import AbstractScalableBatchSystem, AbstractBatchSystem
diff --git a/src/toil/version.py b/src/toil/version.py
--- a/src/toil/version.py
+++ b/src/toil/version.py
@@ -13,3 +13,5 @@
 # limitations under the License.
 
 version = '3.4.0a1'
+
+cgcloud_version = '1.6.0a1.dev361'


Test patch:
diff --git a/run_tests.py b/run_tests.py
--- a/run_tests.py
+++ b/run_tests.py
@@ -22,7 +22,8 @@
     'AwsJobStoreCacheTest',
     'AzureJobStoreCacheTest',
     'FileJobStoreCacheTest',
-    'GoogleJobStoreCacheTest']
+    'GoogleJobStoreCacheTest',
+    'AWSProvisionerTest']
 
 
 def run_tests(keywords, index, args):
diff --git a/src/toil/test/__init__.py b/src/toil/test/__init__.py
--- a/src/toil/test/__init__.py
+++ b/src/toil/test/__init__.py
@@ -312,6 +312,24 @@ def experimental(test_item):
             'Set TOIL_TEST_EXPERIMENTAL="True" to include this experimental test.')(test_item)
 
 
+def integrative(test_item):
+    """
+    Use this to decorate integration tests so as to skip them during regular builds. We define
+    integration tests as A) involving other, non-Toil software components that we develop and/or
+    B) having a higher cost (time or money). Note that brittleness does not qualify a test for
+    being integrative. Neither does involvement of external services such as AWS, since that
+    would cover most of Toil's test.
+    """
+    # We'll pytest.mark_test the test as integrative but we'll also unittest.skip it via an
+    # environment variable.
+    test_item = _mark_test('integrative', test_item)
+    if less_strict_bool(os.getenv('TOIL_TEST_INTEGRATIVE')):
+        return test_item
+    else:
+        return unittest.skip(
+            'Set TOIL_TEST_INTEGRATIVE="True" to include this integration test.')(test_item)
+
+
 methodNamePartRegex = re.compile('^[a-zA-Z_0-9]+$')
 
 
diff --git a/src/toil/test/provisioners/aws/__init__.py b/src/toil/test/provisioners/aws/__init__.py
new file mode 100644
diff --git a/src/toil/test/provisioners/aws/provisionerTest.py b/src/toil/test/provisioners/aws/provisionerTest.py
new file mode 100644
--- /dev/null
+++ b/src/toil/test/provisioners/aws/provisionerTest.py
@@ -0,0 +1,143 @@
+import logging
+import os
+import pipes
+import subprocess
+from contextlib import contextmanager
+from urlparse import urlparse
+from uuid import uuid4
+
+from bd2k.util.iterables import concat
+from cgcloud.lib.test import CgcloudTestCase
+
+from toil.test import integrative, ToilTest
+from toil.version import version as toil_version, cgcloud_version
+
+log = logging.getLogger(__name__)
+
+
+@integrative
+class AWSProvisionerTest(ToilTest, CgcloudTestCase):
+    """
+    Tests Toil on a Mesos cluster in AWS provisioned by CGCloud. Uses the RNASeq integration test
+    workflow from toil-scripts.
+    """
+    # Whether to skip and log all cgcloud invocations.
+    dryRun = False
+
+    # Create new toil-box AMI before creating cluster. If False, you will almost certainly have
+    # to set CGCLOUD_NAMESPACE to run the tests in a namespace that has a pre-existing image.
+    # Also keep in mind that this image will quickly get stale, as the image will contain the
+    # version of Toil currently checked out, even if it is dirty.
+    createImage = True
+
+    # Delete toil-box AMI when done.
+    cleanup = True
+
+    # A pip-installable release of toil-scripts or a URL pointing to a source distribution.
+    # Typically you would specify a GitHub archive URL of a specific branch, tag or commit. For
+    # example, https://api.github.com/repos/BD2KGenomics/toil-scripts/tarball/master. The first
+    # path component of each tarball entry will be stripped.
+    if True:
+        toilScripts = '2.1.0a1.dev455'
+    else:
+        toilScripts = 'https://api.github.com/repos/BD2KGenomics/toil-scripts/tarball/master'
+
+    toilOptions = ['--batchSystem=mesos',
+                   '--mesosMaster=mesos-master:5050',
+                   '--logDebug',
+                   '--clean=always']
+
+    region = 'us-west-2'
+
+    @classmethod
+    def setUpClass(cls):
+        logging.basicConfig(level=logging.INFO)
+        super(AWSProvisionerTest, cls).setUpClass()
+
+    def setUp(self):
+        super(AWSProvisionerTest, self).setUp()
+        self.saved_cgcloud_plugins = os.environ.get('CGCLOUD_PLUGINS')
+        os.environ['CGCLOUD_PLUGINS'] = 'cgcloud.toil'
+        self.assertTrue(os.path.isfile(self.sdistPath),
+                        'Cannot find source distribution of Toil (%s)' % self.sdistPath)
+        self.jobStore = 'aws:%s:toil-it-%s' % (self.region, uuid4())
+
+    def tearDown(self):
+        if self.saved_cgcloud_plugins is not None:
+            os.environ['CGCLOUD_PLUGINS'] = self.saved_cgcloud_plugins
+        super(AWSProvisionerTest, self).tearDown()
+
+    @integrative
+    def test(self):
+        with self.cgcloudVenv():
+            self._run('pip', 'install', 'cgcloud-toil==' + cgcloud_version)
+            if self.createImage:
+                self._cgcloud('create', '-IT',
+                              '--option', 'toil_sdists=%s[aws,mesos]' % self.sdistPath,
+                              'toil-latest-box')
+            try:
+                self._cgcloud('create-cluster',
+                              '--leader-instance-type=m3.large',
+                              '--instance-type=m3.large',
+                              '--num-workers=10',
+                              'toil')
+                try:
+                    self._leader('virtualenv', '--system-site-packages', '~/venv')
+                    toilScripts = urlparse(self.toilScripts)
+                    if toilScripts.netloc:
+                        self._leader('mkdir toil-scripts'
+                                     '; curl -L ' + toilScripts.geturl() +
+                                     '| tar -C toil-scripts -xvz --strip-components=1')
+                        self._leader('PATH=~/venv/bin:$PATH make -C toil-scripts develop')
+                    else:
+                        version = toilScripts.path
+                        self._leader('~/venv/bin/pip', 'install', 'toil-scripts==' + version)
+                    toilOptions = ' '.join(self.toilOptions)
+                    self._leader('PATH=~/venv/bin:$PATH',
+                                 'TOIL_SCRIPTS_TEST_NUM_SAMPLES=10',
+                                 'TOIL_SCRIPTS_TEST_TOIL_OPTIONS=' + pipes.quote(toilOptions),
+                                 'TOIL_SCRIPTS_TEST_JOBSTORE=' + self.jobStore,
+                                 'python', '-m', 'unittest', '-v',
+                                 'toil_scripts.rnaseq_cgl.test.test_rnaseq_cgl.RNASeqCGLTest'
+                                 '.test_manifest')
+                finally:
+                    self._cgcloud('terminate-cluster', 'toil')
+            finally:
+                if self.cleanup:
+                    self._cgcloud('delete-image', 'toil-latest-box')
+
+    @contextmanager
+    def cgcloudVenv(self):
+        path = self._createTempDir(purpose='cgcloud-venv')
+        self._run('virtualenv', path)
+        binPath = os.path.join(path, 'bin')
+        path = os.environ['PATH']
+        os.environ['PATH'] = os.pathsep.join(concat(binPath, path.split(os.pathsep)))
+        try:
+            yield
+        finally:
+            os.environ['PATH'] = path
+
+    @property
+    def sdistPath(self):
+        return os.path.join(self._projectRootPath(), 'dist', 'toil-%s.tar.gz' % toil_version)
+
+    def _run(self, *args, **kwargs):
+        log.info('Running %r', args)
+        if kwargs.get('capture', False):
+            return subprocess.check_output(args)
+        else:
+            subprocess.check_call(args)
+            return None
+
+    def _cgcloud(self, *args):
+        if not self.dryRun:
+            self._run('cgcloud', *args)
+
+    def _ssh(self, role, *args):
+        self._cgcloud('ssh', role,
+                      '-o', 'UserKnownHostsFile=/dev/null',
+                      '-o', 'StrictHostKeyChecking=no', *args)
+
+    def _leader(self, *args):
+        self._ssh('toil-leader', *args)