Repository: DataBiosphere/toil

Issue:
Kubernetes batch system is missing CI tests
Someone needs to write CI tests for the Kubernetes batch system.


Discussion:

Resolving patch:
diff --git a/setup_gitlab_docker.py b/setup_gitlab_docker.py
new file mode 100644
--- /dev/null
+++ b/setup_gitlab_docker.py
@@ -0,0 +1,19 @@
+import subprocess
+import json
+
+p = subprocess.Popen('aws secretsmanager --region us-west-2 get-secret-value --secret-id /toil/gitlab/quay',
+                     stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+stdout, stderr = p.communicate()
+
+try:
+    keys = json.loads(json.loads(stdout)['SecretString'])
+    process = subprocess.Popen('docker login quay.io -u "{user}" --password-stdin'.format(user=keys['user']),
+                               stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE,
+                               shell=True)
+    stdout, stderr = process.communicate(input=keys['password'])
+    if 'Login Succeeded' in stdout:
+        print('Login Succeeded')
+    else:
+        raise RuntimeError
+except:
+    print('While attempting to log into quay.io:\n' + str(stderr))
diff --git a/src/toil/batchSystems/kubernetes.py b/src/toil/batchSystems/kubernetes.py
--- a/src/toil/batchSystems/kubernetes.py
+++ b/src/toil/batchSystems/kubernetes.py
@@ -167,7 +167,7 @@ def _api(self, kind, max_age_seconds = 5 * 60):
             # We just need the namespace string
             if config_source == 'in_cluster':
                 # Our namespace comes from a particular file.
-                with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace", 'r') as fh
+                with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace", 'r') as fh:
                     return fh.read().strip()
             else:
                 # Find all contexts and the active context.
@@ -558,7 +558,6 @@ def getUpdatedBatchJob(self, maxWait):
                                 return result
                             else:
                                 continue
-
         else:
             # Try polling instead
             while result is None and (datetime.datetime.now() - entry).total_seconds() < maxWait:
diff --git a/version_template.py b/version_template.py
--- a/version_template.py
+++ b/version_template.py
@@ -54,35 +54,21 @@ def _version(shorten=False):
 
 
 def distVersion():
-    """
-    The distribution version identifying a published release on PyPI.
-    """
+    """The distribution version identifying a published release on PyPI."""
     from pkg_resources import parse_version
-    build_number = buildNumber()
-    parsedBaseVersion = parse_version(baseVersion)
-    if isinstance(parsedBaseVersion, tuple):
+    if isinstance(parse_version(baseVersion), tuple):
         raise RuntimeError("Setuptools version 8.0 or newer required. Update by running "
                            "'pip install setuptools --upgrade'")
-
-    if build_number is not None and parsedBaseVersion.is_prerelease:
-        return baseVersion + '.dev' + build_number
-    else:
-        return baseVersion
+    return baseVersion
 
 
 def dockerTag():
-    """
-    The primary tag of the Docker image for the appliance. This uniquely identifies the appliance
-    image.
-    """
+    """The primary tag of the Docker image for the appliance. This uniquely identifies the appliance image."""
     return version()
 
 
 def dockerShortTag():
-    """
-    A secondary, shortened form of :func:`dockerTag` with which to tag the appliance image for
-    convenience.
-    """
+    """A secondary, shortened form of :func:`dockerTag` with which to tag the appliance image for convenience."""
     return shortVersion()
 
 
@@ -94,14 +80,6 @@ def dockerMinimalTag():
     return distVersion()
 
 
-def buildNumber():
-    """
-    The Jenkins build number, if defined, else None.
-    """
-    import os
-    return os.getenv('BUILD_NUMBER')
-
-
 def currentCommit():
     from subprocess import check_output
     try:
@@ -122,10 +100,9 @@ def dockerRegistry():
 def dirty():
     from subprocess import call
     try:
-        return 0 != call('(git diff --exit-code '
-                         '&& git diff --cached --exit-code) > /dev/null', shell=True)
+        return 0 != call('(git diff --exit-code && git diff --cached --exit-code) > /dev/null', shell=True)
     except:
-        return False # In case the git call fails.
+        return False  # In case the git call fails.
 
 
 def expand_(name=None):


Test patch:
diff --git a/src/toil/test/batchSystems/batchSystemTest.py b/src/toil/test/batchSystems/batchSystemTest.py
--- a/src/toil/test/batchSystems/batchSystemTest.py
+++ b/src/toil/test/batchSystems/batchSystemTest.py
@@ -135,11 +135,9 @@ def tearDown(self):
             self.batchSystem.shutdown()
             super(hidden.AbstractBatchSystemTest, self).tearDown()
         
-        @travis_test
         def testAvailableCores(self):
             self.assertTrue(multiprocessing.cpu_count() >= numCores)
         
-        @travis_test
         def testRunJobs(self):
             jobNode1 = JobNode(command='sleep 1000', jobName='test1', unitName=None,
                                jobStoreID='1', requirements=defaultRequirements)
@@ -151,7 +149,17 @@ def testRunJobs(self):
             issuedIDs = self._waitForJobsToIssue(2)
             self.assertEqual(set(issuedIDs), {job1, job2})
 
-            runningJobIDs = self._waitForJobsToStart(2)
+            # Now at some point we want these jobs to become running
+            # But since we may be testing against a live cluster (Kubernetes)
+            # we want to handle weird cases and high cluster load as much as we can.
+
+            # Wait a bit for any Dockers to download and for the
+            # jobs to have a chance to start.
+            # TODO: We insist on neither of these ever finishing when we test
+            # getUpdatedBatchJob, and the sleep time is longer than the time we
+            # should spend waiting for both to start, so if our cluster can
+            # only run one job at a time, we will fail the test.
+            runningJobIDs = self._waitForJobsToStart(2, tries=120)
             self.assertEqual(set(runningJobIDs), {job1, job2})
 
             # Killing the jobs instead of allowing them to complete means this test can run very
@@ -187,7 +195,6 @@ def testRunJobs(self):
             # Make sure killBatchJobs can handle jobs that don't exist
             self.batchSystem.killBatchJobs([10])
         
-        @travis_test
         def testSetEnv(self):
             # Parasol disobeys shell rules and stupidly splits the command at
             # the space character into arguments before exec'ing it, whether
@@ -200,7 +207,6 @@ def testSetEnv(self):
              
             # Turn into a string which convinces bash to take all args and paste them back together and run them
             command = "bash -c \"\\${@}\" bash eval " + script_protected
-            log.critical(command)
             jobNode4 = JobNode(command=command, jobName='test4', unitName=None,
                                jobStoreID='4', requirements=defaultRequirements)
             job4 = self.batchSystem.issueBatchJob(jobNode4)
@@ -216,7 +222,6 @@ def testSetEnv(self):
             self.assertEqual(exitStatus, 23)
             self.assertEqual(jobID, job5)
         
-        @travis_test
         def testCheckResourceRequest(self):
             if isinstance(self.batchSystem, BatchSystemSupport):
                 checkResourceRequest = self.batchSystem.checkResourceRequest
@@ -234,7 +239,6 @@ def testCheckResourceRequest(self):
                                   memory=10, cores=None, disk=1000)
                 checkResourceRequest(memory=10, cores=1, disk=100)
        
-        @travis_test
         def testScalableBatchSystem(self):
             # If instance of scalable batch system
             pass
@@ -248,11 +252,20 @@ def _waitForJobsToIssue(self, numJobs):
                 time.sleep(1)
             return issuedIDs
 
-        def _waitForJobsToStart(self, numJobs):
+        def _waitForJobsToStart(self, numJobs, tries=20):
+            """
+            Loop until the given number of distinct jobs are in the
+            running state, or until the given number of tries is exhausted
+            (with 1 second polling period).
+
+            Returns the list of IDs that are running.
+            """
             runningIDs = []
-            # prevent an endless loop, give it 20 tries
-            for it in range(20):
-                runningIDs = list(self.batchSystem.getRunningBatchJobIDs().keys())
+            # prevent an endless loop, give it a few tries
+            for it in range(tries):
+                running = self.batchSystem.getRunningBatchJobIDs()
+                log.info('Running jobs now: {}'.format(running))
+                runningIDs = list(running.keys())
                 if len(runningIDs) == numJobs:
                     break
                 time.sleep(1)
@@ -384,11 +397,12 @@ def testIgnoreNode(self):
         issuedID = self._waitForJobsToIssue(1)
         self.assertEqual(set(issuedID), {job})
 
-        runningJobIDs = self._waitForJobsToStart(1)
+        # Wait until a job starts or we go a while without that happening
+        runningJobIDs = self._waitForJobsToStart(1, tries=20)
         # Make sure job is NOT running
         self.assertEqual(set(runningJobIDs), set({}))
 
-
+@travis_test
 class SingleMachineBatchSystemTest(hidden.AbstractBatchSystemTest):
     """
     Tests against the single-machine batch system
@@ -744,6 +758,7 @@ def createBatchSystem(self):
     def tearDown(self):
         super(HTCondorBatchSystemTest, self).tearDown()
 
+@travis_test
 class SingleMachineBatchSystemJobTest(hidden.AbstractBatchSystemJobTest):
     """
     Tests Toil workflow against the SingleMachine batch system