Repository: DataBiosphere/toil

Issue:
Tests run on Kubernetes still have wrong thread counts
In our Kubernetes Gitlab runner, where the containers have a CPU limit of 4000m (i.e. 4 cores), we get test vailures like this (https://ucsc-ci.com/databiosphere/toil/-/jobs/36110):

```
_________ SingleMachinePromisedRequirementsTest.testConcurrencyDynamic _________
Traceback (most recent call last):
  File "/builds/databiosphere/toil/src/toil/test/src/promisedRequirementTest.py", line 58, in testConcurrencyDynamic
    self.assertEqual(maxValue, old_div(self.cpuCount, coresPerJob))
  File "/usr/lib/python2.7/unittest/case.py", line 513, in assertEqual
    assertion_func(first, second, msg=msg)
  File "/usr/lib/python2.7/unittest/case.py", line 506, in _baseAssertEqual
    raise self.failureException(msg)
AssertionError: 13 != 96

_________ SingleMachinePromisedRequirementsTest.testConcurrencyStatic __________
Traceback (most recent call last):
  File "/builds/databiosphere/toil/src/toil/test/src/promisedRequirementTest.py", line 81, in testConcurrencyStatic
    self.assertEqual(maxValue, old_div(self.cpuCount, coresPerJob))
  File "/usr/lib/python2.7/unittest/case.py", line 513, in assertEqual
    assertion_func(first, second, msg=msg)
  File "/usr/lib/python2.7/unittest/case.py", line 506, in _baseAssertEqual
    raise self.failureException(msg)
AssertionError: 13 != 96
```

Somehow it is still seeing the 96 cores of the physical machine, and also somehow it is doing 13 something with a 4000m CPU limit.

See also: https://github.com/DataBiosphere/toil/pull/2903#issuecomment-570728718 where I complained about this before. I thought I might have fixed it, but apparently I didn't.

┆Issue is synchronized with this [Jira Task](https://ucsc-cgl.atlassian.net/browse/TOIL-495)
┆Issue Number: TOIL-495



Discussion:

Resolving patch:
diff --git a/src/toil/batchSystems/kubernetes.py b/src/toil/batchSystems/kubernetes.py
--- a/src/toil/batchSystems/kubernetes.py
+++ b/src/toil/batchSystems/kubernetes.py
@@ -568,8 +568,15 @@ def getUpdatedBatchJob(self, maxWait):
                                 if containerStatuses is None or len(containerStatuses) == 0: 
                                     logger.debug("No job container statuses for job %s" % (pod.metadata.owner_references[0].name))
                                     return (int(pod.metadata.owner_references[0].name[len(self.jobPrefix):]), -1, 0)
-                                logger.info("REASON: %s Exit Code: %s" % (pod.status.container_statuses[0].state.terminated.reason,
-                                    pod.status.container_statuses[0].state.terminated.exit_code))
+
+                                # Get termination onformation from the pod
+                                termination = pod.status.container_statuses[0].state.terminated
+                                logger.info("REASON: %s Exit Code: %s", termination.reason, termination.exit_code)
+                                
+                                if termination.exit_code != 0:
+                                    # The pod failed. Dump information about it.
+                                    logger.debug('Failed pod information: %s', str(pod))
+                                    logger.warning('Log from failed pod: %s', self._getLogForPod(pod))
                                 jobID = int(pod.metadata.owner_references[0].name[len(self.jobPrefix):])
                                 terminated = pod.status.container_statuses[0].state.terminated
                                 runtime = slow_down((terminated.finished_at - terminated.started_at).total_seconds())
diff --git a/src/toil/lib/threading.py b/src/toil/lib/threading.py
--- a/src/toil/lib/threading.py
+++ b/src/toil/lib/threading.py
@@ -17,9 +17,11 @@
 from __future__ import absolute_import
 from future.utils import raise_
 from builtins import range
+import logging
 import math
 import sys
 import threading
+import traceback
 if sys.version_info >= (3, 0):
     from threading import BoundedSemaphore
 else:
@@ -27,6 +29,8 @@
 
 import psutil
 
+log = logging.getLogger(__name__)
+
 class BoundedEmptySemaphore( BoundedSemaphore ):
     """
     A bounded semaphore that is initially empty.
@@ -118,17 +122,28 @@ def cpu_count():
     Ignores the cgroup's cpu shares value, because it's extremely difficult to
     interpret. See https://github.com/kubernetes/kubernetes/issues/81021.
 
+    Caches result for efficiency.
+
     :return: Integer count of available CPUs, minimum 1.
     :rtype: int
     """
 
+    cached = getattr(cpu_count, 'result', None)
+    if cached is not None:
+        # We already got a CPU count.    
+        return cached
+
     # Get the fallback answer of all the CPUs on the machine
     total_machine_size = psutil.cpu_count(logical=True)
 
+    log.debug('Total machine size: %d cores', total_machine_size) 
+
     try:
         with open('/sys/fs/cgroup/cpu/cpu.cfs_quota_us', 'r') as stream:
             # Read the quota
-            quota = int(stream.read)
+            quota = int(stream.read())
+
+        log.debug('CPU quota: %d', quota)
 
         if quota == -1:
             # Assume we can use the whole machine
@@ -136,14 +151,24 @@ def cpu_count():
 
         with open('/sys/fs/cgroup/cpu/cpu.cfs_period_us', 'r') as stream:
             # Read the period in which we are allowed to burn the quota
-            period = int(stream.read)
+            period = int(stream.read())
+
+        log.debug('CPU quota period: %d', period)
 
         # The thread count is how many multiples of a wall clcok period we can burn in that period.
         cgroup_size = int(math.ceil(float(quota)/float(period)))
+
+        log.debug('Cgroup size in cores: %d', cgroup_size)
+
     except:
         # We can't actually read these cgroup fields. Maybe we are a mac or something.
+        log.debug('Could not inspect cgroup: %s', traceback.format_exc())
         cgroup_size = float('inf')
 
     # Return the smaller of the actual thread count and the cgroup's limit, minimum 1.
-    return max(1, min(cgroup_size, total_machine_size))
+    result = max(1, min(cgroup_size, total_machine_size))
+    log.debug('cpu_count: %s', str(result))
+    # Make sure to remember it for the next call
+    setattr(cpu_count, 'result', result)
+    return result
     


Test patch:
diff --git a/src/toil/test/__init__.py b/src/toil/test/__init__.py
--- a/src/toil/test/__init__.py
+++ b/src/toil/test/__init__.py
@@ -19,6 +19,7 @@
 import re
 import shutil
 import signal
+import sys
 import tempfile
 import threading
 import time
@@ -399,7 +400,6 @@ def needs_docker(test_item):
     else:
         return unittest.skip("Install docker to include this test.")(test_item)
 
-
 def needs_encryption(test_item):
     """
     Use as a decorator before test classes or methods to only run them if PyNaCl is installed
@@ -432,6 +432,10 @@ def needs_cwl(test_item):
 
 
 def needs_appliance(test_item):
+    """
+    Use as a decorator before test classes or methods to only run them if
+    the Toil appliance Docker image is downloaded.
+    """
     import json
     test_item = _mark_test('appliance', test_item)
     if os.getenv('TOIL_SKIP_DOCKER', '').lower() == 'true':
@@ -454,6 +458,23 @@ def needs_appliance(test_item):
             assert False, 'Expected `docker inspect` to return zero or one image.'
     else:
         return unittest.skip('Install Docker to include this test.')(test_item)
+    
+def needs_downloadable_appliance(test_item):
+    """
+    Use as a decorator before test classes or methods to only run them if
+    the Toil appliance Docker image ought to be available for download.
+
+    For now, this just skips if running on Python 3.
+    TODO: When the appliance build is set up for Python 3 (when
+    https://github.com/DataBiosphere/toil/issues/2742 is fixed), this behavior
+    should be changed.
+    """
+
+    if sys.version_info[0] == 2:
+        return test_item
+    else:
+        return unittest.skip("Skipping test that needs Toil Appliance, available only for Python 2")(test_item)
+
 
 
 def integrative(test_item):
diff --git a/src/toil/test/batchSystems/batchSystemTest.py b/src/toil/test/batchSystems/batchSystemTest.py
--- a/src/toil/test/batchSystems/batchSystemTest.py
+++ b/src/toil/test/batchSystems/batchSystemTest.py
@@ -55,6 +55,7 @@
                        needs_slurm,
                        needs_torque,
                        needs_htcondor,
+                       needs_downloadable_appliance,
                        slow,
                        tempFileContaining,
                        travis_test)
@@ -345,6 +346,7 @@ def _createConfig(self):
 
 @needs_kubernetes
 @needs_aws_s3
+@needs_downloadable_appliance
 class KubernetesBatchSystemTest(hidden.AbstractBatchSystemTest):
     """
     Tests against the Kubernetes batch system
diff --git a/src/toil/test/docs/scriptsTest.py b/src/toil/test/docs/scriptsTest.py
--- a/src/toil/test/docs/scriptsTest.py
+++ b/src/toil/test/docs/scriptsTest.py
@@ -51,7 +51,7 @@ def checkExpectedOut(self, script, expectedOutput):
 
         # Check that the expected output is there
         index = outerr.find(expectedOutput)
-        self.assertGreater(index, -1, index)
+        self.assertGreater(index, -1, "Expected:\n{}\nOutput:\n{}".format(expectedOutput, outerr))
 
     """Check the exit code and look for a pattern"""
     def checkExpectedPattern(self, script, expectedPattern):
@@ -60,7 +60,7 @@ def checkExpectedPattern(self, script, expectedPattern):
         # Check that the expected output pattern is there
         pattern = re.compile(expectedPattern, re.DOTALL)
         n = re.search(pattern, outerr)
-        self.assertNotEqual(n, None, n)
+        self.assertNotEqual(n, None, "Pattern:\n{}\nOutput:\n{}".format(expectedPattern, outerr))
 
     @needs_cwl
     def testCwlexample(self):
diff --git a/src/toil/test/src/promisedRequirementTest.py b/src/toil/test/src/promisedRequirementTest.py
--- a/src/toil/test/src/promisedRequirementTest.py
+++ b/src/toil/test/src/promisedRequirementTest.py
@@ -48,6 +48,7 @@ def testConcurrencyDynamic(self):
             Asserts that promised core resources are allocated properly using a dynamic Toil workflow
             """
             for coresPerJob in self.allocatedCores:
+                log.debug('Testing %d cores per job with CPU count %d', coresPerJob, self.cpuCount)
                 tempDir = self._createTempDir('testFiles')
                 counterPath = self.getCounterPath(tempDir)
 
@@ -63,6 +64,7 @@ def testConcurrencyStatic(self):
             Asserts that promised core resources are allocated properly using a static DAG
             """
             for coresPerJob in self.allocatedCores:
+                log.debug('Testing %d cores per job with CPU count %d', coresPerJob, self.cpuCount)
                 tempDir = self._createTempDir('testFiles')
                 counterPath = self.getCounterPath(tempDir)