Repository: common-workflow-language/cwltest (language: python)

Issue:
Collect output from timed-out tests
This should fix https://github.com/common-workflow-language/cwltest/issues/121
Losing JUnit output for timed-out tests
I tried using `--junit-xml`, and while it does keep the tests output straight from each other, one of my tests that timed out got no output stored in the XML; there's no `system-out` or `system-err` elements like for other tests.

This is because we're using `communicate()` with a timeout, but not doing the [special dance from the docs](https://docs.python.org/3/library/subprocess.html#subprocess.Popen.communicate) to actually get the output from the process if the timeout expires.

_Originally posted by @adamnovak in https://github.com/common-workflow-language/cwltest/issues/120#issuecomment-969028666_


Discussion / hints:

Fix patch:
diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 0000000..4a89de0
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,14 @@
+[run]
+branch = True
+source = cwltest
+omit = cwltest/tests/*
+
+[report]
+exclude_lines =
+    if self.debug:
+    pragma: no cover
+    raise NotImplementedError
+    if __name__ == .__main__.:
+ignore_errors = True
+omit =
+    cwltest/tests/*
diff --git a/.flake8 b/.flake8
new file mode 100644
index 0000000..35967ff
--- /dev/null
+++ b/.flake8
@@ -0,0 +1,4 @@
+[flake8]
+ignore = E203, E266, E501, W503, E211, E731
+max-line-length = 88
+select = B,C,E,F,W,T4,B9
diff --git a/MANIFEST.in b/MANIFEST.in
index 6e775fd..74643c8 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,4 +1,4 @@
-include Makefile gittaggers.py test-requirements.txt mypy_requirements.txt requirements.txt
+include Makefile gittaggers.py test-requirements.txt mypy-requirements.txt requirements.txt
 include cwltest/cwltest-schema.yml
 include cwltest/tests/*
 include cwltest/tests/test-data/*
diff --git a/Makefile b/Makefile
index e67f0e1..ea7110e 100644
--- a/Makefile
+++ b/Makefile
@@ -28,7 +28,7 @@ PACKAGE=cwltest
 PYSOURCES=$(wildcard ${MODULE}/**.py tests/*.py) setup.py
 DEVPKGS=diff_cover black pylint pep257 pydocstyle flake8 tox tox-pyenv \
 	isort wheel autoflake flake8-bugbear pyupgrade bandit \
-	-rtest-requirements.txt -rmypy_requirements.txt
+	-rtest-requirements.txt -rmypy-requirements.txt
 DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pydocstyle sloccount \
 	   python-flake8 python-mock shellcheck
 VERSION=2.2.$(shell TZ=UTC git log --first-parent --max-count=1 \
@@ -112,7 +112,7 @@ diff_pylint_report: pylint_report.txt
 	diff-quality --violations=pylint pylint_report.txt
 
 .coverage: $(PYSOURCES) all
-	python setup.py test --addopts "--cov --cov-config=.coveragerc --cov-report= -n auto ${PYTEST_EXTRA}"
+	python setup.py test --addopts "--cov --cov-config=.coveragerc --cov-report= ${PYTEST_EXTRA}"
 
 coverage.xml: .coverage
 	coverage xml
diff --git a/mypy_requirements.txt b/mypy-requirements.txt
similarity index 100%
rename from mypy_requirements.txt
rename to mypy-requirements.txt
diff --git a/setup.py b/setup.py
index bb098fb..9c1d31b 100644
--- a/setup.py
+++ b/setup.py
@@ -61,6 +61,7 @@
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
         "Typing :: Typed",
     ],
 )
diff --git a/tox.ini b/tox.ini
index c0f3f1a..82b219b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,18 +1,17 @@
 [tox]
 envlist =
-  py{36,37,38,39}-lint,
-  py{36,37,38,39}-unit,
-  py{36,37,38,39}-bandit,
-  py{36,37,38,39}-mypy,
+  py{36,37,38,39,310}-lint,
+  py{36,37,38,39,310}-unit,
+  py{36,37,38,39,310}-bandit,
+  py{36,37,38,39,310}-mypy,
   py39-pipconflictchecker,
-  py39-lint-readme,
+  py39-lintreadme,
   py39-pydocstyle
 
-skipsdist = True
 skip_missing_interpreters = True
 
 [pytest]
-addopts=-n auto --pyargs cwltest
+addopts = --pyargs cwltest
 testpaths = cwltest/tests
 
 [gh-actions]
@@ -21,42 +20,48 @@ python =
   3.7: py37
   3.8: py38
   3.9: py39
+  3.10: py310
 
 [testenv]
 description =
-  py{36,37,38,39}-unit: Run the unit tests
-  py{36,37,38,39}-lint: Lint the Python code
-  py{36,37,38,39}-bandit: Search for common security issues
-  py{36,37,38,39}-mypy: Check for type safety
+  py{36,37,38,39,310}-unit: Run the unit tests
+  py{36,37,38,39,310}-lint: Lint the Python code
+  py{36,37,38,39,310}-bandit: Search for common security issues
+  py{36,37,38,39,310}-mypy: Check for type safety
   py39-pydocstyle: docstring style checker
-  py39-lint-readme: Lint the README.rst->.md conversion
+  py39-lintreadme: Lint the README.rst->.md conversion
 
 passenv =
   CI
   GITHUB_*
 deps =
-  py{36,37,38,39}-{unit,lint,bandit,mypy}: -rrequirements.txt
-  py{36,37,38,39}-{unit,mypy}: -rtest-requirements.txt
-  py{36,37,38,39}-lint: flake8-bugbear
-  py{36,37,38,39}-lint: black
-  py{36,37,38,39}-bandit: bandit
-  py{36,37,38,39}-mypy: -rmypy_requirements.txt
+  py{36,37,38,39,310}-{unit,mypy}: -rrequirements.txt
+  py{36,37,38,39,310}-{unit,mypy}: -rtest-requirements.txt
+  py{36,37,38,39,310}-lint: flake8-bugbear
+  py{36,37,38,39,310}-lint: black
+  py{36,37,38,39,310}-bandit: bandit
+  py{36,37,38,39,310}-mypy: -rmypy-requirements.txt
 
 setenv =
-  py{36,37,38,39}-unit: LC_ALL = C.UTF-8
+  py{36,37,38,39,310}-unit: LC_ALL = C.UTF-8
 
 commands =
-  py{36,37,38,39}-unit: python -m pip install -U pip setuptools wheel
-  py{36,37,38,39}-unit: make coverage-report coverage.xml PYTEST_EXTRA={posargs}
-  py{36,37,38,39}-bandit: bandit --recursive cwltest
-  py{36,37,38,39}-lint: make flake8
-  py{36,37,38,39}-lint: make format-check
-  py{36,37,38,39}-mypy: make mypy
+  py{36,37,38,39,310}-unit: python -m pip install -U pip setuptools wheel
+  py{36,37,38,39,310}-unit: make coverage-report coverage.xml PYTEST_EXTRA={posargs}
+  py{36,37,38,39,310}-bandit: bandit --recursive cwltest --exclude cwltest/tests/*
+  py{36,37,38,39,310}-lint: make flake8
+  py{36,37,38,39,310}-lint: make format-check
+  py{36,37,38,39,310}-mypy: make mypy
 
 whitelist_externals =
-  py{36,37,38,39}-lint: flake8
-  py{36,37,38,39}-lint: black
-  py{36,37,38,39}-{mypy,shellcheck,lint,unit}: make
+  py{36,37,38,39,310}-lint: flake8
+  py{36,37,38,39,310}-lint: black
+  py{36,37,38,39,310}-{mypy,shellcheck,lint,unit}: make
+
+skip_install =
+  py{36,37,38,39,310}-lint: true
+  py{36,37,38,39,310}-bandit: true
+
 
 [testenv:py39-pydocstyle]
 whitelist_externals = make
@@ -64,6 +69,7 @@ commands = make diff_pydocstyle_report
 deps =
     pydocstyle
     diff-cover
+skip_install = true
 
 [testenv:py39-pipconflictchecker]
 commands = pipconflictchecker
@@ -72,7 +78,8 @@ deps =
   pip-conflict-checker
   pip==9.0.3
 
-[testenv:py39-lint-readme]
+[testenv:py39-lintreadme]
+description = Lint the README.rst->.md conversion
 commands =
   python setup.py sdist
   python setup.py bdist_wheel
@@ -81,3 +88,4 @@ deps =
     twine
     wheel
     readme_renderer[md]
+skip_install = true


Test patch:
diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml
index cf34d1d..573fb61 100644
--- a/.github/workflows/ci-tests.yml
+++ b/.github/workflows/ci-tests.yml
@@ -6,6 +6,10 @@ on:
   pull_request:
     branches: [ main ]
 
+concurrency:
+  group: build-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
 
   tox:
@@ -16,7 +20,7 @@ jobs:
     strategy:
       matrix:
         py-ver-major: [3]
-        py-ver-minor: [6, 7, 8, 9]
+        py-ver-minor: [6, 7, 8, 9, 10]
         step: [lint, unit, mypy]
 
     env:
@@ -54,7 +58,7 @@ jobs:
 
       - name: Upload coverage to Codecov
         if: ${{ matrix.step == 'unit' }}
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v2.1.0
         with:
           fail_ci_if_error: true
 
@@ -65,11 +69,11 @@ jobs:
 
     strategy:
       matrix:
-        step: [lint-readme, pydocstyle]
+        step: [lintreadme, pydocstyle]
 
     env:
-      py-semver: 3.9
-      TOXENV: ${{ format('py39-{0}', matrix.step) }}
+      py-semver: "3.10"
+      TOXENV: ${{ format('py310-{0}', matrix.step) }}
 
     steps:
       - uses: actions/checkout@v2
@@ -110,7 +114,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.9
+          python-version: "3.10"  # quoted, otherwise that turns into the number 3.1
 
       - name: Cache for pip
         uses: actions/cache@v2
diff --git a/cwltest/__init__.py b/cwltest/__init__.py
index 577a19f..9e0ec7a 100755
--- a/cwltest/__init__.py
+++ b/cwltest/__init__.py
@@ -204,6 +204,11 @@ def run_test(
             " ".join([quote(tc) for tc in test_command]),
         )
         _logger.error(test.get("doc"))
+        # Kill and re-communicate to get the logs and reap the child, as
+        # instructed in the subprocess docs.
+        if process:
+            process.kill()
+            outstr, outerr = process.communicate()
         return TestResult(2, outstr, outerr, timeout, args.classname, "Test timed out")
     finally:
         if process is not None and process.returncode is None:
diff --git a/cwltest/tests/mock_cwl_runner.py b/cwltest/tests/mock_cwl_runner.py
index a352f7d..478e504 100644
--- a/cwltest/tests/mock_cwl_runner.py
+++ b/cwltest/tests/mock_cwl_runner.py
@@ -1,12 +1,15 @@
 import argparse
+import sys
+from time import sleep
 
 from cwltest import UNSUPPORTED_FEATURE
 
 UNSUPPORTED_FEATURE_TOOL = "return-unsupported.cwl"
 ERROR_TOOL = "return-1.cwl"
+TIMEOUT_TOOL = "timeout.cwl"
 
 
-def main():  # type: ()->int
+def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("processfile")
     parser.add_argument("jobfile")
@@ -20,6 +23,11 @@ def main():  # type: ()->int
         exit(UNSUPPORTED_FEATURE)
     elif args.processfile.endswith(ERROR_TOOL):
         exit(1)
+    elif args.processfile.endswith(TIMEOUT_TOOL):
+        print("timeout stderr", file=sys.stderr)
+        sys.stderr.flush()
+        sleep(100000)
+        exit(1)
 
     exit(0)
 
diff --git a/cwltest/tests/test-data/timeout.cwl b/cwltest/tests/test-data/timeout.cwl
new file mode 100644
index 0000000..a487298
--- /dev/null
+++ b/cwltest/tests/test-data/timeout.cwl
@@ -0,0 +1,5 @@
+class: CommandLineTool
+cwlVersion: v1.0
+inputs: []
+outputs: []
+baseCommand: [sleep, "15"]
diff --git a/cwltest/tests/test-data/timeout.yml b/cwltest/tests/test-data/timeout.yml
new file mode 100644
index 0000000..30231d6
--- /dev/null
+++ b/cwltest/tests/test-data/timeout.yml
@@ -0,0 +1,9 @@
+- job: v1.0/empty.json
+  output:
+    output_txt:
+      class: File
+      checksum: sha1$47a013e660d408619d894b20806b1d5086aab03b
+      location: output.txt
+      size: 13
+  tool: timeout.cwl
+  doc: Test of timeout stdout and stderr capture
diff --git a/cwltest/tests/test-data/v1.0/empty.json b/cwltest/tests/test-data/v1.0/empty.json
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/cwltest/tests/test-data/v1.0/empty.json
@@ -0,0 +1,1 @@
+{}
diff --git a/cwltest/tests/test_argparse.py b/cwltest/tests/test_argparse.py
index 6a5c067..729cd5c 100644
--- a/cwltest/tests/test_argparse.py
+++ b/cwltest/tests/test_argparse.py
@@ -1,20 +1,13 @@
-import unittest
 from cwltest import arg_parser
 
 
-class TestArgparse(unittest.TestCase):
-    def setUp(self):
-        self.parser = arg_parser()
-
-    def test_arg(self):
-        parsed = self.parser.parse_args(
-            ["--test", "test_name", "-n", "52", "--tool", "cwltool", "-j", "4"]
-        )
-        self.assertEqual(parsed.test, "test_name")
-        self.assertEqual(parsed.n, "52")
-        self.assertEqual(parsed.tool, "cwltool")
-        self.assertEqual(parsed.j, 4)
-
-
-if __name__ == "__main__":
-    unittest.main()
+def test_arg():
+    """Basic test of the argparse."""
+    parser = arg_parser()
+    parsed = parser.parse_args(
+        ["--test", "test_name", "-n", "52", "--tool", "cwltool", "-j", "4"]
+    )
+    assert parsed.test == "test_name"
+    assert parsed.n == "52"
+    assert parsed.tool == "cwltool"
+    assert parsed.j == 4
diff --git a/cwltest/tests/test_categories.py b/cwltest/tests/test_categories.py
index d1e1de2..77e8711 100644
--- a/cwltest/tests/test_categories.py
+++ b/cwltest/tests/test_categories.py
@@ -1,97 +1,92 @@
-import unittest
 import re
 import os
 from os import linesep as n
 from os import sep as p
+from pathlib import Path
 
 from .util import run_with_mock_cwl_runner, get_data
 import defusedxml.ElementTree as ET
 import schema_salad.ref_resolver
 
 
-class TestCategories(unittest.TestCase):
-    maxDiff = None
+def test_unsupported_with_required_tests():
+    args = [
+        "--test",
+        schema_salad.ref_resolver.file_uri(
+            get_data("tests/test-data/required-unsupported.yml")
+        ),
+    ]
+    try:
+        cwd = os.getcwd()
+        os.chdir(get_data("tests/test-data/"))
+        error_code, stdout, stderr = run_with_mock_cwl_runner(args)
+    finally:
+        os.chdir(cwd)
 
-    def test_unsupported_with_required_tests(self):
-        args = [
-            "--test",
-            schema_salad.ref_resolver.file_uri(
-                get_data("tests/test-data/required-unsupported.yml")
-            ),
-        ]
-        try:
-            cwd = os.getcwd()
-            os.chdir(get_data("tests/test-data/"))
-            error_code, stdout, stderr = run_with_mock_cwl_runner(args)
-        finally:
-            os.chdir(cwd)
+    assert error_code == 1
+    print(stderr)
+    stderr = re.sub(r" '?--outdir=[^ ]*", "", stderr)
+    if os.name == "nt":
+        q = "'"
+    else:
+        q = ""
+    assert (
+        "Test [1/2] Required test that is unsupported (without tags){n}"
+        "{n}"
+        "Test 1 failed: mock-cwl-runner --quiet return-unsupported.cwl {q}v1.0{p}cat-job.json{q}{n}"
+        "Required test that is unsupported (without tags){n}"
+        "Does not support required feature{n}"
+        "{n}"
+        "Test [2/2] Required test that is unsupported (with tags){n}"
+        "{n}"
+        "Test 2 failed: mock-cwl-runner --quiet return-unsupported.cwl {q}v1.0{p}cat-job.json{q}{n}"
+        "Required test that is unsupported (with tags){n}"
+        "Does not support required feature{n}"
+        "{n}"
+        "0 tests passed, 2 failures, 0 unsupported features{n}".format(n=n, p=p, q=q)
+    ) == stderr
 
-        self.assertEqual(error_code, 1)
-        print(stderr)
-        stderr = re.sub(r" '?--outdir=[^ ]*", "", stderr)
-        if os.name == "nt":
-            q = "'"
-        else:
-            q = ""
-        self.assertEqual(
-            "Test [1/2] Required test that is unsupported (without tags){n}"
-            "{n}"
-            "Test 1 failed: mock-cwl-runner --quiet return-unsupported.cwl {q}v1.0{p}cat-job.json{q}{n}"
-            "Required test that is unsupported (without tags){n}"
-            "Does not support required feature{n}"
-            "{n}"
-            "Test [2/2] Required test that is unsupported (with tags){n}"
-            "{n}"
-            "Test 2 failed: mock-cwl-runner --quiet return-unsupported.cwl {q}v1.0{p}cat-job.json{q}{n}"
-            "Required test that is unsupported (with tags){n}"
-            "Does not support required feature{n}"
-            "{n}"
-            "0 tests passed, 2 failures, 0 unsupported features{n}".format(
-                n=n, p=p, q=q
-            ),
-            stderr,
-        )
 
-    def test_unsupported_with_optional_tests(self):
-        args = [
-            "--test",
-            schema_salad.ref_resolver.file_uri(
-                get_data("tests/test-data/optional-unsupported.yml")
-            ),
-        ]
-        error_code, stdout, stderr = run_with_mock_cwl_runner(args)
-        self.assertEqual(error_code, 0)
-        self.assertEqual(
-            "Test [1/1] Optional test that is unsupported{n}{n}"
-            "0 tests passed, 1 unsupported "
-            "features{n}".format(n=n),
-            stderr,
-        )
+def test_unsupported_with_optional_tests():
+    args = [
+        "--test",
+        schema_salad.ref_resolver.file_uri(
+            get_data("tests/test-data/optional-unsupported.yml")
+        ),
+    ]
+    error_code, stdout, stderr = run_with_mock_cwl_runner(args)
+    assert error_code == 0
+    assert (
+        "Test [1/1] Optional test that is unsupported{n}{n}"
+        "0 tests passed, 1 unsupported "
+        "features{n}".format(n=n)
+    ) == stderr
+
+
+def test_error_with_optional_tests():
+    args = [
+        "--test",
+        schema_salad.ref_resolver.file_uri(
+            get_data("tests/test-data/optional-error.yml")
+        ),
+    ]
+    error_code, stdout, stderr = run_with_mock_cwl_runner(args)
+    assert error_code == 1
+    assert "1 failures" in stderr
 
-    def test_error_with_optional_tests(self):
-        args = [
-            "--test",
-            schema_salad.ref_resolver.file_uri(
-                get_data("tests/test-data/optional-error.yml")
-            ),
-        ]
-        error_code, stdout, stderr = run_with_mock_cwl_runner(args)
-        self.assertEqual(error_code, 1)
-        self.assertIn("1 failures", stderr)
 
-    def test_category_in_junit_xml(self):
-        junit_xml_report = get_data("tests/test-data/junit-report.xml")
-        args = [
-            "--test",
-            schema_salad.ref_resolver.file_uri(
-                get_data("tests/test-data/optional-error.yml")
-            ),
-            "--junit-xml",
-            junit_xml_report,
-        ]
-        run_with_mock_cwl_runner(args)
-        tree = ET.parse(junit_xml_report)
-        root = tree.getroot()
-        category = root.find("testsuite").find("testcase").attrib["class"]
-        self.assertEqual(category, "js, init_work_dir")
-        os.remove(junit_xml_report)
+def test_category_in_junit_xml(tmp_path: Path):
+    junit_xml_report = tmp_path / "junit-report.xml"
+    args = [
+        "--test",
+        schema_salad.ref_resolver.file_uri(
+            get_data("tests/test-data/optional-error.yml")
+        ),
+        "--junit-xml",
+        str(junit_xml_report),
+    ]
+    run_with_mock_cwl_runner(args)
+    tree = ET.parse(junit_xml_report)
+    root = tree.getroot()
+    category = root.find("testsuite").find("testcase").attrib["class"]
+    assert category == "js, init_work_dir"
diff --git a/cwltest/tests/test_compare.py b/cwltest/tests/test_compare.py
index 5a20cae..15003ed 100644
--- a/cwltest/tests/test_compare.py
+++ b/cwltest/tests/test_compare.py
@@ -1,64 +1,63 @@
-import unittest
 from .util import get_data
 from cwltest import CompareFail
 from cwltest.utils import compare_file, compare
 
+import pytest
 
-class TestCompareFile(unittest.TestCase):
-    def test_compare_file(self):
-        expected = {
-            "location": "cores.txt",
-            "size": 2,
-            "class": "File",
-            "checksum": "sha1$7448d8798a4380162d4b56f9b452e2f6f9e24e7a",
-        }
 
-        actual = {
-            "basename": "cores.txt",
-            "checksum": "sha1$7448d8798a4380162d4b56f9b452e2f6f9e24e7a",
-            "class": "File",
-            "location": "file:///var/folders/8x/2df05_7j20j6r8y81w4qf43r0000gn/T/tmpG0EkrS/cores.txt",
-            "path": "/var/folders/8x/2df05_7j20j6r8y81w4qf43r0000gn/T/tmpG0EkrS/cores.txt",
-            "size": 2,
-        }
-        try:
-            compare_file(expected, actual)
-        except CompareFail:
-            self.fail("File comparison failed unexpectedly")
+def test_compare_file():
+    expected = {
+        "location": "cores.txt",
+        "size": 2,
+        "class": "File",
+        "checksum": "sha1$7448d8798a4380162d4b56f9b452e2f6f9e24e7a",
+    }
 
-    def test_compare_contents_success(self):
-        expected = {
-            "location": "cores.txt",
-            "size": 2,
-            "class": "File",
-            "contents": "2\n",
-        }
+    actual = {
+        "basename": "cores.txt",
+        "checksum": "sha1$7448d8798a4380162d4b56f9b452e2f6f9e24e7a",
+        "class": "File",
+        "location": "file:///var/folders/8x/2df05_7j20j6r8y81w4qf43r0000gn/T/tmpG0EkrS/cores.txt",
+        "path": "/var/folders/8x/2df05_7j20j6r8y81w4qf43r0000gn/T/tmpG0EkrS/cores.txt",
+        "size": 2,
+    }
+    compare_file(expected, actual)
 
-        actual = {
-            "basename": "cores.txt",
-            "checksum": "sha1$7448d8798a4380162d4b56f9b452e2f6f9e24e7a",
-            "class": "File",
-            "location": "file:///var/folders/8x/2df05_7j20j6r8y81w4qf43r0000gn/T/tmpG0EkrS/cores.txt",
-            "path": get_data("tests/test-data/cores.txt"),
-            "size": 2,
-        }
-        compare(expected, actual)
 
-    def test_compare_contents_failure(self):
-        expected = {
-            "location": "cores.txt",
-            "size": 2,
-            "class": "File",
-            "contents": "2",
-        }
+def test_compare_contents_success():
+    expected = {
+        "location": "cores.txt",
+        "size": 2,
+        "class": "File",
+        "contents": "2\n",
+    }
 
-        actual = {
-            "basename": "cores.txt",
-            "checksum": "sha1$7448d8798a4380162d4b56f9b452e2f6f9e24e7a",
-            "class": "File",
-            "location": "file:///var/folders/8x/2df05_7j20j6r8y81w4qf43r0000gn/T/tmpG0EkrS/cores.txt",
-            "path": get_data("tests/test-data/cores.txt"),
-            "size": 2,
-        }
-        with self.assertRaises(CompareFail):
-            compare_file(expected, actual)
+    actual = {
+        "basename": "cores.txt",
+        "checksum": "sha1$7448d8798a4380162d4b56f9b452e2f6f9e24e7a",
+        "class": "File",
+        "location": "file:///var/folders/8x/2df05_7j20j6r8y81w4qf43r0000gn/T/tmpG0EkrS/cores.txt",
+        "path": get_data("tests/test-data/cores.txt"),
+        "size": 2,
+    }
+    compare(expected, actual)
+
+
+def test_compare_contents_failure():
+    expected = {
+        "location": "cores.txt",
+        "size": 2,
+        "class": "File",
+        "contents": "2",
+    }
+
+    actual = {
+        "basename": "cores.txt",
+        "checksum": "sha1$7448d8798a4380162d4b56f9b452e2f6f9e24e7a",
+        "class": "File",
+        "location": "file:///var/folders/8x/2df05_7j20j6r8y81w4qf43r0000gn/T/tmpG0EkrS/cores.txt",
+        "path": get_data("tests/test-data/cores.txt"),
+        "size": 2,
+    }
+    with pytest.raises(CompareFail):
+        compare_file(expected, actual)
diff --git a/cwltest/tests/test_prepare.py b/cwltest/tests/test_prepare.py
index d09f039..3c2ff6f 100644
--- a/cwltest/tests/test_prepare.py
+++ b/cwltest/tests/test_prepare.py
@@ -1,29 +1,26 @@
+"""Test prepare_test_command()"""
 import os
-import unittest
 from cwltest import prepare_test_command
 
 
-class TestPrepareCommand(unittest.TestCase):
-    """Test prepare_test_command()"""
-
-    def test_unix_relative_path(self):
-        """Confirm unix style to windows style path corrections."""
-        command = prepare_test_command(
-            tool="cwl-runner",
-            args=[],
-            testargs=None,
-            test={
-                "doc": "General test of command line generation",
-                "output": {"args": ["echo"]},
-                "tool": "v1.0/bwa-mem-tool.cwl",
-                "job": "v1.0/bwa-mem-job.json",
-                "tags": ["required"],
-            },
-            cwd=os.getcwd(),
-        )
-        if os.name == "nt":
-            self.assertEqual(command[3], "v1.0\\bwa-mem-tool.cwl")
-            self.assertEqual(command[4], "v1.0\\bwa-mem-job.json")
-        else:
-            self.assertEqual(command[3], "v1.0/bwa-mem-tool.cwl")
-            self.assertEqual(command[4], "v1.0/bwa-mem-job.json")
+def test_unix_relative_path():
+    """Confirm unix style to windows style path corrections."""
+    command = prepare_test_command(
+        tool="cwl-runner",
+        args=[],
+        testargs=None,
+        test={
+            "doc": "General test of command line generation",
+            "output": {"args": ["echo"]},
+            "tool": "v1.0/bwa-mem-tool.cwl",
+            "job": "v1.0/bwa-mem-job.json",
+            "tags": ["required"],
+        },
+        cwd=os.getcwd(),
+    )
+    if os.name == "nt":
+        assert command[3] == "v1.0\\bwa-mem-tool.cwl"
+        assert command[4] == "v1.0\\bwa-mem-job.json"
+    else:
+        assert command[3] == "v1.0/bwa-mem-tool.cwl"
+        assert command[4] == "v1.0/bwa-mem-job.json"
diff --git a/cwltest/tests/test_short_names.py b/cwltest/tests/test_short_names.py
index 6efda7f..84a594c 100644
--- a/cwltest/tests/test_short_names.py
+++ b/cwltest/tests/test_short_names.py
@@ -1,56 +1,52 @@
-import unittest
-
 import os
 from os import linesep as n
+from pathlib import Path
 
 from .util import run_with_mock_cwl_runner, get_data
 import defusedxml.ElementTree as ET
 
 
-class TestShortNames(unittest.TestCase):
-    def test_stderr_output(self):
-        args = ["--test", get_data("tests/test-data/short-names.yml")]
-        error_code, stdout, stderr = run_with_mock_cwl_runner(args)
-        self.assertIn(
-            "Test [1/1] opt-error: Test with a short name{n}".format(n=n), stderr
-        )
-
-    def test_run_by_short_name(self):
-        short_name = "opt-error"
-        args = [
-            "--test",
-            get_data("tests/test-data/with-and-without-short-names.yml"),
-            "-s",
-            short_name,
-        ]
-        error_code, stdout, stderr = run_with_mock_cwl_runner(args)
-        self.assertIn("Test [2/2] opt-error: Test with a short name", stderr)
-        self.assertNotIn("Test [1/2]", stderr)
-
-    def test_list_tests(self):
-        args = [
-            "--test",
-            get_data("tests/test-data/with-and-without-short-names.yml"),
-            "-l",
-        ]
-        error_code, stdout, stderr = run_with_mock_cwl_runner(args)
-        self.assertEqual(
-            "[1] Test without a short name{n}"
-            "[2] opt-error: Test with a short name{n}".format(n=n),
-            stdout,
-        )
-
-    def test_short_name_in_junit_xml(self):
-        junit_xml_report = get_data("tests/test-data/junit-report.xml")
-        args = [
-            "--test",
-            get_data("tests/test-data/short-names.yml"),
-            "--junit-xml",
-            junit_xml_report,
-        ]
-        run_with_mock_cwl_runner(args)
-        tree = ET.parse(junit_xml_report)
-        root = tree.getroot()
-        category = root.find("testsuite").find("testcase").attrib["file"]
-        self.assertEqual(category, "opt-error")
-        os.remove(junit_xml_report)
+def test_stderr_output():
+    args = ["--test", get_data("tests/test-data/short-names.yml")]
+    error_code, stdout, stderr = run_with_mock_cwl_runner(args)
+    assert f"Test [1/1] opt-error: Test with a short name{n}" in stderr
+
+
+def test_run_by_short_name():
+    short_name = "opt-error"
+    args = [
+        "--test",
+        get_data("tests/test-data/with-and-without-short-names.yml"),
+        "-s",
+        short_name,
+    ]
+    error_code, stdout, stderr = run_with_mock_cwl_runner(args)
+    assert "Test [2/2] opt-error: Test with a short name" in stderr
+    assert "Test [1/2]" not in stderr
+
+
+def test_list_tests():
+    args = [
+        "--test",
+        get_data("tests/test-data/with-and-without-short-names.yml"),
+        "-l",
+    ]
+    error_code, stdout, stderr = run_with_mock_cwl_runner(args)
+    assert (
+        f"[1] Test without a short name{n}" f"[2] opt-error: Test with a short name{n}"
+    ) in stdout
+
+
+def test_short_name_in_junit_xml(tmp_path: Path):
+    junit_xml_report = tmp_path / "junit-report.xml"
+    args = [
+        "--test",
+        get_data("tests/test-data/short-names.yml"),
+        "--junit-xml",
+        str(junit_xml_report),
+    ]
+    run_with_mock_cwl_runner(args)
+    tree = ET.parse(junit_xml_report)
+    root = tree.getroot()
+    category = root.find("testsuite").find("testcase").attrib["file"]
+    assert category == "opt-error"
diff --git a/cwltest/tests/test_timeout.py b/cwltest/tests/test_timeout.py
new file mode 100644
index 0000000..431e665
--- /dev/null
+++ b/cwltest/tests/test_timeout.py
@@ -0,0 +1,42 @@
+import re
+import os
+from os import linesep as n
+from os import sep as p
+from pathlib import Path
+
+import defusedxml.ElementTree as ET
+
+from .util import run_with_mock_cwl_runner, get_data
+import schema_salad.ref_resolver
+
+
+def test_timeout_stderr_stdout(tmp_path):
+    junit_xml_report = tmp_path / "junit-report.xml"
+
+    args = [
+        "--test",
+        schema_salad.ref_resolver.file_uri(get_data("tests/test-data/timeout.yml")),
+        "--timeout",
+        "5",
+        "--junit-xml",
+        str(junit_xml_report),
+    ]
+    try:
+        cwd = os.getcwd()
+        os.chdir(get_data("tests/test-data/"))
+        error_code, stdout, stderr = run_with_mock_cwl_runner(args)
+    finally:
+        os.chdir(cwd)
+
+    assert error_code == 1
+    assert "Test 1 timed out" in stderr
+    tree = ET.parse(junit_xml_report)
+    try:
+        root = tree.getroot()
+        timeout_text = root.find("testsuite").find("testcase").find("failure").text
+        timeout_stderr = root.find("testsuite").find("testcase").find("system-err").text
+        assert "Test timed out" in timeout_text
+        assert "timeout stderr" in timeout_stderr
+    except AttributeError as e:
+        print(junit_xml_report.read_text())
+        raise e
diff --git a/cwltest/tests/util.py b/cwltest/tests/util.py
index 414db00..25879bc 100644
--- a/cwltest/tests/util.py
+++ b/cwltest/tests/util.py
@@ -1,5 +1,4 @@
 import os
-import sys
 
 import subprocess  # nosec
 
@@ -11,9 +10,9 @@
 
 
 def get_data(filename):
-    filename = os.path.normpath(
-        filename
-    )  # normalizing path depending on OS or else it will cause problem when joining path
+    filename = os.path.normpath(filename)
+    # normalizing path depending on OS or else it will cause problem when
+    # joining path
     filepath = None
     try:
         filepath = resource_filename(Requirement.parse("cwltest"), filename)
diff --git a/cwltest/utils.py b/cwltest/utils.py
index 984d8a8..50e1cb5 100644
--- a/cwltest/utils.py
+++ b/cwltest/utils.py
@@ -111,7 +111,7 @@ def check_keys(keys, expected, actual):
         except CompareFail as e:
             raise CompareFail.format(
                 expected, actual, f"field '{k}' failed comparison: {str(e)}"
-            )
+            ) from e
 
 
 def compare_file(expected, actual):
@@ -159,7 +159,7 @@ def compare_dict(expected, actual):
         except CompareFail as e:
             raise CompareFail.format(
                 expected, actual, f"failed comparison for key '{c}': {e}"
-            )
+            ) from e
     extra_keys = set(actual.keys()).difference(list(expected.keys()))
     for k in extra_keys:
         if actual[k] is not None:
@@ -194,13 +194,13 @@ def compare(expected, actual):  # type: (Any, Any) -> None
                 try:
                     compare(expected[c], actual[c])
                 except CompareFail as e:
-                    raise CompareFail.format(expected, actual, e)
+                    raise CompareFail.format(expected, actual, e) from e
         else:
             if expected != actual:
                 raise CompareFail.format(expected, actual)
 
     except Exception as e:
-        raise CompareFail(str(e))
+        raise CompareFail(str(e)) from e
 
 
 def get_test_number_by_key(tests, key, value):
diff --git a/release-test.sh b/release-test.sh
index dcb6d90..e840392 100755
--- a/release-test.sh
+++ b/release-test.sh
@@ -7,6 +7,8 @@ export LC_ALL=C
 
 package=cwltest
 module=cwltest
+extras=""
+
 if [ "$GITHUB_ACTIONS" = "true" ]; then
     # We are running as a GH Action
     repo=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git
@@ -16,8 +18,8 @@ else
     HEAD=$(git rev-parse HEAD)
 fi
 run_tests="bin/py.test --pyargs ${module}"
-pipver=20.3b1 # minimum required version of pip for Python 3.9
-setuptoolsver=41.1.0 # required for Python 3.9
+pipver=20.3.3  # minimum required version of pip for Python 3.10
+setuptoolsver=50.0.0  # required for Python 3.10
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 
 rm -Rf testenv? || /bin/true
@@ -33,7 +35,7 @@ then
 	rm -f testenv1/lib/python-wheels/setuptools* \
 		&& pip install --force-reinstall -U pip==${pipver} \
 		&& pip install setuptools==${setuptoolsver} wheel
-	pip install -rtest-requirements.txt
+	pip install -rtest-requirements.txt ".${extras}"
 	make test
 	pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install
 	mkdir testenv1/not-${module}
@@ -59,7 +61,7 @@ rm -f lib/python-wheels/setuptools* \
 	&& pip install --force-reinstall -U pip==${pipver} \
         && pip install setuptools==${setuptoolsver} wheel
 # The following can fail if you haven't pushed your commits to ${repo}
-pip install -e "git+${repo}@${HEAD}#egg=${package}"
+pip install -e "git+${repo}@${HEAD}#egg=${package}${extras}"
 pushd src/${package}
 pip install -rtest-requirements.txt
 make dist
@@ -83,13 +85,13 @@ rm -f lib/python-wheels/setuptools* \
         && pip install setuptools==${setuptoolsver} wheel
 package_tar=$(find . -name "${package}*tar.gz")
 pip install "-r${DIR}/test-requirements.txt"
-pip install "${package_tar}"
+pip install "${package_tar}${extras}"
 mkdir out
 tar --extract --directory=out -z -f ${package}*.tar.gz
 pushd out/${package}*
 make dist
 make test
-pip install "-r${DIR}/mypy_requirements.txt"
+pip install "-r${DIR}/mypy-requirements.txt"
 make mypy
 pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install
 mkdir ../not-${module}
@@ -107,7 +109,7 @@ source bin/activate
 rm -f lib/python-wheels/setuptools* \
 	&& pip install --force-reinstall -U pip==${pipver} \
         && pip install setuptools==${setuptoolsver} wheel
-pip install ${module}*.whl
+pip install "$(ls ${module}*.whl)${extras}"
 pip install "-r${DIR}/test-requirements.txt"
 mkdir not-${module}
 pushd not-${module}
diff --git a/test-requirements.txt b/test-requirements.txt
index fb43315..eb7dc7f 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -1,3 +1,2 @@
 pytest >= 6.2, < 6.3
 pytest-cov
-pytest-xdist