Repository: LUMC/pytest-workflow (language: python)

Issue:
Switch to github CI. Test on 3.9 explicitly.
### Checklist
- [x] Pull request details were added to HISTORY.rst



Discussion / hints:

Fix patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 00000000..4a1e549f
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,91 @@
+name: Continous integration
+
+on:
+  pull_request:
+  push:
+    branches:
+      - develop
+      - master
+
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - 3.6
+    steps:
+      - uses: actions/checkout@v2.3.4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+      - name: Install tox
+        run: pip install tox
+      - name: Lint
+        run: tox -e lint
+  docs:
+    needs: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2.3.4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+      - name: Install tox
+        run: pip install tox
+      - name: Build docs
+        run: tox -e docs
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - 3.6
+          - 3.7
+          - 3.8
+          - 3.9
+    needs: lint
+    steps:
+      - uses: actions/checkout@v2.3.4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+      - name: Install tox
+        run: pip install tox
+      - name: Run tests
+        run: tox -e py3
+      - name: Upload coverage report
+        if: ${{ matrix.python-version == 3.6 }} # Only upload coverage once
+        uses: codecov/codecov-action@v1
+
+  test-functional:
+    runs-on: ubuntu-latest
+    needs: test
+    strategy:
+      matrix:
+        python-version: [3.7]
+        test-program: [cromwell, snakemake, miniwdl]
+    steps:
+      - uses: actions/checkout@v2.3.4
+
+      # Setup python program requirements
+      - name: Set up Python ${{ matrix.python-version }}
+        if: ${{ matrix.test-program != 'cromwell' }}
+        uses: actions/setup-python@v2
+      - name: Install tox
+        if: ${{ matrix.test-program != 'cromwell' }}
+        run: pip install tox
+
+      # Setup cromwell requirements
+      - name: Install conda
+        if: ${{ matrix.test-program == 'cromwell' }}
+        uses: conda-incubator/setup-miniconda@v2.0.1 # https://github.com/conda-incubator/setup-miniconda.
+        with:
+          channels: conda-forge,defaults
+      - name: Install cromwell and tox
+        shell: bash -l {0}  # Needed for conda
+        if: ${{ matrix.test-program == 'cromwell' }}
+        run: conda install cromwell tox
+
+      # Test
+      - name: Test
+        shell: bash -l {0}  # Needed for conda
+        run: tox -e ${{ matrix.test-program }}
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index e1f523d2..00000000
--- a/.travis.yml
+++ /dev/null
@@ -1,44 +0,0 @@
-language: python
-install:
-  - pip install tox
-dist: xenial
-python: 3.6  # Use the oldest supported version of python as default.
-script:
-    - tox -e $TOX_ENV
-matrix:
-  include:
-    # TEST DOCS AND LINTING
-    # Use default python3 version here.
-    - env: TOX_ENV=docs
-    - env: TOX_ENV=lint
-    # UNIT TESTS
-    # On most recent versions of python.
-    - env: TOX_ENV=py36
-      install:
-        - pip install codecov
-        - pip install tox
-      after_success:
-        - codecov -v  # -v to make sure coverage upload works.
-    - python: 3.7
-      env: TOX_ENV=py37
-    - python: 3.8
-      env: TOX_ENV=py38
-
-    # FUNCTIONAL TESTS
-    # Use default python here.
-    - env: TOX_ENV=snakemake
-    - env: TOX_ENV=miniwdl
-    - env: TOX_ENV=cromwell
-      install:
-        # Install conda
-        - export MINICONDA=${HOME}/miniconda
-        - export PATH=${MINICONDA}/bin:${PATH}
-        - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
-        - bash miniconda.sh -b -f -p ${MINICONDA}
-        - conda config --set always_yes yes
-        - conda config --add channels defaults
-        - conda config --add channels bioconda
-        - conda config --add channels conda-forge
-        # TODO: Upgrade cromwell once https://github.com/broadinstitute/cromwell/pull/5456 is fixed.
-        - conda create -n my_env cromwell=48 tox  # Install tox for good integration within the conda env.
-        - source activate my_env
diff --git a/HISTORY.rst b/HISTORY.rst
index 0a5f2d61..a59b7f4d 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -7,6 +7,17 @@ Changelog
 .. This document is user facing. Please word the changes in such a way
 .. that users understand how the changes affect the new version.
 
+version 1.5.0
+---------------------------
++ Add support for python 3.9
++ Update the print statement for starting jobs to be more structured. This will
+  make the output easier to to read and use, since different fields (stdout,
+  stderr, command, etc) are all on their own line.
++ Do not crash when directories can not be removed due to permission errors.
+  Instead display a message to notify the users which directories could not be
+  removed. These issues occurred sometimes when tests involving docker were
+  run.
+
 version 1.4.0
 ---------------------------
 + Usage of the ``name`` keyword argument in workflow marks is now deprecated.
diff --git a/README.rst b/README.rst
index 52dcd23e..b8de9c2f 100644
--- a/README.rst
+++ b/README.rst
@@ -41,8 +41,8 @@ For our complete documentation checkout our
 
 Installation
 ============
-Pytest-workflow requires Python 3.6 or higher. It is tested on Python 3.6, 3.7
-and 3.8. Python 2 is not supported.
+Pytest-workflow requires Python 3.6 or higher. It is tested on Python 3.6, 3.7,
+3.8 and 3.9. Python 2 is not supported.
 
 - Make sure your virtual environment is activated.
 - Install using pip ``pip install pytest-workflow``
diff --git a/docs/examples.rst b/docs/examples.rst
index d3cfad9a..4b823449 100644
--- a/docs/examples.rst
+++ b/docs/examples.rst
@@ -88,7 +88,7 @@ inside the test output directory. If you have a workflow with the output:
             Array[File] stats = moo_task.stats_files
         }
 
-Inside the ``output_links`` directory the directories ``moo_file`` and
+Inside the ``out`` directory the directories ``moo_file`` and
 ``stats`` will be created. Inside these directories will be the produced files.
 
 The following yaml file tests a WDL pipeline run with miniwdl.
diff --git a/docs/installation.rst b/docs/installation.rst
index 6689b52b..dd939b33 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -2,7 +2,7 @@
 Installation
 ============
 
-Pytest-workflow is tested on python 3.6, 3.7 and 3.8. Python 2 is not
+Pytest-workflow is tested on python 3.6, 3.7, 3.8 and 3.9. Python 2 is not
 supported.
 
 In a virtual environment
diff --git a/mypy.ini b/mypy.ini
deleted file mode 100644
index f1e5d681..00000000
--- a/mypy.ini
+++ /dev/null
@@ -1,3 +0,0 @@
-[mypy]
-# This ignores a lot of errors that are a huge headache to fix.
-ignore_missing_imports = True
diff --git a/setup.py b/setup.py
index 5f5a0409..9433b815 100644
--- a/setup.py
+++ b/setup.py
@@ -14,19 +14,18 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with pytest-workflow.  If not, see <https://www.gnu.org/licenses/
 
-from setuptools import find_packages, setup
+from pathlib import Path
 
-with open("README.rst", "r") as readme_file:
-    LONG_DESCRIPTION = readme_file.read()
+from setuptools import find_packages, setup
 
 setup(
     name="pytest-workflow",
-    version="1.4.0",
+    version="1.5.0",
     description="A pytest plugin for configuring workflow/pipeline tests "
                 "using YAML files",
     author="Leiden University Medical Center",
     author_email="sasc@lumc.nl",  # A placeholder for now
-    long_description=LONG_DESCRIPTION,
+    long_description=Path("README.rst").read_text(),
     long_description_content_type="text/x-rst",
     license="AGPL-3.0-or-later",
     keywords="pytest workflow pipeline yaml yml wdl cromwell snakemake",
@@ -43,6 +42,7 @@
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
         "Development Status :: 5 - Production/Stable",
         "License :: OSI Approved :: "
         "GNU Affero General Public License v3 or later (AGPLv3+)",
diff --git a/tox.ini b/tox.ini
index e7b2920b..6a232478 100644
--- a/tox.ini
+++ b/tox.ini
@@ -20,7 +20,7 @@ commands =
     # Skip 'assert used' issue as this is a pytest plugin.'
     # Use csv format because it has cleanest output.
     bandit -s B101 -f csv -r src/ tests/
-    mypy src/pytest_workflow tests/
+    mypy --ignore-missing-imports src/pytest_workflow tests/
 
 # Documentation should build on python version 3
 [testenv:docs]


Test patch:
diff --git a/src/pytest_workflow/file_tests.py b/src/pytest_workflow/file_tests.py
index 133eb453..8eca7151 100644
--- a/src/pytest_workflow/file_tests.py
+++ b/src/pytest_workflow/file_tests.py
@@ -15,13 +15,14 @@
 # along with pytest-workflow.  If not, see <https://www.gnu.org/licenses/
 
 """All tests for workflow files"""
-import hashlib
+
 from pathlib import Path
 
 import pytest
 
 from .content_tests import ContentTestCollector
 from .schema import FileTest
+from .util import file_md5sum
 from .workflow import Workflow
 
 
@@ -141,18 +142,3 @@ def repr_failure(self, excinfo, style=None):
             f"Observed md5sum '{self.observed_md5sum}' not equal to expected "
             f"md5sum '{self.expected_md5sum}' for file '{self.filepath}'"
         )
-
-
-# block_size 64k with python is a few percent faster than linux native md5sum.
-def file_md5sum(filepath: Path, block_size=64 * 1024) -> str:
-    """
-    Generates a md5sum for a file. Reads file in blocks to save memory.
-    :param filepath: a pathlib. Path to the file
-    :param block_size: Block size in bytes
-    :return: a md5sum as hexadecimal string.
-    """
-    hasher = hashlib.md5()  # nosec: only used for file integrity
-    with filepath.open('rb') as file_handler:  # Read the file in bytes
-        for block in iter(lambda: file_handler.read(block_size), b''):
-            hasher.update(block)
-    return hasher.hexdigest()
diff --git a/src/pytest_workflow/plugin.py b/src/pytest_workflow/plugin.py
index 168dd381..477ee74c 100644
--- a/src/pytest_workflow/plugin.py
+++ b/src/pytest_workflow/plugin.py
@@ -20,11 +20,11 @@
 import tempfile
 import warnings
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 
 from _pytest.config import Config as PytestConfig
 from _pytest.config.argparsing import Parser as PytestParser
-from _pytest.mark import MarkDecorator
+from _pytest.mark import Mark
 from _pytest.python import FunctionDefinition, Metafunc
 
 import pytest
@@ -34,7 +34,7 @@
 from .content_tests import ContentTestCollector
 from .file_tests import FileTestCollector
 from .schema import WorkflowTest, workflow_tests_from_schema
-from .util import is_in_dir, link_tree, replace_whitespace, rm_dirs
+from .util import is_in_dir, link_tree, replace_whitespace
 from .workflow import Workflow, WorkflowQueue
 
 
@@ -189,8 +189,7 @@ def pytest_collection():
     print()
 
 
-def get_workflow_names_from_workflow_marker(marker: MarkDecorator
-                                            ) -> List[str]:
+def get_workflow_names_from_workflow_marker(marker: Mark) -> Tuple[Any, ...]:
     if 'name' in marker.kwargs:
         raise DeprecationWarning(
             "Using pytest.mark.workflow(name='workflow name') is "
@@ -211,18 +210,17 @@ def pytest_generate_tests(metafunc: Metafunc):
         return
 
     definition: FunctionDefinition = metafunc.definition
-    marker: Optional[MarkDecorator] = definition.get_closest_marker(
-        name="workflow")
+    marker: Optional[Mark] = definition.get_closest_marker(name="workflow")
     if marker is None:
         raise ValueError("workflow_dir can only be requested in tests marked"
                          " with the workflow mark.")
 
-    workflow_names: List[str] = get_workflow_names_from_workflow_marker(marker)
+    workflow_names = get_workflow_names_from_workflow_marker(marker)
     if not workflow_names:
         raise ValueError(f"A workflow name or names should be defined in "
                          f"the workflow marker of {definition.nodeid}")
 
-    workflow_temp_dir = metafunc.config.workflow_temp_dir
+    workflow_temp_dir = metafunc.config.workflow_temp_dir  # type: ignore
     workflow_dirs = [Path(workflow_temp_dir, replace_whitespace(name))
                      for name in workflow_names]
     metafunc.parametrize("workflow_dir", workflow_dirs,
@@ -234,14 +232,12 @@ def pytest_collection_modifyitems(config: PytestConfig,
     """Here we skip all tests related to workflows that are not executed"""
 
     for item in items:
-        marker: Optional[MarkDecorator] = item.get_closest_marker(
-            name="workflow")
+        marker = item.get_closest_marker(name="workflow")
 
         if marker is None:
             continue
 
-        workflow_names: List[str] = get_workflow_names_from_workflow_marker(
-            marker)
+        workflow_names = get_workflow_names_from_workflow_marker(marker)
         if len(workflow_names) == 1:
             workflow_name = workflow_names[0]
         elif "workflow_dir" in item.fixturenames:
@@ -252,7 +248,7 @@ def pytest_collection_modifyitems(config: PytestConfig,
             raise NotImplementedError(f"Cannot determine workflow name for "
                                       f"{item.nodeid}")
 
-        if workflow_name not in config.executed_workflows.keys():
+        if workflow_name not in config.executed_workflows.keys():  # type: ignore  # noqa: E501
             skip_marker = pytest.mark.skip(
                 reason=f"'{workflow_name}' has not run.")
             item.add_marker(skip_marker)
@@ -260,7 +256,7 @@ def pytest_collection_modifyitems(config: PytestConfig,
 
 def pytest_runtestloop(session: pytest.Session):
     """This runs after collection, but before the tests."""
-    session.config.workflow_queue.process(
+    session.config.workflow_queue.process(  # type: ignore
         session.config.getoption("workflow_threads")
     )
 
@@ -273,7 +269,7 @@ def pytest_collectstart(collector: pytest.Collector):
 
         # Executed workflows contains workflow name as key and nodeid as value.
         executed_workflows: Dict[str, str] = (
-            collector.config.executed_workflows)
+            collector.config.executed_workflows)  # type: ignore
 
         if name in executed_workflows.keys():
             raise ValueError(
@@ -283,25 +279,42 @@ def pytest_collectstart(collector: pytest.Collector):
 
 
 def pytest_sessionfinish(session: pytest.Session, exitstatus: int):
-    # No cleanup needed if we keep workflow directories
-    # Or if there are no directories to cleanup. (I.e. pytest-workflow plugin
-    # was not used.)
-    if (session.config.getoption("keep_workflow_wd") or
-            len(session.config.workflow_cleanup_dirs) == 0):
-        pass
-    elif session.config.getoption("keep_workflow_wd_on_fail"):
-        # Ony cleanup if there are cleanup_dirs.
-        if exitstatus == 0:
-            print("All tests succeeded. Removing temporary directories and "
-                  "logs.")
-            rm_dirs(session.config.workflow_cleanup_dirs)
-        else:
-            print("One or more tests failed. Keeping temporary directories "
-                  "and logs.")
-    else:  # When no flags are set. Remove temporary directories and logs.
-        print("Removing temporary directories and logs. Use '--kwd' or "
-              "'--keep-workflow-wd' to disable this behaviour.")
-        rm_dirs(session.config.workflow_cleanup_dirs)
+    directories: List[Path] = session.config.workflow_cleanup_dirs  # type: ignore # noqa: E501
+    # No cleanup needed if there are no directories to cleanup. (I.e.
+    # pytest-workflow plugin was not used.)
+    if len(directories) == 0:
+        return
+
+    keep_workflow_wd: bool = session.config.getoption("keep_workflow_wd")
+    keep_workflow_wd_on_fail: bool = session.config.getoption(
+        "keep_workflow_wd_on_fail")
+    no_flags = not (keep_workflow_wd_on_fail and keep_workflow_wd)
+    success: bool = exitstatus == 0
+    removal: bool = not (keep_workflow_wd or (keep_workflow_wd_on_fail
+                                              and not success))
+
+    remove_msg = (f"{'Removing' if removal else 'Keeping'} "
+                  f"temporary directories and logs.")
+    # Only print success message if removal was dependent on success.
+    success_msg = (("All tests succeeded." if success else
+                   "One or more tests failed.")
+                   if keep_workflow_wd_on_fail else '')
+    # Only print message about flags if user did not use any flags.
+    no_flag_msg = ("Use '--kwd' or '--keep-workflow-wd' to disable this "
+                   "behaviour." if no_flags else "")
+    print(" ".join([success_msg, remove_msg, no_flag_msg]))
+
+    if removal:
+        unremovable_dirs: List[Path] = []
+        for directory in directories:
+            try:
+                shutil.rmtree(str(directory))
+            except PermissionError:
+                unremovable_dirs.append(directory)
+        if unremovable_dirs:
+            print(f"Unable to remove the following directories due to "
+                  f"permission errors: "
+                  f"{' ,'.join(str(path) for path in unremovable_dirs)}.")
 
 
 class YamlFile(pytest.File):
diff --git a/src/pytest_workflow/schema.py b/src/pytest_workflow/schema.py
index 21121217..df8084a8 100644
--- a/src/pytest_workflow/schema.py
+++ b/src/pytest_workflow/schema.py
@@ -24,12 +24,11 @@
 
 from .util import replace_whitespace
 
-SCHEMA = Path(__file__).parent / Path("schema") / Path("schema.json")
+SCHEMA = Path(__file__).parent / "schema" / "schema.json"
 DEFAULT_EXIT_CODE = 0
 DEFAULT_FILE_SHOULD_EXIST = True
 
-with SCHEMA.open() as schema_fh:
-    JSON_SCHEMA = json.load(schema_fh)
+JSON_SCHEMA = json.loads(SCHEMA.read_text())
 
 
 def workflow_tests_from_schema(schema):
diff --git a/src/pytest_workflow/util.py b/src/pytest_workflow/util.py
index 0e26b94a..ed0a09ea 100644
--- a/src/pytest_workflow/util.py
+++ b/src/pytest_workflow/util.py
@@ -1,9 +1,8 @@
+import hashlib
 import os
 import re
-import shutil
 import warnings
 from pathlib import Path
-from typing import List
 
 
 # This function was created to ensure the same conversion is used throughout
@@ -18,11 +17,6 @@ def replace_whitespace(string: str, replace_with: str = '_') -> str:
     return re.sub(r'\s+', replace_with, string)
 
 
-def rm_dirs(directories: List[Path]):
-    for directory in directories:
-        shutil.rmtree(str(directory))
-
-
 def is_in_dir(child: Path, parent: Path, strict: bool = False) -> bool:
     """
     Checks if child path is in parent path. Works for non-existent paths if
@@ -59,7 +53,22 @@ def link_tree(src: Path, dest: Path) -> None:
         for path in os.listdir(str(src)):
             link_tree(Path(src, path), Path(dest, path))
     elif src.is_file() or src.is_symlink():
-        os.symlink(str(src), str(dest), target_is_directory=False)
+        dest.symlink_to(src, target_is_directory=False)
     else:  # Only copy files and symlinks, no devices etc.
         warnings.warn(f"Unsupported filetype. Skipping copying: '{str(src)}' "
                       f"to '{str(dest)}'.")
+
+
+# block_size 64k with python is a few percent faster than linux native md5sum.
+def file_md5sum(filepath: Path, block_size=64 * 1024) -> str:
+    """
+    Generates a md5sum for a file. Reads file in blocks to save memory.
+    :param filepath: a pathlib. Path to the file
+    :param block_size: Block size in bytes
+    :return: a md5sum as hexadecimal string.
+    """
+    hasher = hashlib.md5()  # nosec: only used for file integrity
+    with filepath.open('rb') as file_handler:  # Read the file in bytes
+        for block in iter(lambda: file_handler.read(block_size), b''):
+            hasher.update(block)
+    return hasher.hexdigest()
diff --git a/src/pytest_workflow/workflow.py b/src/pytest_workflow/workflow.py
index 969a2cc6..fbc9a63d 100644
--- a/src/pytest_workflow/workflow.py
+++ b/src/pytest_workflow/workflow.py
@@ -210,10 +210,11 @@ def worker(self):
                 break
             else:
                 print(
-                    f"start '{workflow.name}' with command "
-                    f"'{workflow.command}' in '{workflow.cwd}'. "
-                    f"stdout: '{workflow.stdout_file}'. "
-                    f"stderr: '{workflow.stderr_file}'.")
+                    f"\n{workflow.name}:\n"
+                    f"\tcommand:   {workflow.command}\n"
+                    f"\tdirectory: {workflow.cwd}\n"
+                    f"\tstdout:    {workflow.stdout_file}\n"
+                    f"\tstderr:    {workflow.stderr_file}")
                 workflow.run()
                 # Collect the workflow errors.
                 self._process_errors.extend(workflow.errors)
diff --git a/tests/functional/simple_wdl_test_cases.yml b/tests/functional/simple_wdl_test_cases.yml
index 8dd71a60..6dcbc2df 100644
--- a/tests/functional/simple_wdl_test_cases.yml
+++ b/tests/functional/simple_wdl_test_cases.yml
@@ -46,34 +46,34 @@
   command: >-
     miniwdl run -i simple.json -d test-output/. simple.wdl
   files:
-    - path: test-output/output_links/rand_files/0/0.bin
-    - path: test-output/output_links/rand_files/1/1.bin
-    - path: test-output/output_links/rand_files/2/2.bin
-    - path: test-output/output_links/rand_files/3/3.bin
-    - path: test-output/output_links/rand_files/4/4.bin
-    - path: test-output/output_links/rand_files/5/5.bin
-    - path: test-output/output_links/rand_files/6/6.bin
-    - path: test-output/output_links/rand_files/7/7.bin
-    - path: test-output/output_links/rand_files/8/8.bin
-    - path: test-output/output_links/rand_files/9/9.bin
-    - path: test-output/output_links/base64_files/0/0.txt
-    - path: test-output/output_links/base64_files/1/1.txt
-    - path: test-output/output_links/base64_files/2/2.txt
-    - path: test-output/output_links/base64_files/3/3.txt
-    - path: test-output/output_links/base64_files/4/4.txt
-    - path: test-output/output_links/base64_files/5/5.txt
-    - path: test-output/output_links/base64_files/6/6.txt
-    - path: test-output/output_links/base64_files/7/7.txt
-    - path: test-output/output_links/base64_files/8/8.txt
-    - path: test-output/output_links/base64_files/9/9.txt
-    - path: test-output/output_links/gzip_files/0/0.txt.gz
-    - path: test-output/output_links/gzip_files/1/1.txt.gz
-    - path: test-output/output_links/gzip_files/2/2.txt.gz
-    - path: test-output/output_links/gzip_files/3/3.txt.gz
-    - path: test-output/output_links/gzip_files/4/4.txt.gz
-    - path: test-output/output_links/gzip_files/5/5.txt.gz
-    - path: test-output/output_links/gzip_files/6/6.txt.gz
-    - path: test-output/output_links/gzip_files/7/7.txt.gz
-    - path: test-output/output_links/gzip_files/8/8.txt.gz
-    - path: test-output/output_links/gzip_files/9/9.txt.gz
-    - path: test-output/output_links/zipped_file/all_data.gz
+    - path: test-output/out/rand_files/0/0.bin
+    - path: test-output/out/rand_files/1/1.bin
+    - path: test-output/out/rand_files/2/2.bin
+    - path: test-output/out/rand_files/3/3.bin
+    - path: test-output/out/rand_files/4/4.bin
+    - path: test-output/out/rand_files/5/5.bin
+    - path: test-output/out/rand_files/6/6.bin
+    - path: test-output/out/rand_files/7/7.bin
+    - path: test-output/out/rand_files/8/8.bin
+    - path: test-output/out/rand_files/9/9.bin
+    - path: test-output/out/base64_files/0/0.txt
+    - path: test-output/out/base64_files/1/1.txt
+    - path: test-output/out/base64_files/2/2.txt
+    - path: test-output/out/base64_files/3/3.txt
+    - path: test-output/out/base64_files/4/4.txt
+    - path: test-output/out/base64_files/5/5.txt
+    - path: test-output/out/base64_files/6/6.txt
+    - path: test-output/out/base64_files/7/7.txt
+    - path: test-output/out/base64_files/8/8.txt
+    - path: test-output/out/base64_files/9/9.txt
+    - path: test-output/out/gzip_files/0/0.txt.gz
+    - path: test-output/out/gzip_files/1/1.txt.gz
+    - path: test-output/out/gzip_files/2/2.txt.gz
+    - path: test-output/out/gzip_files/3/3.txt.gz
+    - path: test-output/out/gzip_files/4/4.txt.gz
+    - path: test-output/out/gzip_files/5/5.txt.gz
+    - path: test-output/out/gzip_files/6/6.txt.gz
+    - path: test-output/out/gzip_files/7/7.txt.gz
+    - path: test-output/out/gzip_files/8/8.txt.gz
+    - path: test-output/out/gzip_files/9/9.txt.gz
+    - path: test-output/out/zipped_file/all_data.gz
diff --git a/tests/functional/test_functional.py b/tests/functional/test_functional.py
index 5500f0b7..bf77bfa9 100644
--- a/tests/functional/test_functional.py
+++ b/tests/functional/test_functional.py
@@ -17,10 +17,9 @@
 
 import pytest
 
-SIMPLE_WDL_YAML = Path(__file__).parent / Path("simple_wdl_test_cases.yml")
-SNAKEFILE_YAML = (Path(__file__).parent /
-                  Path("simple_snakefile_test_cases.yml"))
-PIPELINE_DIR = Path(__file__).parent.parent / Path("pipelines")
+SIMPLE_WDL_YAML = Path(__file__).parent / "simple_wdl_test_cases.yml"
+SNAKEFILE_YAML = Path(__file__).parent / "simple_snakefile_test_cases.yml"
+PIPELINE_DIR = Path(__file__).parent.parent / "pipelines"
 SIMPLE_WDL = Path(PIPELINE_DIR, "wdl", "simple.wdl")
 SIMPLE_WDL_JSON = Path(PIPELINE_DIR, "wdl", "simple.json")
 SIMPLE_WDL_OPTIONS_JSON = Path(PIPELINE_DIR, "wdl", "simple.options.json")
diff --git a/tests/test_content_functions.py b/tests/test_content_functions.py
index 9fe5b7cb..743d4d4c 100644
--- a/tests/test_content_functions.py
+++ b/tests/test_content_functions.py
@@ -22,8 +22,8 @@
 
 from pytest_workflow.content_tests import check_content, check_regex_content
 
-LICENSE = Path(__file__).parent / Path("content_files", "LICENSE")
-LICENSE_ZIPPED = LICENSE.parent / Path("LICENSE.gz")
+LICENSE = Path(__file__).parent / "content_files" / "LICENSE"
+LICENSE_ZIPPED = LICENSE.parent / "LICENSE.gz"
 
 # Yes we are checking the AGPLv3+. I am pretty sure some strings will not be
 # there
diff --git a/tests/test_file_functions.py b/tests/test_file_functions.py
deleted file mode 100644
index 383f40ee..00000000
--- a/tests/test_file_functions.py
+++ /dev/null
@@ -1,37 +0,0 @@
-# Copyright (C) 2018 Leiden University Medical Center
-# This file is part of pytest-workflow
-#
-# pytest-workflow is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# pytest-workflow is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with pytest-workflow.  If not, see <https://www.gnu.org/licenses/
-
-import hashlib
-import os
-from pathlib import Path
-
-import pytest
-
-from pytest_workflow.file_tests import file_md5sum
-
-HASH_FILE_DIR = Path(__file__).parent / Path("hash_files")
-
-HASH_FILES_RELATIVE = os.listdir(str(HASH_FILE_DIR.absolute()))
-
-HASH_FILES = [Path(HASH_FILE_DIR, x) for x in HASH_FILES_RELATIVE]
-
-
-@pytest.mark.parametrize("hash_file", HASH_FILES)
-def test_file_md5sum(hash_file: Path):
-    # No sec added because this hash is only used for checking file integrity
-    whole_file_md5 = hashlib.md5(hash_file.read_bytes()).hexdigest()  # nosec
-    per_line_md5 = file_md5sum(hash_file)
-    assert whole_file_md5 == per_line_md5
diff --git a/tests/test_schema.py b/tests/test_schema.py
index 57cfcb9d..378be288 100644
--- a/tests/test_schema.py
+++ b/tests/test_schema.py
@@ -28,18 +28,18 @@
 
 import yaml
 
-VALID_YAML_DIR = Path(__file__).parent / Path("yamls", "valid")
+VALID_YAML_DIR = Path(__file__).parent / "yamls" / "valid"
 VALID_YAMLS = os.listdir(VALID_YAML_DIR)
 
 
 @pytest.mark.parametrize("yaml_path", VALID_YAMLS)
 def test_validate_schema(yaml_path):
-    with Path(VALID_YAML_DIR / Path(yaml_path)).open() as yaml_fh:
+    with Path(VALID_YAML_DIR, yaml_path).open() as yaml_fh:
         validate_schema(yaml.safe_load(yaml_fh))
 
 
 def test_workflowtest():
-    with Path(VALID_YAML_DIR / Path("dream_file.yaml")).open() as yaml_fh:
+    with Path(VALID_YAML_DIR, "dream_file.yaml").open() as yaml_fh:
         test_yaml = yaml.safe_load(yaml_fh)
         tests = [WorkflowTest.from_schema(x) for x in test_yaml]
         assert tests[0].name == "simple echo"
@@ -53,7 +53,7 @@ def test_workflowtest():
 
 
 def test_workflowtest_regex():
-    with Path(VALID_YAML_DIR / Path("regex_file.yaml")).open() as yaml_fh:
+    with Path(VALID_YAML_DIR, "regex_file.yaml").open() as yaml_fh:
         test_yaml = yaml.safe_load(yaml_fh)
         tests = [WorkflowTest.from_schema(x) for x in test_yaml]
         assert tests[0].name == "simple echo"
@@ -145,7 +145,7 @@ def test_validate_schema_contains_conflict(instance):
 
 
 def test_workflow_tests_from_schema():
-    with Path(VALID_YAML_DIR / Path("dream_file.yaml")).open() as yaml_fh:
+    with Path(VALID_YAML_DIR, "dream_file.yaml").open() as yaml_fh:
         test_yaml = yaml.safe_load(yaml_fh)
         workflow_tests = workflow_tests_from_schema(test_yaml)
         assert len(workflow_tests) == 2
diff --git a/tests/test_success_messages.py b/tests/test_success_messages.py
index 26ec702c..566e486b 100644
--- a/tests/test_success_messages.py
+++ b/tests/test_success_messages.py
@@ -101,7 +101,7 @@
     ["test_succeeding.yml::failing grep::stdout::does not contain 'grep' PASSED"],  # noqa: E501
     ["test_succeeding.yml::failing grep::stderr::contains ''grep --help''"],
     ["test_succeeding.yml::zipped file::moo.gz::content::contains 'moo' PASSED"],  # noqa: E501
-    ["start 'moo file' with command 'bash -c 'echo moo > moo.txt'' in"],
+    ["moo file:\n\tcommand:   bash -c 'echo moo > moo.txt'\n\tdirectory"],
     ["'moo file' done."],
     ["test_succeeding.yml::regex::exit code should be 0 PASSED"],
     ["test_succeeding.yml::regex::stdout::contains 'ello' PASSED"],
@@ -118,7 +118,7 @@ def succeeding_tests_output(tmpdir_factory: TempdirFactory):
     has to be run again.
     This fixture runs the succeeding tests once with pytest -v"""
     tempdir = str(tmpdir_factory.mktemp("succeeding_tests"))
-    test_file = Path(Path(tempdir) / Path("test_succeeding.yml"))
+    test_file = Path(tempdir, "test_succeeding.yml")
     with test_file.open("w") as file_handler:
         file_handler.write(SUCCEEDING_TESTS_YAML)
     process_out = subprocess.run(args=["pytest", "-v"],  # nosec
diff --git a/tests/test_temp_directory.py b/tests/test_temp_directory.py
index 5f676b43..85319c50 100644
--- a/tests/test_temp_directory.py
+++ b/tests/test_temp_directory.py
@@ -18,6 +18,7 @@
 import re
 import shutil
 import tempfile
+import textwrap
 from pathlib import Path
 
 from .test_success_messages import SIMPLE_ECHO
@@ -26,18 +27,18 @@
 def test_directory_kept(testdir):
     testdir.makefile(".yml", test=SIMPLE_ECHO)
     result = testdir.runpytest("-v", "--keep-workflow-wd")
-    working_dir = re.search(r"with command 'echo moo' in '([\w/_-]*)'",
+    working_dir = re.search(r"command:   echo moo\n\tdirectory: ([\w/_-]*)",
                             result.stdout.str()).group(1)
     assert Path(working_dir).exists()
-    assert Path(working_dir / Path("log.out")).exists()
-    assert Path(working_dir / Path("log.err")).exists()
+    assert Path(working_dir, "log.out").exists()
+    assert Path(working_dir, "log.err").exists()
     shutil.rmtree(str(testdir))
 
 
 def test_directory_not_kept(testdir):
     testdir.makefile(".yml", test=SIMPLE_ECHO)
     result = testdir.runpytest("-v")
-    working_dir = re.search(r"with command 'echo moo' in '([\w/_-]*)'",
+    working_dir = re.search(r"command:   echo moo\n\tdirectory: ([\w/_-]*)",
                             result.stdout.str()).group(1)
     assert not Path(working_dir).exists()
     assert ("Removing temporary directories and logs. Use '--kwd' or "
@@ -49,10 +50,10 @@ def test_basetemp_correct(testdir):
     testdir.makefile(".yml", test=SIMPLE_ECHO)
     tempdir = tempfile.mkdtemp()
     result = testdir.runpytest("-v", "--basetemp", tempdir)
-    message = (f"start 'simple echo' with command 'echo moo' in "
-               f"'{tempdir}/simple_echo'. "
-               f"stdout: '{tempdir}/simple_echo/log.out'. "
-               f"stderr: '{tempdir}/simple_echo/log.err'.")
+    message = (f"\tcommand:   echo moo\n"
+               f"\tdirectory: {tempdir}/simple_echo\n"
+               f"\tstdout:    {tempdir}/simple_echo/log.out\n"
+               f"\tstderr:    {tempdir}/simple_echo/log.err")
     assert message in result.stdout.str()
 
 
@@ -62,7 +63,7 @@ def test_basetemp_can_be_used_twice(testdir):
     # First run to fill up tempdir
     testdir.runpytest("-v", "--keep-workflow-wd", "--basetemp", tempdir)
     # Make sure directory is there.
-    assert (Path(tempdir) / Path("simple_echo")).exists()
+    assert Path(tempdir, "simple_echo").exists()
     # Run again with same basetemp.
     result = testdir.runpytest("-v", "--keep-workflow-wd", "--basetemp",
                                tempdir)
@@ -78,7 +79,7 @@ def test_basetemp_will_be_created(testdir):
     # This creates an empty dir
     tempdir_base = tempfile.mkdtemp()
     # This path should not exist
-    tempdir = Path(tempdir_base) / Path("non") / Path("existing")
+    tempdir = Path(tempdir_base, "non", "existing")
     # If pytest-workflow does not handle non-existing nested directories well
     # it should crash.
     result = testdir.runpytest("-v", "--keep-workflow-wd", "--basetemp",
@@ -91,7 +92,7 @@ def test_basetemp_will_be_created(testdir):
 def test_basetemp_can_not_be_in_rootdir(testdir):
     testdir.makefile(".yml", test=SIMPLE_ECHO)
     testdir_path = Path(str(testdir.tmpdir))
-    tempdir = testdir_path / Path("tmp")
+    tempdir = testdir_path / "tmp"
     result = testdir.runpytest("-v", "--basetemp", str(tempdir))
     message = f"'{str(tempdir)}' is a subdirectory of '{str(testdir_path)}'"
     assert message in result.stderr.str()
@@ -112,11 +113,11 @@ def test_directory_kept_on_fail(testdir):
     testdir.makefile(".yml", test=FAIL_TEST)
     result = testdir.runpytest("-v", "--keep-workflow-wd-on-fail")
     working_dir = re.search(
-        r"with command 'bash -c 'exit 1'' in '([\w/_-]*)'",
+        r"command:   bash -c 'exit 1'\n\tdirectory: ([\w/_-]*)",
         result.stdout.str()).group(1)
     assert Path(working_dir).exists()
-    assert Path(working_dir / Path("log.out")).exists()
-    assert Path(working_dir / Path("log.err")).exists()
+    assert Path(working_dir, "log.out").exists()
+    assert Path(working_dir, "log.err").exists()
     assert ("One or more tests failed. Keeping temporary directories and "
             "logs." in result.stdout.str())
     shutil.rmtree(working_dir)
@@ -126,7 +127,7 @@ def test_directory_not_kept_on_succes(testdir):
     testdir.makefile(".yml", test=SUCCESS_TEST)
     result = testdir.runpytest("-v", "--kwdof")
     working_dir = re.search(
-        r"with command 'bash -c 'exit 0'' in '([\w/_-]*)'",
+        r"command:   bash -c 'exit 0'\n\tdirectory: ([\w/_-]*)",
         result.stdout.str()).group(1)
     assert not Path(working_dir).exists()
     assert ("All tests succeeded. Removing temporary directories and logs." in
@@ -139,9 +140,24 @@ def test_directory_of_symlinks(testdir):
     Path(str(subdir), "subfile.txt").write_text("test")
     result = testdir.runpytest("-v", "--symlink", "--kwd")
     working_dir = re.search(
-        r"with command 'echo moo' in '([\w/_-]*)'",
+        r"command:   echo moo\n\tdirectory: ([\w/_-]*)",
         result.stdout.str()).group(1)
     assert Path(working_dir, "test.yml").is_symlink()
     assert Path(working_dir, "subdir").is_dir()
     assert Path(working_dir, "subdir", "subfile.txt").is_symlink()
     shutil.rmtree(working_dir)
+
+
+def test_directory_unremovable_message(testdir):
+    # Following directory contains nested contents owned by root.
+    test = textwrap.dedent("""
+    - name: Create unremovable dir
+      command: >-
+        bash -c "docker run -v $(pwd):$(pwd) -w $(pwd) debian \
+        bash -c 'mkdir test && touch test/test'"
+    """)
+    testdir.makefile(".yaml", test=test)
+    result = testdir.runpytest()
+    assert ("Unable to remove the following directories due to permission "
+            "errors" in result.stdout.str())
+    assert result.ret == 0
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 6683dfd1..54ae7967 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -13,13 +13,15 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with pytest-workflow.  If not, see <https://www.gnu.org/licenses/
+import hashlib
 import shutil
 import tempfile
 from pathlib import Path
 
 import pytest
 
-from pytest_workflow.util import is_in_dir, link_tree, replace_whitespace
+from pytest_workflow.util import file_md5sum, is_in_dir, link_tree, \
+    replace_whitespace
 
 WHITESPACE_TESTS = [
     ("bla\nbla", "bla_bla"),
@@ -49,7 +51,7 @@ def test_is_in_dir(child: str, parent: str, in_dir: bool):
 
 
 def test_link_tree():
-    pipelines_dir = Path(__file__).parent / Path("pipelines")
+    pipelines_dir = Path(__file__).parent / "pipelines"
     tempdir = Path(tempfile.mkdtemp(), "test")
     link_tree(pipelines_dir, tempdir)
     snakemake = Path(tempdir, "snakemake")
@@ -83,3 +85,14 @@ def test_link_tree_warning():
         link_tree(urandom, dest)
     # Destination should not be created
     assert not dest.exists()
+
+
+HASH_FILE_DIR = Path(__file__).parent / "hash_files"
+
+
+@pytest.mark.parametrize("hash_file", HASH_FILE_DIR.iterdir())
+def test_file_md5sum(hash_file: Path):
+    # No sec added because this hash is only used for checking file integrity
+    whole_file_md5 = hashlib.md5(hash_file.read_bytes()).hexdigest()  # nosec
+    per_line_md5 = file_md5sum(hash_file)
+    assert whole_file_md5 == per_line_md5
diff --git a/tests/test_warnings.py b/tests/test_warnings.py
index 13c7157b..71e0167b 100644
--- a/tests/test_warnings.py
+++ b/tests/test_warnings.py
@@ -14,13 +14,18 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with pytest-workflow.  If not, see <https://www.gnu.org/licenses/
 
+import shutil
+import tempfile
+
 from .test_success_messages import MOO_FILE
 
 
 # Add a test to make sure no deprecation, or other warnings occur due to
 # changes in upstream libraries.
 def test_no_warnings(testdir):
+    basetemp = tempfile.mkdtemp()
     testdir.makefile(".yml", test_a=MOO_FILE)
-    result = testdir.runpytest()
+    result = testdir.runpytest("--basetemp", basetemp)
     outcomes = result.parseoutcomes()
     assert outcomes.get('warnings', 0) == 0
+    shutil.rmtree(basetemp)
diff --git a/tests/test_workflow_dependent_tests.py b/tests/test_workflow_dependent_tests.py
index 421b9bc6..48959a23 100644
--- a/tests/test_workflow_dependent_tests.py
+++ b/tests/test_workflow_dependent_tests.py
@@ -83,7 +83,7 @@ def test_workflow_dir_arg(test, testdir):
     testdir.makefile(".yml", test_asimple=SIMPLE_ECHO)
     testdir.makefile(".py", test_fixture=test)
     result = testdir.runpytest()
-    result.assert_outcomes(passed=5, failed=0, error=0, skipped=0)
+    result.assert_outcomes(passed=5, failed=0, errors=0, skipped=0)
 
 
 @pytest.mark.parametrize("test", [TEST_FIXTURE_ARGS])
@@ -161,7 +161,7 @@ def test_mark_wrong_key_with_fixture(testdir):
             "workflow marker of test_fixture.py::test_fixture_impl"
             ) in result.stdout.str()
     # Assert that no tests were run.
-    result.assert_outcomes(passed=0, failed=0, skipped=0, error=1)
+    result.assert_outcomes(passed=0, failed=0, skipped=0, errors=1)
 
 
 def test_fixture_usable_for_file_tests(testdir):
@@ -181,7 +181,7 @@ def test_fixture_usable_for_file_tests(testdir):
 
     @pytest.mark.workflow("number files")
     def test_div_by_three(workflow_dir):
-        number_file = workflow_dir / Path("123.txt")
+        number_file = workflow_dir / "123.txt"
 
         with number_file.open('rt') as file_h:
             number_file_content = file_h.read()
@@ -192,7 +192,7 @@ def test_div_by_three(workflow_dir):
     testdir.makefile(".yml", test_aworkflow=test_workflow)
     testdir.makefile(".py", test_div=test_div_by_three)
     result = testdir.runpytest("-v")
-    result.assert_outcomes(passed=4, failed=0, skipped=0, error=0)
+    result.assert_outcomes(passed=4, failed=0, skipped=0, errors=0)
 
 
 def test_same_custom_test_multiple_times(testdir):
@@ -222,14 +222,14 @@ def test_same_custom_test_multiple_times(testdir):
     @pytest.mark.workflow("one_two_three", "two_three_four",
      "three_four_five")
     def test_div_by_three(workflow_dir):
-        number_file = workflow_dir / Path("numbers.txt")
+        number_file = workflow_dir / "numbers.txt"
         assert int(number_file.read_text()) % 3 == 0
     """)
 
     testdir.makefile(".yml", test_aworkflow=test_workflow)
     testdir.makefile(".py", test_div=test_div_by_three)
     result = testdir.runpytest("-v")
-    result.assert_outcomes(passed=9, failed=0, skipped=0, error=0)
+    result.assert_outcomes(passed=9, failed=0, skipped=0, errors=0)
 
 
 TEST_WORKFLOWS = textwrap.dedent("""\
@@ -258,7 +258,7 @@ def test_div_by_three(workflow_dir):
 
 @pytest.mark.workflow("one_two_three", "two_three_five", "three_four_five")
 def test_div_by_three(workflow_dir):
-    number_file = workflow_dir / Path("numbers.txt")
+    number_file = workflow_dir / "numbers.txt"
     assert int(number_file.read_text()) % 3 == 0
 """)
 
@@ -267,7 +267,7 @@ def test_same_custom_test_multiple_times_one_error(testdir):
     testdir.makefile(".yml", test_aworkflow=TEST_WORKFLOWS)
     testdir.makefile(".py", test_div=TEST_DIV_BY_THREE)
     result = testdir.runpytest("-v")
-    result.assert_outcomes(passed=8, failed=1, skipped=0, error=0)
+    result.assert_outcomes(passed=8, failed=1, skipped=0, errors=0)
     assert ("test_div.py::test_div_by_three[two_three_five] FAILED "
             in result.stdout.str())
 
@@ -276,6 +276,6 @@ def test_custom_tests_properly_skipped(testdir):
     testdir.makefile(".yml", test_aworkflow=TEST_WORKFLOWS)
     testdir.makefile(".py", test_div=TEST_DIV_BY_THREE)
     result = testdir.runpytest("-v", "--tag", "one_two_three")
-    result.assert_outcomes(passed=3, failed=0, skipped=2, error=0)
+    result.assert_outcomes(passed=3, failed=0, skipped=2, errors=0)
     assert ("test_div.py::test_div_by_three[two_three_five] SKIPPED "
             in result.stdout.str())