Repository: EducationalTestingService/rsmtool (python)

Issue:
Refactor CLI tests to call `main()` functions instead of calling the entry points 
CodeCov wants people to switch to their [binary uploader](https://docs.codecov.com/docs/codecov-uploader) instead of using the `codecov` package. However, in my tests (as part of #636), I found that when I switch to the new uploader, we were losing coverage – mainly because the [coverage subprocess measurement](https://coverage.readthedocs.io/en/7.2.7/subprocess.html) was not working as intended. 

Since we can't stay on the codecov python package forever, we will need to refactor our CLI tests so that instead of calling the entry point directly via `subprocess()`, we call the `main` function as we do in SKLL. 

There are two tasks in this issue: 
- [ ] Refactor CLI tests
- [ ] Switch to the binary CodeCov uploader 

Pull request:
Refactor CLI tests to modernize codecov
- All tools main functions now take arguments. This is needed to refactor the CLI tests so that they call the main functions directly instead of using subprocess calls.
- Refactor the CLI tests since we want to be able to use the command-line `codecov` tool which does not measure subprocess coverage.
- Switch to `codecov` binary in the Gitlab CI plan and install `curl` so we can download that binary in the first place.
- Ignore cleanup errors in `test_cli.py`.

Notes:
1. A side-benefit of this change is that `test_cli.py` now runs 3x faster since we aren't using subprocesses.
2. The coverage change is expected since we are changing the total number of lines in the code.

Closes #638. 


Patch:
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0e642c7e..a01ddc97 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,13 +6,16 @@ stages:
 variables:
   PYVERSION: "3.8"
   BINPATH: "/root/rsmenv/bin"
+  LOGCAPTURE_LEVEL: "WARNING"
   CODECOV_TOKEN: "488304b7-b1c5-4fc7-bfb6-9e7cbcb36a08"
 
 # set up basic job
 .runtests:
   before_script:
-    - "mamba create --prefix /root/rsmenv -c conda-forge -c ets --file requirements.txt python=${PYVERSION} --yes"
+    - "mamba create --prefix /root/rsmenv -c conda-forge -c ets --file requirements.txt python=${PYVERSION} curl --yes"
     - /root/rsmenv/bin/pip install -e .
+    - /root/rsmenv/bin/curl -o /root/rsmenv/bin/codecov https://uploader.codecov.io/latest/linux/codecov
+    - chmod +x /root/rsmenv/bin/codecov
     - echo "import os" > sitecustomize.py
     - echo "try:" >> sitecustomize.py
     - echo "    import coverage" >> sitecustomize.py
@@ -22,6 +25,7 @@ variables:
     - echo "    pass" >> sitecustomize.py
   script:
     - "/root/rsmenv/bin/nose2 -s tests ${TESTFILES}"
+    - "/root/rsmenv/bin/coverage xml"
   after_script:
     - /root/rsmenv/bin/codecov
 
diff --git a/requirements.txt b/requirements.txt
index 1928e886..35848356 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,3 @@
-codecov
 doc2dash
 ipython
 jupyter
diff --git a/rsmtool/rsmcompare.py b/rsmtool/rsmcompare.py
index 58158171..ca2697bf 100644
--- a/rsmtool/rsmcompare.py
+++ b/rsmtool/rsmcompare.py
@@ -178,7 +178,11 @@ def run_comparison(config_file_or_obj_or_dict, output_dir):
     )
 
 
-def main():  # noqa: D103
+def main(argv=None):  # noqa: D103
+    # if no arguments are passed, then use sys.argv
+    if argv is None:
+        argv = sys.argv[1:]
+
     # set up the basic logging configuration
     formatter = LogFormatter()
 
@@ -200,22 +204,22 @@ def main():  # noqa: D103
     parser = setup_rsmcmd_parser("rsmcompare", uses_output_directory=True, uses_subgroups=True)
 
     # if we have no arguments at all then just show the help message
-    if len(sys.argv) < 2:
-        sys.argv.append("-h")
+    if len(argv) < 1:
+        argv.append("-h")
 
     # if the first argument is not one of the valid sub-commands
     # or one of the valid optional arguments, then assume that they
     # are arguments for the "run" sub-command. This allows the
     # old style command-line invocations to work without modification.
-    if sys.argv[1] not in VALID_PARSER_SUBCOMMANDS + [
+    if argv[0] not in VALID_PARSER_SUBCOMMANDS + [
         "-h",
         "--help",
         "-V",
         "--version",
     ]:
-        args_to_pass = ["run"] + sys.argv[1:]
+        args_to_pass = ["run"] + argv
     else:
-        args_to_pass = sys.argv[1:]
+        args_to_pass = argv
     args = parser.parse_args(args=args_to_pass)
 
     # call the appropriate function based on which sub-command was run
diff --git a/rsmtool/rsmeval.py b/rsmtool/rsmeval.py
index 930b38ca..38be9d9e 100644
--- a/rsmtool/rsmeval.py
+++ b/rsmtool/rsmeval.py
@@ -219,7 +219,11 @@ def run_evaluation(
     reporter.create_report(pred_analysis_config, csvdir, figdir, context="rsmeval")
 
 
-def main():  # noqa: D103
+def main(argv=None):  # noqa: D103
+    # if no arguments are passed, then use sys.argv
+    if argv is None:
+        argv = sys.argv[1:]
+
     # set up the basic logging configuration
     formatter = LogFormatter()
 
@@ -246,22 +250,22 @@ def main():  # noqa: D103
     )
 
     # if we have no arguments at all then just show the help message
-    if len(sys.argv) < 2:
-        sys.argv.append("-h")
+    if len(argv) < 1:
+        argv.append("-h")
 
     # if the first argument is not one of the valid sub-commands
     # or one of the valid optional arguments, then assume that they
     # are arguments for the "run" sub-command. This allows the
     # old style command-line invocations to work without modification.
-    if sys.argv[1] not in VALID_PARSER_SUBCOMMANDS + [
+    if argv[0] not in VALID_PARSER_SUBCOMMANDS + [
         "-h",
         "--help",
         "-V",
         "--version",
     ]:
-        args_to_pass = ["run"] + sys.argv[1:]
+        args_to_pass = ["run"] + argv
     else:
-        args_to_pass = sys.argv[1:]
+        args_to_pass = argv
     args = parser.parse_args(args=args_to_pass)
 
     # call the appropriate function based on which sub-command was run
diff --git a/rsmtool/rsmexplain.py b/rsmtool/rsmexplain.py
index c4010516..563a5ae2 100644
--- a/rsmtool/rsmexplain.py
+++ b/rsmtool/rsmexplain.py
@@ -525,8 +525,12 @@ def generate_report(explanation, output_dir, ids, configuration, logger=None, wa
     reporter.create_explanation_report(configuration, csvdir, reportdir)
 
 
-def main():
+def main(argv=None):
     """Run rsmexplain and generate explanation reports."""
+    # if no arguments are passed, then use sys.argv
+    if argv is None:
+        argv = sys.argv[1:]
+
     # set up the basic logging configuration
     formatter = LogFormatter()
 
@@ -546,22 +550,22 @@ def main():
     parser = setup_rsmcmd_parser("rsmexplain", uses_output_directory=True, allows_overwriting=True)
 
     # if no arguments provided then just show the help message
-    if len(sys.argv) < 2:
-        sys.argv.append("-h")
+    if len(argv) < 1:
+        argv.append("-h")
 
     # if the first argument is not one of the valid sub-commands
     # or one of the valid optional arguments, then assume that they
     # are arguments for the "run" sub-command. This allows the
     # old style command-line invocations to work without modification.
-    if sys.argv[1] not in VALID_PARSER_SUBCOMMANDS + [
+    if argv[0] not in VALID_PARSER_SUBCOMMANDS + [
         "-h",
         "--help",
         "-V",
         "--version",
     ]:
-        args_to_pass = ["run"] + sys.argv[1:]
+        args_to_pass = ["run"] + argv
     else:
-        args_to_pass = sys.argv[1:]
+        args_to_pass = argv
     args = parser.parse_args(args=args_to_pass)
 
     if args.subcommand == "run":
diff --git a/rsmtool/rsmpredict.py b/rsmtool/rsmpredict.py
index eb697f37..eac7c264 100644
--- a/rsmtool/rsmpredict.py
+++ b/rsmtool/rsmpredict.py
@@ -505,7 +505,11 @@ def compute_and_save_predictions(
         )
 
 
-def main():  # noqa: D103
+def main(argv=None):  # noqa: D103
+    # if no arguments are passed, then use sys.argv
+    if argv is None:
+        argv = sys.argv[1:]
+
     # set up the basic logging configuration
     formatter = LogFormatter()
 
@@ -540,22 +544,22 @@ def main():  # noqa: D103
     )
 
     # if we have no arguments at all then just show the help message
-    if len(sys.argv) < 2:
-        sys.argv.append("-h")
+    if len(argv) < 1:
+        argv.append("-h")
 
     # if the first argument is not one of the valid sub-commands
     # or one of the valid optional arguments, then assume that they
     # are arguments for the "run" sub-command. This allows the
     # old style command-line invocations to work without modification.
-    if sys.argv[1] not in VALID_PARSER_SUBCOMMANDS + [
+    if argv[0] not in VALID_PARSER_SUBCOMMANDS + [
         "-h",
         "--help",
         "-V",
         "--version",
     ]:
-        args_to_pass = ["run"] + sys.argv[1:]
+        args_to_pass = ["run"] + argv
     else:
-        args_to_pass = sys.argv[1:]
+        args_to_pass = argv
     args = parser.parse_args(args=args_to_pass)
 
     # call the appropriate function based on which sub-command was run
diff --git a/rsmtool/rsmsummarize.py b/rsmtool/rsmsummarize.py
index da4c8fe8..46daa81b 100644
--- a/rsmtool/rsmsummarize.py
+++ b/rsmtool/rsmsummarize.py
@@ -235,7 +235,11 @@ def run_summary(
     reporter.create_summary_report(configuration, all_experiments, csvdir)
 
 
-def main():  # noqa: D103
+def main(argv=None):  # noqa: D103
+    # if no arguments are passed, then use sys.argv
+    if argv is None:
+        argv = sys.argv[1:]
+
     # set up the basic logging configuration
     formatter = LogFormatter()
 
@@ -259,22 +263,22 @@ def main():  # noqa: D103
     )
 
     # if we have no arguments at all then just show the help message
-    if len(sys.argv) < 2:
-        sys.argv.append("-h")
+    if len(argv) < 1:
+        argv.append("-h")
 
     # if the first argument is not one of the valid sub-commands
     # or one of the valid optional arguments, then assume that they
     # are arguments for the "run" sub-command. This allows the
     # old style command-line invocations to work without modification.
-    if sys.argv[1] not in VALID_PARSER_SUBCOMMANDS + [
+    if argv[0] not in VALID_PARSER_SUBCOMMANDS + [
         "-h",
         "--help",
         "-V",
         "--version",
     ]:
-        args_to_pass = ["run"] + sys.argv[1:]
+        args_to_pass = ["run"] + argv
     else:
-        args_to_pass = sys.argv[1:]
+        args_to_pass = argv
     args = parser.parse_args(args=args_to_pass)
 
     # call the appropriate function based on which sub-command was run
diff --git a/rsmtool/rsmtool.py b/rsmtool/rsmtool.py
index f68c8d7a..de602d31 100644
--- a/rsmtool/rsmtool.py
+++ b/rsmtool/rsmtool.py
@@ -347,7 +347,11 @@ def run_experiment(
     reporter.create_report(pred_analysis_config, csvdir, figdir)
 
 
-def main():  # noqa: D103
+def main(argv=None):  # noqa: D103
+    # if no arguments are passed, then use sys.argv
+    if argv is None:
+        argv = sys.argv[1:]
+
     # set up the basic logging configuration
     formatter = LogFormatter()
 
@@ -374,22 +378,22 @@ def main():  # noqa: D103
     )
 
     # if we have no arguments at all then just show the help message
-    if len(sys.argv) < 2:
-        sys.argv.append("-h")
+    if len(argv) < 1:
+        argv.append("-h")
 
     # if the first argument is not one of the valid sub-commands
     # or one of the valid optional arguments, then assume that they
     # are arguments for the "run" sub-command. This allows the
     # old style command-line invocations to work without modification.
-    if sys.argv[1] not in VALID_PARSER_SUBCOMMANDS + [
+    if argv[0] not in VALID_PARSER_SUBCOMMANDS + [
         "-h",
         "--help",
         "-V",
         "--version",
     ]:
-        args_to_pass = ["run"] + sys.argv[1:]
+        args_to_pass = ["run"] + argv
     else:
-        args_to_pass = sys.argv[1:]
+        args_to_pass = argv
     args = parser.parse_args(args=args_to_pass)
 
     # call the appropriate function based on which sub-command was run
diff --git a/rsmtool/rsmxval.py b/rsmtool/rsmxval.py
index d687b1dc..79834932 100644
--- a/rsmtool/rsmxval.py
+++ b/rsmtool/rsmxval.py
@@ -268,7 +268,11 @@ def run_cross_validation(
     )
 
 
-def main():  # noqa: D103
+def main(argv=None):  # noqa: D103
+    # if no arguments are passed, then use sys.argv
+    if argv is None:
+        argv = sys.argv[1:]
+
     # set up the basic logging configuration
     formatter = LogFormatter()
 
@@ -295,22 +299,22 @@ def main():  # noqa: D103
     )
 
     # if we have no arguments at all then just show the help message
-    if len(sys.argv) < 2:
-        sys.argv.append("-h")
+    if len(argv) < 1:
+        argv.append("-h")
 
     # if the first argument is not one of the valid sub-commands
     # or one of the valid optional arguments, then assume that they
     # are arguments for the "run" sub-command. This allows the
     # old style command-line invocations to work without modification.
-    if sys.argv[1] not in VALID_PARSER_SUBCOMMANDS + [
+    if argv[0] not in VALID_PARSER_SUBCOMMANDS + [
         "-h",
         "--help",
         "-V",
         "--version",
     ]:
-        args_to_pass = ["run"] + sys.argv[1:]
+        args_to_pass = ["run"] + argv
     else:
-        args_to_pass = sys.argv[1:]
+        args_to_pass = argv
     args = parser.parse_args(args=args_to_pass)
 
     # call the appropriate function based on which sub-command was run


Test patch:
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 74590fe9..d3683689 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -1,11 +1,20 @@
+import logging
 import os
-import shlex
-import subprocess
-import sys
 import unittest
+from contextlib import redirect_stderr, redirect_stdout
+from io import StringIO
 from pathlib import Path
+from shutil import rmtree
 from tempfile import NamedTemporaryFile, TemporaryDirectory
-
+from unittest.mock import patch
+
+from rsmtool.rsmcompare import main as rsmcompare_main
+from rsmtool.rsmeval import main as rsmeval_main
+from rsmtool.rsmexplain import main as rsmexplain_main
+from rsmtool.rsmpredict import main as rsmpredict_main
+from rsmtool.rsmsummarize import main as rsmsummarize_main
+from rsmtool.rsmtool import main as rsmtool_main
+from rsmtool.rsmxval import main as rsmxval_main
 from rsmtool.test_utils import (
     check_file_output,
     check_generated_output,
@@ -27,6 +36,19 @@ else:
 STRICT_MODE = os.environ.get("STRICT", None)
 IGNORE_WARNINGS = False if STRICT_MODE else True
 
+# create a dictionary mapping tool names to their main functions
+# NOTE: technically we could have just used `globals()` instead
+# of this dictionary, but that would not be as safe
+MAIN_FUNCTIONS = {
+    "rsmtool": rsmtool_main,
+    "rsmeval": rsmeval_main,
+    "rsmcompare": rsmcompare_main,
+    "rsmpredict": rsmpredict_main,
+    "rsmsummarize": rsmsummarize_main,
+    "rsmxval": rsmxval_main,
+    "rsmexplain": rsmexplain_main,
+}
+
 
 class TestToolCLI(unittest.TestCase):
     """Test class for ToolCLI tests."""
@@ -54,30 +76,25 @@ class TestToolCLI(unittest.TestCase):
     @classmethod
     def tearDownClass(cls):
         for tempdir in cls.temporary_directories:
-            tempdir.cleanup()
+            rmtree(tempdir.name, ignore_errors=True)
         for tempfile in cls.temporary_files:
             os.unlink(tempfile)
 
     def check_no_args(self, context):
         """Check running the tool with no arguments."""
-        # if the BINPATH environment variable is defined
-        # use that to construct the command instead of just
-        # the name; this is needed for the CI builds where
-        # we do not always activate the conda environment
-        binpath = os.environ.get("BINPATH", None)
-        if binpath is not None:
-            cmd = f"{binpath}/{context}"
-        else:
-            cmd = f"{context}"
+        # call the main function for the given context
+        # with no arguments and check that it exits
+        # with exit code 0 and prints out the usage
+        main_function = MAIN_FUNCTIONS[context]
+        with redirect_stdout(StringIO()) as captured_stdout:
+            with self.assertRaises(SystemExit) as excm:
+                main_function([])
+
+        # check that the exit code is 0
+        self.assertEqual(excm.exception.code, 0)
 
-        proc = subprocess.run(
-            shlex.split(cmd, posix="win" not in sys.platform),
-            check=False,
-            stderr=subprocess.DEVNULL,
-            stdout=subprocess.PIPE,
-        )
-        self.assertEqual(proc.returncode, 0)
-        self.assertTrue(b"usage: " + bytes(context, encoding="utf-8") in proc.stdout)
+        # check that the appropriate usage was printed out
+        self.assertTrue(f"usage: {context}" in captured_stdout.getvalue().lower())
 
     def test_no_args(self):
         # test that the tool without any arguments prints help messag
@@ -265,18 +282,16 @@ class TestToolCLI(unittest.TestCase):
             expected_output = expectedfh.read().strip()
             self.assertEqual(output, expected_output)
 
-    def check_tool_cmd(
-        self, context, subcmd, output_dir=None, working_dir=None, generate_output_file=None
-    ):
+    def check_tool_cmd(self, context, args, output_dir=None, generate_output_file=None):
         """
-        Test that the ``subcmd`` invocation for ``context`` works as expected.
+        Test that the invocation for ``context`` with ``args`` works as expected.
 
         Parameters
         ----------
         context : str
             Name of the tool being tested.
-        subcmd : str
-            The tool command-line invocation that is being tested.
+        args : List[str]
+            The list of arguments to be passed to the tool.
         output_dir : str, optional
             Directory containing the output for "run" subcommands.
             Will be ``None`` for "generate" subcommands.
@@ -290,43 +305,35 @@ class TestToolCLI(unittest.TestCase):
            written to a specific output file.
            Defaults to ``None``.
         """
-        # if the BINPATH environment variable is defined
-        # use that to construct the command instead of just
-        # the name; this is needed for the CI builds where
-        # we do not always activate the conda environment
-        binpath = os.environ.get("BINPATH", None)
-        if binpath is not None:
-            cmd = f"{binpath}/{context} {subcmd}"
-        else:
-            cmd = f"{context} {subcmd}"
+        # get the main function for the given context
+        main_function = MAIN_FUNCTIONS[context]
+
+        # increase the logging level for the tool's logger since we don't
+        # want to see the logging messages when we run the tool for this test
+        context_logger = logging.getLogger(f"rsmtool.{context}")
+        context_logger.setLevel(logging.CRITICAL)
+
+        # do the same for the `utils.commandline` logger since we don't want to
+        # see the logging messages for configuration generation either
+        commandline_logger = logging.getLogger("rsmtool.utils.commandline")
+        commandline_logger.setLevel(logging.CRITICAL)
+
+        # call the main function for the given context with the arguments
+        # but redirect STDOUT and STDERR to StringIO objects
+        with redirect_stdout(StringIO()) as captured_stdout, redirect_stderr(StringIO()):
+            main_function(args)
+
+        # restore the logging level for both loggers
+        context_logger.setLevel(logging.INFO)
+        commandline_logger.setLevel(logging.INFO)
 
         # run different checks depending on the given command type
-        cmd_type = "generate" if " generate" in cmd else "run"
+        cmd_type = "generate" if "generate" in args else "run"
         if cmd_type == "run":
-            # for run subcommands, we can ignore the messages printed to stdout
-            proc = subprocess.run(
-                shlex.split(cmd, posix="win" not in sys.platform),
-                check=True,
-                cwd=working_dir,
-                stderr=subprocess.PIPE,
-                stdout=subprocess.DEVNULL,
-                encoding="utf-8",
-            )
-            # then check that the commmand ran successfully
-            self.assertTrue(proc.returncode == 0)
-            # and, finally, that the output was as expected
+            # check that the output generated is as expected
             self.validate_run_output(context, output_dir)
         else:
-            # for generate subcommands, we ignore the warnings printed to stderr
-            subgroups = "--subgroups" in cmd
-            proc = subprocess.run(
-                shlex.split(cmd, posix="win" not in sys.platform),
-                check=True,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-                encoding="utf-8",
-            )
-            self.assertTrue(proc.returncode == 0)
+            subgroups = "--subgroups" in args
             # if an output file was specified, we get the output to validate
             # by reading that in instead of from STDOUT; in that case STDOUT
             # only contains a message which we can also validate
@@ -334,7 +341,7 @@ class TestToolCLI(unittest.TestCase):
                 with open(generate_output_file, "r") as outfh:
                     output_to_check = outfh.read().strip()
             else:
-                output_to_check = proc.stdout.strip()
+                output_to_check = captured_stdout.getvalue().strip()
 
             self.validate_generate_output(context, output_to_check, subgroups=subgroups)
 
@@ -357,8 +364,61 @@ class TestToolCLI(unittest.TestCase):
 
             # and test the default subcommand
             config_file = getattr(self, f"{context}_config_file")
-            subcmd = f"{config_file} {tempdir.name}"
-            yield self.check_tool_cmd, context, subcmd, tempdir.name, None, None
+            args = [str(config_file), tempdir.name]
+            yield self.check_tool_cmd, context, args, tempdir.name, None
+
+    def check_run_without_output_directory(self, context, args):
+        """
+        Check that the "run" subcommand works without an output directory.
+
+        Parameters
+        ----------
+        context : str
+            Name of the tool being tested.
+        args : List[str]
+            The list of arguments to be passed to the tool.
+        """
+        # get the current working directory
+        current_dir = os.getcwd()
+
+        # patch the appropriate runner function for the tool being tested
+        if context == "rsmtool":
+            patcher = patch("rsmtool.rsmtool.run_experiment")
+        elif context == "rsmeval":
+            patcher = patch("rsmtool.rsmeval.run_evaluation")
+        elif context == "rsmcompare":
+            patcher = patch("rsmtool.rsmcompare.run_comparison")
+        elif context == "rsmsummarize":
+            patcher = patch("rsmtool.rsmsummarize.run_summary")
+        elif context == "rsmxval":
+            patcher = patch("rsmtool.rsmxval.run_cross_validation")
+        elif context == "rsmexplain":
+            patcher = patch("rsmtool.rsmexplain.generate_explanation")
+
+        # start the patcher
+        mocked_runner = patcher.start()
+
+        # increase the logging level for the tool's logger since we don't
+        # want to see the logging messages when we run the tool for this test
+        context_logger = logging.getLogger(f"rsmtool.{context}")
+        context_logger.setLevel(logging.CRITICAL)
+
+        # call the main function for the given context with the arguments
+        main_function = MAIN_FUNCTIONS[context]
+        main_function(args)
+
+        # restore the logging level for the tool's logger
+        context_logger.setLevel(logging.INFO)
+
+        # stop the patcher
+        patcher.stop()
+
+        # check that the runner was called with the right arguments
+        # including the current working directory as the output directory
+        config_file = mocked_runner.call_args[0][0]
+        output_dir = mocked_runner.call_args[0][1]
+        self.assertEqual(config_file, str(config_file))
+        self.assertEqual(output_dir, current_dir)
 
     def test_run_without_output_directory(self):
         # test that "run" subcommand works without an output directory
@@ -378,27 +438,27 @@ class TestToolCLI(unittest.TestCase):
 
             # and test the run subcommand without an output directory
             config_file = getattr(self, f"{context}_config_file")
-            subcmd = f"run {config_file}"
-            # we call check_tool_cmd with a working directory here to simulate
-            # the usage of the current working directory when the output directory
-            # is not specified
-            yield self.check_tool_cmd, context, subcmd, tempdir.name, tempdir.name, None
+            args = ["run", str(config_file)]
+            yield self.check_run_without_output_directory, context, args
 
-    def check_run_bad_overwrite(self, cmd):
+    def check_run_bad_overwrite(self, context, args):
         """Check that the overwriting error is raised properly."""
-        with self.assertRaises(subprocess.CalledProcessError) as e:
-            _ = subprocess.run(
-                shlex.split(cmd, posix="win" not in sys.platform),
-                check=True,
-                stderr=subprocess.DEVNULL,
-                stdout=subprocess.DEVNULL,
-            )
-            self.assertTrue("already contains" in e.msg)
-            self.assertTrue("OSError" in e.msg)
+        # increase the logging level for the tool's logger since we don't
+        # want to see the logging messages when we run the tool for this test
+        context_logger = logging.getLogger(f"rsmtool.{context}")
+        context_logger.setLevel(logging.CRITICAL)
+
+        # run the command and check that it fails with the expected error that
+        # the output directory already contains a non-empty output sub-directory
+        with self.assertRaisesRegex(OSError, "already contains a non-empty"):
+            main_function = MAIN_FUNCTIONS[context]
+            main_function(args)
+
+        # restore the logging level for the tool's logger
+        context_logger.setLevel(logging.INFO)
 
     def test_run_bad_overwrite(self):
         # test that the "run" command fails to overwrite when "-f" is not specified
-
         # this applies to all tools except rsmpredict, rsmcompare, and rsmxval
         for context in ["rsmtool", "rsmeval", "rsmsummarize", "rsmexplain"]:
             tempdir = TemporaryDirectory()
@@ -409,17 +469,10 @@ class TestToolCLI(unittest.TestCase):
             fake_file = Path(tempdir.name) / "output" / "foo.csv"
             fake_file.touch()
 
+            # now run the tool again and check that it fails
             config_file = getattr(self, f"{context}_config_file")
-            # if the BINPATH environment variable is defined
-            # use that to construct the command instead of just
-            # the name; this is needed for the CI builds where
-            # we do not always activate the conda environment
-            binpath = os.environ.get("BINPATH", None)
-            if binpath is not None:
-                cmd = f"{binpath}/{context} {config_file} {tempdir.name}"
-            else:
-                cmd = f"{context} {config_file} {tempdir.name}"
-            yield self.check_run_bad_overwrite, cmd
+            args = [str(config_file), tempdir.name]
+            yield self.check_run_bad_overwrite, context, args
 
     def test_run_good_overwrite(self):
         #  test that the "run" command does overwrite when "-f" is specified
@@ -435,19 +488,21 @@ class TestToolCLI(unittest.TestCase):
             fake_file.touch()
 
             config_file = getattr(self, f"{context}_config_file")
-            subcmd = f"{config_file} {tempdir.name} -f"
-            yield self.check_tool_cmd, context, subcmd, tempdir.name, None, None
+            args = [str(config_file), tempdir.name, "-f"]
+            yield self.check_tool_cmd, context, args, tempdir.name, None
 
     def test_rsmpredict_run_features_file(self):
         tempdir = TemporaryDirectory()
         self.temporary_directories.append(tempdir)
 
-        subcmd = (
-            f"{self.rsmpredict_config_file} {tempdir.name} "
-            f"--features {tempdir.name}/preprocessed_features.csv"
-        )
+        args = [
+            str(self.rsmpredict_config_file),
+            tempdir.name,
+            "--features",
+            f"{tempdir.name}/preprocessed_features.csv",
+        ]
 
-        self.check_tool_cmd("rsmpredict", subcmd, tempdir.name, None)
+        self.check_tool_cmd("rsmpredict", args, tempdir.name, None)
 
         # check the features file separately
         file1 = Path(tempdir.name) / "preprocessed_features.csv"
@@ -467,7 +522,7 @@ class TestToolCLI(unittest.TestCase):
             "rsmxval",
             "rsmexplain",
         ]:
-            yield self.check_tool_cmd, context, "generate", None, None, None
+            yield self.check_tool_cmd, context, ["generate"], None, None
 
     def test_generate_with_groups(self):
         # test that the "generate --subgroups" subcommand for all tools works
@@ -477,7 +532,7 @@ class TestToolCLI(unittest.TestCase):
         # rsmxval and rsmexplain; rsmxval does support subgroups but since it has no sections
         # fields, it's not relevant for this test
         for context in ["rsmtool", "rsmeval", "rsmcompare"]:
-            yield self.check_tool_cmd, context, "generate --subgroups", None, None, None
+            yield self.check_tool_cmd, context, ["generate", "--subgroups"], None, None
 
     def test_generate_with_output_file(self):
         # test that the "generate --output <file>" subcommand for all tools works
@@ -497,8 +552,7 @@ class TestToolCLI(unittest.TestCase):
             yield (
                 self.check_tool_cmd,
                 context,
-                f"generate --output {tempf.name}",
-                None,
+                ["generate", "--output", tempf.name],
                 None,
                 tempf.name,
             )
@@ -508,8 +562,8 @@ class TestToolCLI(unittest.TestCase):
         # as expected when written to output files
 
         # this applies to all tools except rsmpredict, rsmsummarize,
-        # rsmxval and rsmexplain; rsmxval does support subgroups but since it has no sections
-        # fields, it's not relevant for this test
+        # rsmxval and rsmexplain; rsmxval does support subgroups but
+        # since it has no sections fields, it's not relevant for this test
         for context in ["rsmtool", "rsmeval", "rsmcompare"]:
             tempf = NamedTemporaryFile(mode="w", suffix=".json", delete=False)
             tempf.close()
@@ -517,8 +571,7 @@ class TestToolCLI(unittest.TestCase):
             yield (
                 self.check_tool_cmd,
                 context,
-                f"generate --subgroups --output {tempf.name}",
-                None,
+                ["generate", "--subgroups", "--output", tempf.name],
                 None,
                 tempf.name,
             )