Repository: maciej-gol/pytest-bisect-tests

Issue:
Pytest collection failure
When pytest collection fails, the user has no way to discover what went wrong.

The following is what the user sees:
```
Traceback (most recent call last):
  File "/home/holmanb/.venv/bin/pytest-bisect-tests", line 8, in <module>
    sys.exit(main())
             ^^^^^^
  File "/home/holmanb/git/pytest-bisect-tests/pytest_bisect_tests/cli.py", line 42, in main
    test_names=pytest_runner.collect_tests(),
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/holmanb/git/pytest-bisect-tests/pytest_bisect_tests/pytest_runner.py", line 48, in collect_tests
    raise RuntimeError(
RuntimeError: Failed to collect tests
```

It would be nice to output pytest stderr when this happens.

Discussion:

Patch:
diff --git a/integration_data/collection_error.py b/integration_data/collection_error.py
new file mode 100644
index 0000000..4f33a9a
--- /dev/null
+++ b/integration_data/collection_error.py
@@ -0,0 +1,4 @@
+raise Exception("This is a test collection error")
+
+def test_ok():
+    ...


Test patch:
diff --git a/pytest_bisect_tests/cli.py b/pytest_bisect_tests/cli.py
index 7e2edcf..3101fcb 100644
--- a/pytest_bisect_tests/cli.py
+++ b/pytest_bisect_tests/cli.py
@@ -1,8 +1,10 @@
 import argparse
+import sys
 from pytest_bisect_tests.bisect import run_bisect
 
 from pytest_bisect_tests.pytest_runner import (
     PytestRunner,
+    TestCollectionError,
 )
 
 
@@ -34,8 +36,17 @@ def main() -> None:
         collect_options=args.collect_options,
         stdout=args.stdout,
     )
+    try:
+        test_names = pytest_runner.collect_tests()
+    except TestCollectionError:
+        if not args.stdout:
+            print("Failed to collect tests. Use --stdout to see the output from pytest test collection.")
+        else:
+            print("Failed to collect tests. Consult output from pytest.")
+        sys.exit(1)
+
     result = run_bisect(
-        test_names=pytest_runner.collect_tests(),
+        test_names=test_names,
         failing_test=args.failing_test,
         test_runner=pytest_runner.run,
     )
diff --git a/pytest_bisect_tests/pytest_runner.py b/pytest_bisect_tests/pytest_runner.py
index 2424f1b..cf5b47b 100644
--- a/pytest_bisect_tests/pytest_runner.py
+++ b/pytest_bisect_tests/pytest_runner.py
@@ -4,6 +4,9 @@ import shlex
 import subprocess
 from typing import List, Optional
 
+class TestCollectionError(Exception):
+    pass
+
 
 class PytestRunner:
     def __init__(
@@ -42,7 +45,7 @@ class PytestRunner:
             with os.fdopen(r, closefd=False) as f:
                 tests = [l.strip() for l in f.readlines()]
             if process.wait() != 0:
-                raise RuntimeError("Failed to collect tests.")
+                raise TestCollectionError("Failed to collect tests.")
             return tests
         finally:
             os.closerange(r, w)
diff --git a/tests/integration_test.py b/tests/integration_test.py
index 3ce68e8..05c9c9b 100644
--- a/tests/integration_test.py
+++ b/tests/integration_test.py
@@ -11,11 +11,13 @@ HERE = Path(__file__).parent
 
 
 def _standalone_caller(
-    failing_test: str, collect_options: str, run_options: str
+    failing_test: str, collect_options: str, run_options: str, stdout: bool = False
 ) -> Tuple[str, int]:
     args = ["pytest-bisect-tests", "--failing-test", failing_test]
     if collect_options:
         args.extend(["--collect-options", collect_options])
+    if stdout:
+        args.append("--stdout")
     if run_options:
         args.extend(["--run-options", run_options])
 
@@ -91,3 +93,26 @@ def test_should_work_with_items_modifying_plugins(plugin_caller: PluginCaller) -
 
     assert "Faulty test: integration_data/test_groups_test.py::test_faulty" in out
     assert code == 0
+
+def test_cli_should_communicate_stdout_argument() -> None:
+    out, code = _standalone_caller(
+        failing_test="integration_data/collection_error.py::test_non_existing_fixture",
+        collect_options="",
+        run_options="collection_error.py",
+    )
+
+    assert code == 1
+    assert "Failed to collect tests. Use --stdout to see the output from pytest test collection." in out
+
+
+def test_cli_should_show_output_from_collection_with_stdout() -> None:
+    out, code = _standalone_caller(
+        failing_test="integration_data/collection_error.py::test_non_existing_fixture",
+        collect_options="",
+        run_options="collection_error.py",
+        stdout=True,
+    )
+
+    assert code == 1
+    assert "Exception: This is a test collection error" in out
+    assert "Failed to collect tests. Consult output from pytest." in out