Repository: pantsbuild/pants

Issue:
Out of band test (de-)selection can fail new python backend test runs
Both old and new python backends support passing args to `py.test` directly. If `-k` is used to sub-select tests to run from the given target set, a run can fail in the new backend if:
1. All tests are de-selected (`-k` intersects no tests in the target set).
2. All tests for a single target are de-selected and the test run mode is `--no-fast`.

This behavior contradicts our own internal short-circuiting of `py.test` executions with syntesized `0` exit codes when we detect no test targets or no test sources up-front.


Discussion:
Found this working #4587
I also noticed that test targets with 0 discoverable tests (similar to the `-k intersects no tests` mode) now present as a failure. previously, these would run without failure in the v1 python backend.. tho I don't particularly see a problem with that, it appears related.
Yup, this is a behavior change in the newer `py.test` used by the new backend.

Resolving patch:
diff --git a/src/python/pants/backend/python/tasks2/pytest_run.py b/src/python/pants/backend/python/tasks2/pytest_run.py
--- a/src/python/pants/backend/python/tasks2/pytest_run.py
+++ b/src/python/pants/backend/python/tasks2/pytest_run.py
@@ -34,17 +34,32 @@
 from pants.util.xml_parser import XmlParser
 
 
-class PythonTestResult(object):
+class PytestResult(object):
   @staticmethod
   def exception():
-    return PythonTestResult('EXCEPTION')
+    return PytestResult('EXCEPTION')
 
-  @staticmethod
-  def rc(value):
-    return PythonTestResult('SUCCESS' if value == 0 else 'FAILURE', rc=value)
+  _SUCCESS_EXIT_CODES = (
+    0,
+
+    # This is returned by pytest when no tests are collected (EXIT_NOTESTSCOLLECTED).
+    # We already short-circuit test runs with no test _targets_ to return 0 emulated exit codes and
+    # we should do the same for cases when there are test targets but tests themselves have been
+    # de-selected out of band via `py.test -k`.
+    5
+  )
+
+  @classmethod
+  def _map_exit_code(cls, value):
+    return 0 if value in cls._SUCCESS_EXIT_CODES else value
+
+  @classmethod
+  def rc(cls, value):
+    exit_code = cls._map_exit_code(value)
+    return PytestResult('SUCCESS' if exit_code == 0 else 'FAILURE', rc=exit_code)
 
   def with_failed_targets(self, failed_targets):
-    return PythonTestResult(self._msg, self._rc, failed_targets)
+    return PytestResult(self._msg, self._rc, failed_targets)
 
   def __init__(self, msg, rc=None, failed_targets=None):
     self._rc = rc
@@ -422,7 +437,7 @@ def _do_run_tests_with_args(self, pex, args):
       with self.context.new_workunit(name='run',
                                      labels=[WorkUnitLabel.TOOL, WorkUnitLabel.TEST]) as workunit:
         rc = self._spawn_and_wait(pex, workunit=workunit, args=args, setsid=True, env=env)
-        return PythonTestResult.rc(rc)
+        return PytestResult.rc(rc)
     except ErrorWhileTesting:
       # _spawn_and_wait wraps the test runner in a timeout, so it could
       # fail with a ErrorWhileTesting. We can't just set PythonTestResult
@@ -433,7 +448,7 @@ def _do_run_tests_with_args(self, pex, args):
     except Exception:
       self.context.log.error('Failed to run test!')
       self.context.log.info(traceback.format_exc())
-      return PythonTestResult.exception()
+      return PytestResult.exception()
 
   def _map_relsrc_to_targets(self, targets):
     pex_src_root = os.path.relpath(
@@ -495,7 +510,7 @@ def _run_tests(self, targets):
 
   def _do_run_tests(self, targets):
     if not targets:
-      return PythonTestResult.rc(0)
+      return PytestResult.rc(0)
 
     buildroot = get_buildroot()
     source_chroot = os.path.relpath(
@@ -506,7 +521,7 @@ def _do_run_tests(self, targets):
         sources_map[os.path.join(source_chroot, p)] = os.path.join(t.target_base, p)
 
     if not sources_map:
-      return PythonTestResult.rc(0)
+      return PytestResult.rc(0)
 
     with self._test_runner(targets, sources_map) as (pex, test_args):
       # Validate that the user didn't provide any passthru args that conflict


Test patch:
diff --git a/tests/python/pants_test/backend/python/tasks/test_pytest_run.py b/tests/python/pants_test/backend/python/tasks/test_pytest_run.py
--- a/tests/python/pants_test/backend/python/tasks/test_pytest_run.py
+++ b/tests/python/pants_test/backend/python/tasks/test_pytest_run.py
@@ -24,21 +24,21 @@ class PythonTestBuilderTestBase(PythonTaskTestBase):
   def task_type(cls):
     return PytestRun
 
-  def run_tests(self, targets, **options):
+  def run_tests(self, targets, *passthrough_args, **options):
     test_options = {
       'colors': False,
       'level': 'info'  # When debugging a test failure it may be helpful to set this to 'debug'.
     }
     test_options.update(options)
     self.set_options(**test_options)
-    context = self.context(target_roots=targets)
+    context = self.context(target_roots=targets, passthru_args=list(passthrough_args))
     pytest_run_task = self.create_task(context)
     with pushd(self.build_root):
       pytest_run_task.execute()
 
-  def run_failing_tests(self, targets, failed_targets, **options):
+  def run_failing_tests(self, targets, failed_targets, *passthrough_args, **options):
     with self.assertRaises(ErrorWhileTesting) as cm:
-      self.run_tests(targets=targets, **options)
+      self.run_tests(targets=targets, *passthrough_args, **options)
     self.assertEqual(set(failed_targets), set(cm.exception.failed_targets))
 
 
@@ -262,6 +262,12 @@ def test_error_outside_function(self):
   def test_green(self):
     self.run_tests(targets=[self.green])
 
+  def test_out_of_band_deselect_fast_success(self):
+    self.run_tests([self.green, self.red], '-kno_tests_should_match_at_all', fast=True)
+
+  def test_out_of_band_deselect_no_fast_success(self):
+    self.run_tests([self.green, self.red], '-ktest_core_green', fast=False)
+
   def test_red(self):
     self.run_failing_tests(targets=[self.red], failed_targets=[self.red])
 
diff --git a/tests/python/pants_test/backend/python/tasks2/test_pytest_run.py b/tests/python/pants_test/backend/python/tasks2/test_pytest_run.py
--- a/tests/python/pants_test/backend/python/tasks2/test_pytest_run.py
+++ b/tests/python/pants_test/backend/python/tasks2/test_pytest_run.py
@@ -28,23 +28,23 @@ def task_type(cls):
 
   _CONFTEST_CONTENT = '# I am an existing root-level conftest file.'
 
-  def run_tests(self, targets, **options):
+  def run_tests(self, targets, *passthru_args, **options):
     """Run the tests in the specified targets, with the specified PytestRun task options.
 
     Returns the path of the sources pex, so that calling code can map files from the
     source tree to files as pytest saw them.
     """
-    context = self._prepare_test_run(targets, **options)
+    context = self._prepare_test_run(targets, *passthru_args, **options)
     self._do_run_tests(context)
     return context
 
-  def run_failing_tests(self, targets, failed_targets, **options):
-    context = self._prepare_test_run(targets, **options)
+  def run_failing_tests(self, targets, failed_targets, *passthru_args, **options):
+    context = self._prepare_test_run(targets, *passthru_args, **options)
     with self.assertRaises(ErrorWhileTesting) as cm:
       self._do_run_tests(context)
     self.assertEqual(set(failed_targets), set(cm.exception.failed_targets))
 
-  def _prepare_test_run(self, targets, **options):
+  def _prepare_test_run(self, targets, *passthru_args, **options):
     self.reset_build_graph()
     test_options = {
       'colors': False,
@@ -59,7 +59,8 @@ def _prepare_test_run(self, targets, **options):
     rr_task_type = self.synthesize_task_subtype(ResolveRequirements, 'rr_scope')
     gs_task_type = self.synthesize_task_subtype(GatherSources, 'gs_scope')
     context = self.context(for_task_types=[si_task_type, rr_task_type, gs_task_type],
-                           target_roots=targets)
+                           target_roots=targets,
+                           passthru_args=list(passthru_args))
     si_task_type(context, os.path.join(self.pants_workdir, 'si')).execute()
     rr_task_type(context, os.path.join(self.pants_workdir, 'rr')).execute()
     gs_task_type(context, os.path.join(self.pants_workdir, 'gs')).execute()
@@ -305,6 +306,12 @@ def test_error_outside_function(self):
   def test_green(self):
     self.run_tests(targets=[self.green])
 
+  def test_out_of_band_deselect_fast_success(self):
+    self.run_tests([self.green, self.red], '-kno_tests_should_match_at_all', fast=True)
+
+  def test_out_of_band_deselect_no_fast_success(self):
+    self.run_tests([self.green, self.red], '-ktest_core_green', fast=False)
+
   def test_red(self):
     self.run_failing_tests(targets=[self.red], failed_targets=[self.red])