Repository: pantsbuild/pants

Issue:
for non-test-related pytest failures, pants still tries to parse junit.xml
On Pants v1.3.0, in `src/python/pants/backend/python/tasks2/pytest_run.py`, the `_run_pytest` method will invoke the `_get_failed_targets_from_junitxml` method regardless of whether pex got far enough to even invoke tests. This results in an irrelevant error message that pants had an "Error parsing xml file" when in fact the subprocess invocation never got that far to even contemplate generating the junit XML output.

For example, if pex fails to find pytest, then pytest will never be invoked. The junit XML output won't exist in this case. 


Discussion:

Resolving patch:
diff --git a/src/python/pants/backend/python/tasks2/pytest_run.py b/src/python/pants/backend/python/tasks2/pytest_run.py
--- a/src/python/pants/backend/python/tasks2/pytest_run.py
+++ b/src/python/pants/backend/python/tasks2/pytest_run.py
@@ -512,14 +512,17 @@ def _run_tests(self, targets):
     partitions = self._partition(targets)
 
     results = {}
+    failure = False
     for partition in partitions:
       try:
         rv = self._do_run_tests(partition)
       except ErrorWhileTesting as e:
         rv = PytestResult.from_error(e)
       results[partition] = rv
-      if not rv.success and self.get_options().fail_fast:
-        break
+      if not rv.success:
+        failure = True
+        if self.get_options().fail_fast:
+          break
 
     for partition in sorted(results):
       rv = results[partition]
@@ -532,6 +535,9 @@ def _run_tests(self, targets):
                       for target in _rv.failed_targets]
     if failed_targets:
       raise ErrorWhileTesting(failed_targets=failed_targets)
+    elif failure:
+      # A low-level test execution failure occurred before tests were run.
+      raise TaskError()
 
   def _do_run_tests(self, targets):
     with self.invalidated(targets,
@@ -581,7 +587,18 @@ def _run_pytest(self, targets):
       args.extend(test_args)
       args.extend(sources_map.keys())
 
+      # We want to ensure our reporting based off junit xml is from this run so kill results from
+      # prior runs.
+      if os.path.exists(junitxml_path):
+        os.unlink(junitxml_path)
+
       result = self._do_run_tests_with_args(pex, args)
+
+      # There was a problem prior to test execution preventing junit xml file creation so just let
+      # the failure result bubble.
+      if not os.path.exists(junitxml_path):
+        return result
+
       external_junit_xml_dir = self.get_options().junit_xml_dir
       if external_junit_xml_dir:
         safe_mkdir(external_junit_xml_dir)
@@ -591,7 +608,7 @@ def _run_pytest(self, targets):
       def parse_error_handler(parse_error):
         # Simple error handler to pass to xml parsing function.
         raise TaskError('Error parsing xml file at {}: {}'
-          .format(parse_error.xml_path, parse_error.cause))
+                        .format(parse_error.xml_path, parse_error.cause))
 
       all_tests_info = self.parse_test_info(junitxml_path, parse_error_handler,
                                             ['file', 'name', 'classname'])


Test patch:
diff --git a/tests/python/pants_test/backend/python/tasks2/test_pytest_run.py b/tests/python/pants_test/backend/python/tasks2/test_pytest_run.py
--- a/tests/python/pants_test/backend/python/tasks2/test_pytest_run.py
+++ b/tests/python/pants_test/backend/python/tasks2/test_pytest_run.py
@@ -13,11 +13,12 @@
 
 from pants.backend.python.tasks2.gather_sources import GatherSources
 from pants.backend.python.tasks2.pytest_prep import PytestPrep
-from pants.backend.python.tasks2.pytest_run import PytestRun
+from pants.backend.python.tasks2.pytest_run import PytestResult, PytestRun
 from pants.backend.python.tasks2.resolve_requirements import ResolveRequirements
 from pants.backend.python.tasks2.select_interpreter import SelectInterpreter
-from pants.base.exceptions import ErrorWhileTesting
+from pants.base.exceptions import ErrorWhileTesting, TaskError
 from pants.util.contextutil import pushd
+from pants.util.dirutil import safe_mkdtemp, safe_rmtree
 from pants.util.timeout import TimeoutReached
 from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase
 
@@ -87,6 +88,67 @@ def test_empty(self):
     self.run_tests(targets=[])
 
 
+class PytestTestFailedPexRun(PytestTestBase):
+  class AlwaysFailingPexRunPytestRun(PytestRun):
+    @classmethod
+    def set_up(cls):
+      junitxml_dir = safe_mkdtemp()
+      cls.junitxml_path = os.path.join(junitxml_dir, 'junit.xml')
+      cls._get_junit_xml_path = lambda *args, **kwargs: cls.junitxml_path
+      return lambda: safe_rmtree(junitxml_dir)
+
+    def _do_run_tests_with_args(self, *args, **kwargs):
+      return PytestResult.rc(42)
+
+  @classmethod
+  def task_type(cls):
+    return cls.AlwaysFailingPexRunPytestRun
+
+  def setUp(self):
+    super(PytestTestFailedPexRun, self).setUp()
+    self.create_file(
+      'tests/test_green.py',
+      dedent("""
+          import unittest
+
+          class GreenTest(unittest.TestCase):
+            def test_green(self):
+              self.assertTrue(True)
+        """))
+    self.add_to_build_file('tests', 'python_tests(sources=["test_green.py"])')
+    self.tests = self.target('tests')
+
+    self.addCleanup(self.AlwaysFailingPexRunPytestRun.set_up())
+
+  def do_test_failed_pex_run(self):
+    with self.assertRaises(TaskError) as cm:
+      self.run_tests(targets=[self.tests])
+
+    # We expect a `TaskError` as opposed to an `ErrorWhileTesting` since execution fails outside
+    # the actual test run.
+    self.assertEqual(TaskError, type(cm.exception))
+
+  def test_failed_pex_run(self):
+    self.do_test_failed_pex_run()
+
+  def test_failed_pex_run_does_not_see_prior_failures(self):
+    # Setup a prior failure.
+    with open(self.AlwaysFailingPexRunPytestRun.junitxml_path, mode='wb') as fp:
+      fp.write(b"""
+          <testsuite errors="0" failures="1" name="pytest" skips="0" tests="1" time="0.001">
+            <testcase classname="tests.test_green.GreenTest"
+                      file=".pants.d/gs/8...6-DefaultFingerprintStrategy_e88d80fa140b/test_green.py"
+                      line="4"
+                      name="test_green"
+                      time="0.0001">
+              <failure message="AssertionError: False is not true"/>
+            </testcase>
+          </testsuite>
+          """)
+
+    self.do_test_failed_pex_run()
+
+
 class PytestTest(PytestTestBase):
   def setUp(self):
     super(PytestTest, self).setUp()