Repository: pantsbuild/pants

Issue:
pytest junit xml not written if tests fail
Specifying the --test-pytest-junit-xml-dir flag does not work if the tests fail.

The problem seems to be in pytest_run.py, line 626 (_do_run_tests):
the checked() call raises an ErrorWhileTesting exception if the run fails, thus skipping the rest of the function, including the call to expose_results.

can the call to checked() be removed? The error handling for ErrorWhileTesting in _run_tests just builds a result from the exception which (I think) matches the original result. 

pants version 1.4.0rc0


Discussion:

Resolving patch:
diff --git a/src/python/pants/backend/python/tasks/pytest_run.py b/src/python/pants/backend/python/tasks/pytest_run.py
--- a/src/python/pants/backend/python/tasks/pytest_run.py
+++ b/src/python/pants/backend/python/tasks/pytest_run.py
@@ -39,17 +39,21 @@
 from pants.util.xml_parser import XmlParser
 
 
-class _Workdirs(datatype('_Workdirs', ['root_dir'])):
+class _Workdirs(datatype('_Workdirs', ['root_dir', 'partition'])):
   @classmethod
-  def for_targets(cls, work_dir, targets):
-    root_dir = os.path.join(work_dir, Target.maybe_readable_identify(targets))
+  def for_partition(cls, work_dir, partition):
+    root_dir = os.path.join(work_dir, Target.maybe_readable_identify(partition))
     safe_mkdir(root_dir, clean=False)
-    return cls(root_dir=root_dir)
+    return cls(root_dir=root_dir, partition=partition)
+
+  @memoized_method
+  def target_set_id(self, *targets):
+    return Target.maybe_readable_identify(targets or self.partition)
 
   @memoized_method
   def junitxml_path(self, *targets):
     xml_path = os.path.join(self.root_dir, 'junitxml',
-                            'TEST-{}.xml'.format(Target.maybe_readable_identify(targets)))
+                            'TEST-{}.xml'.format(self.target_set_id(*targets)))
     safe_mkdir_for(xml_path)
     return xml_path
 
@@ -618,12 +622,10 @@ def _do_run_tests(self, partition):
       # 1.) output -> workdir
       # 2.) [iff all == invalid] workdir -> cache: We do this manually for now.
       # 3.) [iff invalid == 0 and all > 0] cache -> workdir: Done transparently by `invalidated`.
-      # 4.) [iff user-specified final locations] workdir -> final-locations: We perform this step
-      #     as an unconditional post-process.
 
       # 1.) Write all results that will be potentially cached to workdir.
-      workdirs = _Workdirs.for_targets(self.workdir, partition)
-      result = self._run_pytest(workdirs, invalid_tgts).checked()
+      workdirs = _Workdirs.for_partition(self.workdir, partition)
+      result = self._run_pytest_checked(workdirs, invalid_tgts)
 
       cache_vts = self._vts_for_partition(invalidation_check)
       if invalidation_check.all_vts == invalidation_check.invalid_vts:
@@ -644,32 +646,38 @@ def _do_run_tests(self, partition):
         # cache the results. That 1st of others is hopefully CI!
         cache_vts.force_invalidate()
 
-      # 4.) Pluck any results that an end user might need to interact with from the workdir to the
-      # locations they expect.
-      self.expose_results(invalid_tgts, partition, workdirs)
-
       return result
 
-  def expose_results(self, invalid_tgts, partition, workdirs):
+  def _expose_results(self, invalid_tgts, workdirs):
     external_junit_xml_dir = self.get_options().junit_xml_dir
     if external_junit_xml_dir:
       # Either we just ran pytest for a set of invalid targets and generated a junit xml file
       # specific to that (sub)set or else we hit the cache for the whole partition and skipped
       # running pytest, simply retrieving the partition's full junit xml file.
-      junitxml_path = workdirs.junitxml_path(*(invalid_tgts or partition))
+      junitxml_path = workdirs.junitxml_path(*invalid_tgts)
 
       safe_mkdir(external_junit_xml_dir)
       shutil.copy2(junitxml_path, external_junit_xml_dir)
+
     if self.get_options().coverage:
       coverage_output_dir = self.get_options().coverage_output_dir
       if coverage_output_dir:
         target_dir = coverage_output_dir
       else:
-        relpath = Target.maybe_readable_identify(partition)
         pants_distdir = self.context.options.for_global_scope().pants_distdir
+        relpath = workdirs.target_set_id()
         target_dir = os.path.join(pants_distdir, 'coverage', relpath)
       mergetree(workdirs.coverage_path, target_dir)
 
+  def _run_pytest_checked(self, workdirs, targets):
+    result = self._run_pytest(workdirs, targets)
+
+    # Unconditionally pluck any results that an end user might need to interact with from the
+    # workdir to the locations they expect.
+    self._expose_results(targets, workdirs)
+
+    return result.checked()
+
   def _run_pytest(self, workdirs, targets):
     if not targets:
       return PytestResult.rc(0)


Test patch:
diff --git a/tests/python/pants_test/backend/python/tasks/test_pytest_run.py b/tests/python/pants_test/backend/python/tasks/test_pytest_run.py
--- a/tests/python/pants_test/backend/python/tasks/test_pytest_run.py
+++ b/tests/python/pants_test/backend/python/tasks/test_pytest_run.py
@@ -17,7 +17,7 @@
 from pants.backend.python.tasks.resolve_requirements import ResolveRequirements
 from pants.backend.python.tasks.select_interpreter import SelectInterpreter
 from pants.base.exceptions import ErrorWhileTesting, TaskError
-from pants.util.contextutil import pushd
+from pants.util.contextutil import pushd, temporary_dir
 from pants.util.dirutil import safe_mkdtemp, safe_rmtree
 from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase
 from pants_test.tasks.task_test_base import ensure_cached
@@ -31,14 +31,9 @@ def task_type(cls):
   _CONFTEST_CONTENT = '# I am an existing root-level conftest file.'
 
   def run_tests(self, targets, *passthru_args, **options):
-    """Run the tests in the specified targets, with the specified PytestRun task options.
-
-    Returns the path of the sources pex, so that calling code can map files from the
-    source tree to files as pytest saw them.
-    """
+    """Run the tests in the specified targets, with the specified PytestRun task options."""
     context = self._prepare_test_run(targets, *passthru_args, **options)
     self._do_run_tests(context)
-    return context
 
   def run_failing_tests(self, targets, failed_targets, *passthru_args, **options):
     context = self._prepare_test_run(targets, *passthru_args, **options)
@@ -475,14 +470,38 @@ def test_fail_fast_skips_second_red_test_with_isolated_chroot(self):
 
   @ensure_cached(PytestRun, expected_num_artifacts=0)
   def test_red_test_in_class(self):
-    # for test in a class, the failure line is in the following format
-    # F testprojects/tests/python/pants/constants_only/test_fail.py::TestClassName::test_boom
     self.run_failing_tests(targets=[self.red_in_class], failed_targets=[self.red_in_class])
 
   @ensure_cached(PytestRun, expected_num_artifacts=0)
   def test_mixed(self):
     self.run_failing_tests(targets=[self.green, self.red], failed_targets=[self.red])
 
+  def assert_test_info(self, junit_xml_dir, *expected):
+    test_info = PytestRun.parse_test_info(xml_path=junit_xml_dir, error_handler=self.assertIsNone)
+    self.assertEqual({name for (name, _) in expected}, set(test_info.keys()))
+    for name, result in expected:
+      test_details = test_info[name]
+      self.assertEqual(result, test_details['result_code'])
+      self.assertGreater(test_details['time'], 0)
+
+  @ensure_cached(PytestRun, expected_num_artifacts=1)
+  def test_green_junit_xml_dir(self):
+    with temporary_dir() as junit_xml_dir:
+      self.run_tests(targets=[self.green], junit_xml_dir=junit_xml_dir)
+
+      self.assert_test_info(junit_xml_dir, ('test_one', 'success'))
+
+  @ensure_cached(PytestRun, expected_num_artifacts=0)
+  def test_red_junit_xml_dir(self):
+    with temporary_dir() as junit_xml_dir:
+      self.run_failing_tests(targets=[self.red, self.green],
+                             failed_targets=[self.red],
+                             junit_xml_dir=junit_xml_dir,
+                             fast=True,
+                             fail_fast=False)
+
+      self.assert_test_info(junit_xml_dir, ('test_one', 'success'), ('test_two', 'failure'))
+
   def coverage_data_file(self):
     return os.path.join(self.build_root, '.coverage')