Repository: pantsbuild/pants

Issue:
Provide a stable public output directory for junit test reports.
This was discussed in #259 and should be implemented with a deprecation cycle.  A `dist/` home would meet the expectations pants sets for users and the only real questions seem to be:
- always copy/link files under `dist/` or only when an option is specified?
- allow for an alternate location via an option?



Discussion:

Resolving patch:
diff --git a/src/python/pants/backend/jvm/tasks/junit_run.py b/src/python/pants/backend/jvm/tasks/junit_run.py
--- a/src/python/pants/backend/jvm/tasks/junit_run.py
+++ b/src/python/pants/backend/jvm/tasks/junit_run.py
@@ -28,6 +28,7 @@
 from pants.backend.jvm.tasks.jvm_tool_task_mixin import JvmToolTaskMixin
 from pants.backend.jvm.tasks.reports.junit_html_report import JUnitHtmlReport, NoJunitHtmlReport
 from pants.base.build_environment import get_buildroot
+from pants.base.deprecated import deprecated_conditional
 from pants.base.exceptions import ErrorWhileTesting, TargetDefinitionException, TaskError
 from pants.base.workunit import WorkUnitLabel
 from pants.build_graph.files import Files
@@ -189,6 +190,8 @@ def register_options(cls, register):
              help='If true, generate an html summary report of tests that were run.')
     register('--open', type=bool, fingerprint=True,
              help='Attempt to open the html summary report in a browser (implies --html-report)')
+    register('--legacy-report-layout', type=bool, default=True, advanced=True,
+             help='Links JUnit and coverage reports to the legacy location.')
 
     # TODO(jtrobec): Remove direct register when coverage steps are moved to their own subsystem.
     CodeCoverage.register_junit_options(register, cls.register_jvm_tool)
@@ -241,6 +244,7 @@ def __init__(self, *args, **kwargs):
     self._failure_summary = options.failure_summary
     self._open = options.open
     self._html_report = self._open or options.html_report
+    self._legacy_report_layout = options.legacy_report_layout
 
   @memoized_method
   def _args(self, output_dir):
@@ -662,20 +666,34 @@ def _isolation(self, all_targets):
       yield output_dir, reports, coverage
     finally:
       lock_file = '.file_lock'
-      with OwnerPrintingInterProcessFileLock(os.path.join(self.workdir, lock_file)):
-        preserve = (run_dir, lock_file)
-        self._link_current_reports(output_dir, preserve)
-
-  def _link_current_reports(self, output_dir, preserve):
-    # NB: Deposit of the "current" test output in the root workdir (.pants.d/test/junit) is a
-    # defacto public API and so we implement that behavior here to maintain backwards
-    # compatibility for non-pants report file consumers.
-    # TODO(John Sirois): Deprecate this ~API and provide a stable directory solution for test
-    # output: https://github.com/pantsbuild/pants/issues/3879
-
+      preserve = (run_dir, lock_file)
+      dist_dir = os.path.join(self.get_options().pants_distdir,
+                              os.path.relpath(self.workdir, self.get_options().pants_workdir))
+
+      with OwnerPrintingInterProcessFileLock(os.path.join(dist_dir, lock_file)):
+        self._link_current_reports(report_dir=output_dir, link_dir=dist_dir,
+                                   preserve=preserve)
+
+      if self._legacy_report_layout:
+        deprecated_conditional(predicate=lambda: True,
+                               entity_description='[test.junit] legacy_report_layout',
+                               stacklevel=3,
+                               removal_version='1.6.0.dev0',
+                               hint_message='Reports are now linked into {} by default; so scripts '
+                                            'and CI jobs should be pointed there and the option '
+                                            'configured to False in pants.ini until such time as '
+                                            'the option is removed.'.format(dist_dir))
+        # NB: Deposit of the "current" test output in the root workdir (.pants.d/test/junit) is a
+        # defacto public API and so we implement that behavior here to maintain backwards
+        # compatibility for non-pants report file consumers.
+        with OwnerPrintingInterProcessFileLock(os.path.join(self.workdir, lock_file)):
+          self._link_current_reports(report_dir=output_dir, link_dir=self.workdir,
+                                     preserve=preserve)
+
+  def _link_current_reports(self, report_dir, link_dir, preserve):
     # Kill everything not preserved.
-    for name in os.listdir(self.workdir):
-      path = os.path.join(self.workdir, name)
+    for name in os.listdir(link_dir):
+      path = os.path.join(link_dir, name)
       if name not in preserve:
         if os.path.isdir(path):
           safe_rmtree(path)
@@ -688,19 +706,18 @@ def _link_current_reports(self, output_dir, preserve):
     # result in a loss of information from the ignored files. We're OK with this because:
     # a) We're planning on deprecating this loss of information.
     # b) It is the same behavior as existed before batching was added.
-    for root, dirs, files in safe_walk(output_dir, topdown=True):
+    for root, dirs, files in safe_walk(report_dir, topdown=True):
       dirs.sort()  # Ensure a consistent walk order for sanity sake.
       for f in itertools.chain(fnmatch.filter(files, '*.err.txt'),
                                fnmatch.filter(files, '*.out.txt'),
                                fnmatch.filter(files, 'TEST-*.xml')):
         src = os.path.join(root, f)
-        dst = os.path.join(self.workdir, f)
+        dst = os.path.join(link_dir, f)
         safe_delete(dst)
         os.symlink(src, dst)
 
-    for path in os.listdir(output_dir):
+    for path in os.listdir(report_dir):
       if path in ('coverage', 'reports'):
-        src = os.path.join(output_dir, path)
-        dst = os.path.join(self.workdir, path)
+        src = os.path.join(report_dir, path)
+        dst = os.path.join(link_dir, path)
         os.symlink(src, dst)
-


Test patch:
diff --git a/tests/python/pants_test/backend/jvm/tasks/BUILD b/tests/python/pants_test/backend/jvm/tasks/BUILD
--- a/tests/python/pants_test/backend/jvm/tasks/BUILD
+++ b/tests/python/pants_test/backend/jvm/tasks/BUILD
@@ -429,7 +429,9 @@ python_tests(
   name = 'junit_run_integration',
   sources = ['test_junit_run_integration.py'],
   dependencies = [
+    '3rdparty/python:parameterized',
     ':missing_jvm_check',
+    'src/python/pants/base:build_environment',
     'src/python/pants/util:process_handler',
     'tests/python/pants_test:int-test',
   ],
@@ -441,6 +443,8 @@ python_tests(
   name = 'junit_tests_integration',
   sources = ['test_junit_tests_integration.py'],
   dependencies = [
+    '3rdparty/python:parameterized',
+    'src/python/pants/base:build_environment',
     'src/python/pants/util:contextutil',
     'tests/python/pants_test:int-test',
   ],
diff --git a/tests/python/pants_test/backend/jvm/tasks/test_junit_run_integration.py b/tests/python/pants_test/backend/jvm/tasks/test_junit_run_integration.py
--- a/tests/python/pants_test/backend/jvm/tasks/test_junit_run_integration.py
+++ b/tests/python/pants_test/backend/jvm/tasks/test_junit_run_integration.py
@@ -10,10 +10,17 @@
 import time
 from unittest import skipIf
 
+from parameterized import parameterized
+
+from pants.base.build_environment import get_buildroot
 from pants_test.backend.jvm.tasks.missing_jvm_check import is_missing_jvm
 from pants_test.pants_run_integration_test import PantsRunIntegrationTest
 
 
+OUTPUT_MODES = [('legacy_layout', ['--legacy-report-layout'], True),
+                ('nominal_layout', ['--no-legacy-report-layout'], False)]
+
+
 class JunitRunIntegrationTest(PantsRunIntegrationTest):
 
   def _testjvms(self, spec_name):
@@ -35,36 +42,41 @@ def test_junit_run_against_class_succeeds(self):
                                 'testprojects/tests/java/org/pantsbuild/testproject/matcher'])
     self.assert_success(pants_run)
 
-  def test_junit_run_with_cobertura_coverage_succeeds(self):
-    with self.pants_results(['clean-all',
-                             'test.junit',
-                             'testprojects/tests/java/org/pantsbuild/testproject/unicode::',
+  def report_file_path(self, results, relpath, legacy=False):
+    return os.path.join(results.workdir if legacy else os.path.join(get_buildroot(), 'dist'),
+                        relpath)
+
+  @parameterized.expand(OUTPUT_MODES)
+  def test_junit_run_with_cobertura_coverage_succeeds(self, unused_test_name, extra_args, legacy):
+    with self.pants_results(['clean-all', 'test.junit'] + extra_args +
+                            ['testprojects/tests/java/org/pantsbuild/testproject/unicode::',
                              '--test-junit-coverage-processor=cobertura',
                              '--test-junit-coverage']) as results:
       self.assert_success(results)
       # validate that the expected coverage file exists, and it reflects 100% line rate coverage
-      coverage_xml = os.path.join(results.workdir, 'test/junit/coverage/xml/coverage.xml')
+      coverage_xml = self.report_file_path(results, 'test/junit/coverage/xml/coverage.xml', legacy)
       self.assertTrue(os.path.isfile(coverage_xml))
       with codecs.open(coverage_xml, 'r', encoding='utf8') as xml:
         self.assertIn('line-rate="1.0"', xml.read())
       # validate that the html report was able to find sources for annotation
-      cucumber_src_html = os.path.join(
-          results.workdir,
+      cucumber_src_html = self.report_file_path(
+          results,
           'test/junit/coverage/html/'
-          'org.pantsbuild.testproject.unicode.cucumber.CucumberAnnotatedExample.html')
+          'org.pantsbuild.testproject.unicode.cucumber.CucumberAnnotatedExample.html',
+          legacy)
       self.assertTrue(os.path.isfile(cucumber_src_html))
       with codecs.open(cucumber_src_html, 'r', encoding='utf8') as src:
         self.assertIn('String pleasantry()', src.read())
 
-  def test_junit_run_with_jacoco_coverage_succeeds(self):
-    with self.pants_results(['clean-all',
-                             'test.junit',
-                             'testprojects/tests/java/org/pantsbuild/testproject/unicode::',
+  @parameterized.expand(OUTPUT_MODES)
+  def test_junit_run_with_jacoco_coverage_succeeds(self, unused_test_name, extra_args, legacy):
+    with self.pants_results(['clean-all', 'test.junit'] + extra_args +
+                            ['testprojects/tests/java/org/pantsbuild/testproject/unicode::',
                              '--test-junit-coverage-processor=jacoco',
                              '--test-junit-coverage']) as results:
       self.assert_success(results)
       # validate that the expected coverage file exists, and it reflects 100% line rate coverage
-      coverage_xml = os.path.join(results.workdir, 'test/junit/coverage/reports/xml')
+      coverage_xml = self.report_file_path(results, 'test/junit/coverage/reports/xml', legacy)
       self.assertTrue(os.path.isfile(coverage_xml))
       with codecs.open(coverage_xml, 'r', encoding='utf8') as xml:
         self.assertIn(
@@ -72,10 +84,11 @@ def test_junit_run_with_jacoco_coverage_succeeds(self):
           '<method name="&lt;init&gt;" desc="()V" line="13">'
           '<counter type="INSTRUCTION" missed="0" covered="3"/>', xml.read())
       # validate that the html report was able to find sources for annotation
-      cucumber_src_html = os.path.join(
-          results.workdir,
+      cucumber_src_html = self.report_file_path(
+          results,
           'test/junit/coverage/reports/html/'
-          'org.pantsbuild.testproject.unicode.cucumber/CucumberAnnotatedExample.html')
+          'org.pantsbuild.testproject.unicode.cucumber/CucumberAnnotatedExample.html',
+          legacy)
       self.assertTrue(os.path.isfile(cucumber_src_html))
       with codecs.open(cucumber_src_html, 'r', encoding='utf8') as src:
         self.assertIn('class="el_method">pleasantry()</a>', src.read())
@@ -134,17 +147,16 @@ def test_disable_synthetic_jar(self):
                              synthetic_jar_target]).stdout_data
     self.assertIn('Synthetic jar run is not detected', output)
 
-  def do_test_junit_run_with_html_report(self, tests=(), args=()):
+  def do_test_junit_run_with_html_report(self, tests=(), args=(), legacy=False):
     def html_report_test(test):
       return '--test=org.pantsbuild.testproject.htmlreport.HtmlReportTest#{}'.format(test)
 
-    with self.pants_results(['clean-all', 'test.junit'] +
+    with self.pants_results(['clean-all', 'test.junit'] + list(args) +
                             [html_report_test(name) for name in tests] +
                             ['testprojects/tests/java/org/pantsbuild/testproject/htmlreport::',
-                             '--test-junit-html-report'] +
-                            list(args)) as results:
+                             '--test-junit-html-report']) as results:
       self.assert_failure(results)
-      report_html = os.path.join(results.workdir, 'test/junit/reports/junit-report.html')
+      report_html = self.report_file_path(results, 'test/junit/reports/junit-report.html', legacy)
       self.assertTrue(os.path.isfile(report_html))
       with codecs.open(report_html, 'r', encoding='utf8') as src:
         html = src.read()
@@ -153,12 +165,15 @@ def html_report_test(test):
         self.assertIn('testErrors', html)
         self.assertIn('testSkipped', html)
 
-  def test_junit_run_with_html_report(self):
-    self.do_test_junit_run_with_html_report()
+  @parameterized.expand(OUTPUT_MODES)
+  def test_junit_run_with_html_report(self, unused_test_name, extra_args, legacy):
+    self.do_test_junit_run_with_html_report(args=extra_args, legacy=legacy)
 
-  def test_junit_run_with_html_report_merged(self):
+  @parameterized.expand(OUTPUT_MODES)
+  def test_junit_run_with_html_report_merged(self, unused_test_name, extra_args, legacy):
     self.do_test_junit_run_with_html_report(tests=['testPasses',
                                                    'testFails',
                                                    'testErrors',
                                                    'testSkipped'],
-                                            args=['--test-junit-batch-size=3'])
+                                            args=['--batch-size=3'] + extra_args,
+                                            legacy=legacy)
diff --git a/tests/python/pants_test/backend/jvm/tasks/test_junit_tests_integration.py b/tests/python/pants_test/backend/jvm/tasks/test_junit_tests_integration.py
--- a/tests/python/pants_test/backend/jvm/tasks/test_junit_tests_integration.py
+++ b/tests/python/pants_test/backend/jvm/tasks/test_junit_tests_integration.py
@@ -7,31 +7,52 @@
 
 import os
 
+from parameterized import parameterized
+
+from pants.base.build_environment import get_buildroot
 from pants_test.pants_run_integration_test import PantsRunIntegrationTest
 
 
+OUTPUT_MODES = [('legacy_layout', ['--legacy-report-layout'], True),
+                ('nominal_layout', ['--no-legacy-report-layout'], False)]
+
+
 class JunitTestsIntegrationTest(PantsRunIntegrationTest):
 
-  def _assert_output_for_class(self, workdir, classname):
-    out_dir = os.path.join(workdir, 'test', 'junit')
-    self.assertTrue(os.path.exists(os.path.join(out_dir, '{}.out.txt'.format(classname))))
-    self.assertTrue(os.path.exists(os.path.join(out_dir, '{}.err.txt'.format(classname))))
+  def _assert_output_for_class(self, workdir, classname, expect_legacy=True):
+    def get_outdir(basedir):
+      return os.path.join(basedir, 'test', 'junit')
+
+    def get_stdout_file(basedir):
+      return os.path.join(basedir, '{}.out.txt'.format(classname))
 
-  def _assert_junit_output(self, workdir):
-    self._assert_output_for_class(workdir, 'org.pantsbuild.example.hello.greet.GreetingTest')
-    self._assert_output_for_class(workdir, 'org.pantsbuild.example.hello.welcome.WelSpec')
+    def get_stderr_file(basedir):
+      return os.path.join(basedir, '{}.err.txt'.format(classname))
 
-  def test_junit_test_custom_interpreter(self):
+    outdir = get_outdir(os.path.join(get_buildroot(), 'dist'))
+    self.assertTrue(os.path.exists(get_stdout_file(outdir)))
+    self.assertTrue(os.path.exists(get_stderr_file(outdir)))
+
+    legacy_outdir = get_outdir(workdir)
+    self.assertEqual(expect_legacy, os.path.exists(get_stdout_file(legacy_outdir)))
+    self.assertEqual(expect_legacy, os.path.exists(get_stderr_file(legacy_outdir)))
+
+  @parameterized.expand(OUTPUT_MODES)
+  def test_junit_test_custom_interpreter(self, unused_test_name, extra_args, expect_legacy):
     with self.temporary_workdir() as workdir:
-      pants_run = self.run_pants_with_workdir([
-          'test',
-          'examples/tests/java/org/pantsbuild/example/hello/greet',
-          'examples/tests/scala/org/pantsbuild/example/hello/welcome',
-          '--python-setup-interpreter-constraints=CPython>=2.7,<3',
-          '--python-setup-interpreter-constraints=CPython>=3.3'],
+      pants_run = self.run_pants_with_workdir(
+          ['test.junit'] + extra_args +
+          ['examples/tests/java/org/pantsbuild/example/hello/greet',
+           'examples/tests/scala/org/pantsbuild/example/hello/welcome'],
           workdir)
       self.assert_success(pants_run)
-      self._assert_junit_output(workdir)
+
+      self._assert_output_for_class(workdir=workdir,
+                                    classname='org.pantsbuild.example.hello.greet.GreetingTest',
+                                    expect_legacy=expect_legacy)
+      self._assert_output_for_class(workdir=workdir,
+                                    classname='org.pantsbuild.example.hello.welcome.WelSpec',
+                                    expect_legacy=expect_legacy)
 
   def test_junit_test(self):
     with self.temporary_workdir() as workdir:
@@ -41,38 +62,56 @@ def test_junit_test(self):
           workdir)
       self.assert_failure(pants_run)
 
-  def test_junit_test_with_test_option_with_relpath(self):
+  @parameterized.expand(OUTPUT_MODES)
+  def test_junit_test_with_test_option_with_relpath(self,
+                                                    unused_test_name,
+                                                    extra_args,
+                                                    expect_legacy):
     with self.temporary_workdir() as workdir:
-      pants_run = self.run_pants_with_workdir([
-          'test.junit',
-          '--test=examples/tests/java/org/pantsbuild/example/hello/greet/GreetingTest.java',
-          'examples/tests/java/org/pantsbuild/example/hello/greet',
-          'examples/tests/scala/org/pantsbuild/example/hello/welcome'],
+      pants_run = self.run_pants_with_workdir(
+          ['test.junit'] + extra_args +
+          ['--test=examples/tests/java/org/pantsbuild/example/hello/greet/GreetingTest.java',
+           'examples/tests/java/org/pantsbuild/example/hello/greet',
+           'examples/tests/scala/org/pantsbuild/example/hello/welcome'],
           workdir)
       self.assert_success(pants_run)
-      self._assert_output_for_class(workdir, 'org.pantsbuild.example.hello.greet.GreetingTest')
-
-  def test_junit_test_with_test_option_with_dot_slash_relpath(self):
+      self._assert_output_for_class(workdir=workdir,
+                                    classname='org.pantsbuild.example.hello.greet.GreetingTest',
+                                    expect_legacy=expect_legacy)
+
+  @parameterized.expand(OUTPUT_MODES)
+  def test_junit_test_with_test_option_with_dot_slash_relpath(self,
+                                                              unused_test_name,
+                                                              extra_args,
+                                                              expect_legacy):
     with self.temporary_workdir() as workdir:
-      pants_run = self.run_pants_with_workdir([
-          'test.junit',
-          '--test=./examples/tests/java/org/pantsbuild/example/hello/greet/GreetingTest.java',
-          'examples/tests/java/org/pantsbuild/example/hello/greet',
-          'examples/tests/scala/org/pantsbuild/example/hello/welcome'],
+      pants_run = self.run_pants_with_workdir(
+          ['test.junit'] + extra_args +
+          ['--test=./examples/tests/java/org/pantsbuild/example/hello/greet/GreetingTest.java',
+           'examples/tests/java/org/pantsbuild/example/hello/greet',
+           'examples/tests/scala/org/pantsbuild/example/hello/welcome'],
           workdir)
       self.assert_success(pants_run)
-      self._assert_output_for_class(workdir, 'org.pantsbuild.example.hello.greet.GreetingTest')
-
-  def test_junit_test_with_test_option_with_classname(self):
+      self._assert_output_for_class(workdir=workdir,
+                                    classname='org.pantsbuild.example.hello.greet.GreetingTest',
+                                    expect_legacy=expect_legacy)
+
+  @parameterized.expand(OUTPUT_MODES)
+  def test_junit_test_with_test_option_with_classname(self,
+                                                      unused_test_name,
+                                                      extra_args,
+                                                      expect_legacy):
     with self.temporary_workdir() as workdir:
-      pants_run = self.run_pants_with_workdir([
-          'test.junit',
-          '--test=org.pantsbuild.example.hello.greet.GreetingTest',
-          'examples/tests/java/org/pantsbuild/example/hello/greet',
-          'examples/tests/scala/org/pantsbuild/example/hello/welcome'],
+      pants_run = self.run_pants_with_workdir(
+          ['test.junit'] + extra_args +
+          ['--test=org.pantsbuild.example.hello.greet.GreetingTest',
+           'examples/tests/java/org/pantsbuild/example/hello/greet',
+           'examples/tests/scala/org/pantsbuild/example/hello/welcome'],
           workdir)
       self.assert_success(pants_run)
-      self._assert_output_for_class(workdir, 'org.pantsbuild.example.hello.greet.GreetingTest')
+      self._assert_output_for_class(workdir=workdir,
+                                    classname='org.pantsbuild.example.hello.greet.GreetingTest',
+                                    expect_legacy=expect_legacy)
 
   def test_junit_test_requiring_cwd_fails_without_option_specified(self):
     pants_run = self.run_pants([
diff --git a/tests/python/pants_test/tasks/test_changed_target_integration.py b/tests/python/pants_test/tasks/test_changed_target_integration.py
--- a/tests/python/pants_test/tasks/test_changed_target_integration.py
+++ b/tests/python/pants_test/tasks/test_changed_target_integration.py
@@ -122,7 +122,8 @@ def test_compile_changed(self):
   def test_test_changed(self):
     with self.known_commits(), self.temporary_workdir() as workdir:
       cmd = ['test-changed', '--diffspec=HEAD~2..HEAD~1']
-      junit_out = os.path.join(workdir, 'test', 'junit', 'org.pantsbuild.ClassTest.out.txt')
+      junit_out = os.path.join(get_buildroot(), 'dist', 'test', 'junit',
+                               'org.pantsbuild.ClassTest.out.txt')
 
       self.assertFalse(os.path.exists(junit_out))