Repository: pantsbuild/pants

Issue:
RscCompile should comput metacp for the jdk and scala synthetics only once
Currently they are recomputed for each metacp invocation.

Additionally, they should use the `.jdk` symlink introduced in https://github.com/pantsbuild/pants/commit/4d475b37f86f5391504468abddad5eaa619e595d if it is the correct jdk.


Discussion:

Resolving patch:
diff --git a/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py b/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py
--- a/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py
+++ b/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py
@@ -685,6 +685,8 @@ def format_length(self):
     counter = Counter(len(invalid_vts))
 
     jobs = []
+
+    jobs.extend(self.pre_compile_jobs(counter))
     invalid_target_set = set(invalid_targets)
     for ivts in invalid_vts:
       # Invalidated targets are a subset of relevant targets: get the context for this one.
@@ -699,6 +701,12 @@ def format_length(self):
     counter.size = len(jobs)
     return jobs
 
+  def pre_compile_jobs(self, counter):
+    """Override this to provide jobs that are not related to particular targets.
+
+    This is only called when there are invalid targets."""
+    return []
+
   def create_compile_jobs(self, compile_target, all_compile_contexts, invalid_dependencies, ivts,
     counter, classpath_product):
 
diff --git a/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py b/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py
--- a/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py
+++ b/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py
@@ -24,6 +24,8 @@
 from pants.base.build_environment import get_buildroot
 from pants.base.exceptions import TaskError
 from pants.base.workunit import WorkUnitLabel
+from pants.build_graph.address import Address
+from pants.build_graph.target import Target
 from pants.engine.fs import DirectoryToMaterialize, PathGlobs, PathGlobsAndRoot
 from pants.engine.isolated_process import ExecuteProcessRequest, FallibleExecuteProcessResult
 from pants.java.jar.jar_dependency import JarDependency
@@ -54,6 +56,22 @@ def stdout_contents(wu):
     return f.read().rstrip()
 
 
+def _create_desandboxify_fn(possible_beginning_paths):
+  # Takes a collection of possible canonical prefixes, and returns a function that
+  # if it finds a matching prefix, strips the path prior to the prefix and returns it
+  # if it doesn't it returns the original path
+  # TODO remove this after https://github.com/scalameta/scalameta/issues/1791 is released
+  def desandboxify(path):
+    if not path:
+      return path
+    for p in possible_beginning_paths:
+      if p in path:
+        new_path = path[path.index(p):]
+        return new_path
+    return path
+  return desandboxify
+
+
 class RscCompileContext(CompileContext):
   def __init__(self,
                target,
@@ -210,6 +228,9 @@ def _rsc_compilable(self, target):
   def _only_zinc_compileable(self, target):
     return target.has_sources('.java')
 
+  def _is_scala_core_library(self, target):
+    return target.address.spec in ('//:scala-library', '//:scala-library-synthetic')
+
   def create_empty_extra_products(self):
     super(RscCompile, self).create_empty_extra_products()
 
@@ -243,6 +264,81 @@ def _rsc_key_for_target(self, compile_target):
   def _compile_against_rsc_key_for_target(self, compile_target):
     return "compile_against_rsc({})".format(compile_target.address.spec)
 
+  def pre_compile_jobs(self, counter):
+
+    # Create a target for the jdk outlining so that it'll only be done once per run.
+    target = Target('jdk', Address('', 'jdk'), self.context.build_graph)
+    index_dir = os.path.join(self.workdir, '--jdk--', 'index')
+
+    def work_for_vts_rsc_jdk():
+      distribution = self._get_jvm_distribution()
+      jvm_lib_jars_abs = distribution.find_libs(['rt.jar', 'dt.jar', 'jce.jar', 'tools.jar'])
+      self._jvm_lib_jars_abs = jvm_lib_jars_abs
+
+      cp_entries = tuple(jvm_lib_jars_abs)
+
+      counter_val = str(counter()).rjust(counter.format_length(), b' ')
+      counter_str = '[{}/{}] '.format(counter_val, counter.size)
+      self.context.log.info(
+        counter_str,
+        'Metacp-ing ',
+        items_to_report_element(cp_entries, 'jar'),
+        ' in the jdk')
+
+      safe_mkdir(index_dir)
+
+      with Timer() as timer:
+        # Step 1: Convert classpath to SemanticDB
+        # ---------------------------------------
+        rsc_index_dir = fast_relpath(index_dir, get_buildroot())
+        args = [
+          '--verbose',
+          # NB: The directory to dump the semanticdb jars generated by metacp.
+          '--out', rsc_index_dir,
+          os.pathsep.join(cp_entries),
+        ]
+        metacp_wu = self._runtool(
+          'scala.meta.cli.Metacp',
+          'metacp',
+          args,
+          distribution,
+          tgt='jdk',
+          input_files=tuple(
+            # NB: no input files because the jdk is expected to exist on the system in a known
+            #     location.
+            #     Related: https://github.com/pantsbuild/pants/issues/6416
+          ),
+          output_dir=rsc_index_dir)
+        metacp_stdout = stdout_contents(metacp_wu)
+        metacp_result = json.loads(metacp_stdout)
+
+        metai_classpath = self._collect_metai_classpath(metacp_result, jvm_lib_jars_abs)
+
+        # Step 1.5: metai Index the semanticdbs
+        # -------------------------------------
+        self._run_metai_tool(distribution, metai_classpath, rsc_index_dir, tgt='jdk')
+
+        self._jvm_lib_metacp_classpath = [os.path.join(get_buildroot(), x) for x in metai_classpath]
+
+      self._record_target_stats(target,
+        len(self._jvm_lib_metacp_classpath),
+        len([]),
+        timer.elapsed,
+        False,
+        'metacp'
+      )
+
+    return [
+      Job(
+        'metacp(jdk)',
+        functools.partial(
+          work_for_vts_rsc_jdk
+        ),
+        [],
+        self._size_estimator([]),
+      ),
+    ]
+
   def create_compile_jobs(self,
                           compile_target,
                           compile_contexts,
@@ -260,8 +356,6 @@ def work_for_vts_rsc(vts, ctx):
         cp_entries = []
 
         distribution = self._get_jvm_distribution()
-        jvm_lib_jars_abs = distribution.find_libs(['rt.jar', 'dt.jar', 'jce.jar', 'tools.jar'])
-        cp_entries.extend(jvm_lib_jars_abs)
 
         classpath_abs = self._zinc.compile_classpath(
           'rsc_classpath',
@@ -273,6 +367,7 @@ def work_for_vts_rsc(vts, ctx):
         metacp_jar_classpath_abs = [y[1] for y in self._metacp_jars_classpath_product.get_for_targets(
           jar_deps
         )]
+        metacp_jar_classpath_abs.extend(self._jvm_lib_metacp_classpath)
         jar_jar_paths = {y[1] for y in self.context.products.get_data('rsc_classpath').get_for_targets(jar_deps)}
 
         classpath_abs = [c for c in classpath_abs if c not in jar_jar_paths]
@@ -298,41 +393,40 @@ def work_for_vts_rsc(vts, ctx):
 
         tgt, = vts.targets
         with Timer() as timer:
-          # Step 1: Convert classpath to SemanticDB
-          # ---------------------------------------
-          scalac_classpath_path_entries_abs = self.tool_classpath('workaround-metacp-dependency-classpath')
-          scalac_classpath_path_entries = fast_relpath_collection(scalac_classpath_path_entries_abs)
-          rsc_index_dir = fast_relpath(ctx.rsc_index_dir, get_buildroot())
-          args = [
-            '--verbose',
-            # NB: Without this setting, rsc will be missing some symbols
-            #     from the scala library.
-            '--include-scala-library-synthetics', # TODO generate these once and cache them
-            # NB: We need to add these extra dependencies in order to be able
-            #     to find symbols used by the scalac jars.
-            '--dependency-classpath', os.pathsep.join(scalac_classpath_path_entries + list(jar_jar_paths)),
-            # NB: The directory to dump the semanticdb jars generated by metacp.
-            '--out', rsc_index_dir,
-            os.pathsep.join(cp_entries),
-          ]
-          metacp_wu = self._runtool(
-            'scala.meta.cli.Metacp',
-            'metacp',
-            args,
-            distribution,
-            tgt=tgt,
-            input_files=tuple(scalac_classpath_path_entries + classpath_rel),
-            output_dir=rsc_index_dir)
-          metacp_stdout = stdout_contents(metacp_wu)
-          metacp_result = json.loads(metacp_stdout)
-
-
-          metai_classpath = self._collect_metai_classpath(
-            metacp_result, classpath_rel, jvm_lib_jars_abs)
-
-          # Step 1.5: metai Index the semanticdbs
-          # -------------------------------------
-          self._run_metai_tool(distribution, metai_classpath, rsc_index_dir, tgt)
+          if cp_entries:
+            # Step 1: Convert classpath to SemanticDB
+            # ---------------------------------------
+            scalac_classpath_path_entries_abs = self.tool_classpath('workaround-metacp-dependency-classpath')
+            scalac_classpath_path_entries = fast_relpath_collection(scalac_classpath_path_entries_abs)
+            rsc_index_dir = fast_relpath(ctx.rsc_index_dir, get_buildroot())
+            args = [
+              '--verbose',
+              # NB: We need to add these extra dependencies in order to be able
+              #     to find symbols used by the scalac jars.
+              '--dependency-classpath', os.pathsep.join(scalac_classpath_path_entries + list(jar_jar_paths) + self._jvm_lib_jars_abs),
+              # NB: The directory to dump the semanticdb jars generated by metacp.
+              '--out', rsc_index_dir,
+              os.pathsep.join(cp_entries),
+            ]
+            metacp_wu = self._runtool(
+              'scala.meta.cli.Metacp',
+              'metacp',
+              args,
+              distribution,
+              tgt=tgt,
+              input_files=tuple(scalac_classpath_path_entries + classpath_rel),
+              output_dir=rsc_index_dir)
+            metacp_stdout = stdout_contents(metacp_wu)
+            metacp_result = json.loads(metacp_stdout)
+
+            metai_classpath = self._collect_metai_classpath(metacp_result, classpath_rel)
+
+            # Step 1.5: metai Index the semanticdbs
+            # -------------------------------------
+            self._run_metai_tool(distribution, metai_classpath, rsc_index_dir, tgt)
+          else:
+            # NB: there are no unmetacp'd dependencies
+            metai_classpath = []
 
           # Step 2: Outline Scala sources into SemanticDB
           # ---------------------------------------------
@@ -403,24 +497,23 @@ def work_for_vts_rsc(vts, ctx):
       self.register_extra_products_from_contexts([ctx.target], compile_contexts)
 
     def work_for_vts_rsc_jar_library(vts, ctx):
-
-      cp_entries = []
-
       distribution = self._get_jvm_distribution()
-      jvm_lib_jars_abs = distribution.find_libs(['rt.jar', 'dt.jar', 'jce.jar', 'tools.jar'])
-      cp_entries.extend(jvm_lib_jars_abs)
 
       # TODO use compile_classpath
       classpath_abs = [
         path for (conf, path) in
         self.context.products.get_data('rsc_classpath').get_for_target(ctx.target)
       ]
+
       dependency_classpath = self._zinc.compile_classpath(
         'compile_classpath',
         ctx.target,
         extra_cp_entries=self._extra_compile_time_classpath)
       dependency_classpath = fast_relpath_collection(dependency_classpath)
+
       classpath_rel = fast_relpath_collection(classpath_abs)
+
+      cp_entries = []
       cp_entries.extend(classpath_rel)
 
       counter_val = str(counter()).rjust(counter.format_length(), b' ')
@@ -446,16 +539,25 @@ def work_for_vts_rsc_jar_library(vts, ctx):
         rsc_index_dir = fast_relpath(ctx.rsc_index_dir, get_buildroot())
         args = [
           '--verbose',
-          # NB: Without this setting, rsc will be missing some symbols
-          #     from the scala library.
-          '--include-scala-library-synthetics', # TODO generate these once and cache them
           # NB: We need to add these extra dependencies in order to be able
           #     to find symbols used by the scalac jars.
-          '--dependency-classpath', os.pathsep.join(dependency_classpath + scalac_classpath_path_entries),
+          '--dependency-classpath', os.pathsep.join(
+            dependency_classpath +
+            scalac_classpath_path_entries +
+            fast_relpath_collection(self._jvm_lib_jars_abs)
+          ),
           # NB: The directory to dump the semanticdb jars generated by metacp.
           '--out', rsc_index_dir,
           os.pathsep.join(cp_entries),
         ]
+
+        # NB: If we're building a scala library jar,
+        #     also request that metacp generate the indices
+        #     for the scala synthetics.
+        if self._is_scala_core_library(tgt):
+          args = [
+            '--include-scala-library-synthetics',
+          ] + args
         metacp_wu = self._runtool(
           'scala.meta.cli.Metacp',
           'metacp',
@@ -467,8 +569,7 @@ def work_for_vts_rsc_jar_library(vts, ctx):
         metacp_stdout = stdout_contents(metacp_wu)
         metacp_result = json.loads(metacp_stdout)
 
-        metai_classpath = self._collect_metai_classpath(
-          metacp_result, classpath_rel, jvm_lib_jars_abs)
+        metai_classpath = self._collect_metai_classpath(metacp_result, classpath_rel)
 
         # Step 1.5: metai Index the semanticdbs
         # -------------------------------------
@@ -510,7 +611,7 @@ def work_for_vts_rsc_jar_library(vts, ctx):
             work_for_vts_rsc,
             ivts,
             compile_context_pair[0]),
-          [self._rsc_key_for_target(target) for target in invalid_dependencies],
+          [self._rsc_key_for_target(target) for target in invalid_dependencies] + ['metacp(jdk)'],
           self._size_estimator(compile_context_pair[0].sources),
         )
       )
@@ -523,7 +624,7 @@ def work_for_vts_rsc_jar_library(vts, ctx):
             work_for_vts_rsc_jar_library,
             ivts,
             compile_context_pair[0]),
-          [self._rsc_key_for_target(target) for target in invalid_dependencies],
+          [self._rsc_key_for_target(target) for target in invalid_dependencies] + ['metacp(jdk)'],
           self._size_estimator(compile_context_pair[0].sources),
           on_success=ivts.update,
           on_failure=ivts.force_invalidate,
@@ -558,6 +659,8 @@ def work_for_vts_rsc_jar_library(vts, ctx):
           ] + [
             self._rsc_key_for_target(target)
             for target in invalid_dependencies_without_jar_metacps
+          ] + [
+            'metacp(jdk)'
           ],
           self._size_estimator(compile_context_pair[1].sources),
           # NB: right now, only the last job will write to the cache, because we don't
@@ -743,38 +846,35 @@ def _run_metai_tool(self,
       output_dir=rsc_index_dir
     )
 
-  def _collect_metai_classpath(self, metacp_result, classpath_rel, jvm_lib_jars):
+  def _collect_metai_classpath(self, metacp_result, relative_input_paths):
     metai_classpath = []
 
+    relative_workdir = fast_relpath(
+      self.context.options.for_global_scope().pants_workdir,
+      get_buildroot())
     # NB The json uses absolute paths pointing into either the buildroot or
     #    the temp directory of the hermetic build. This relativizes the keys.
-    prefix = None
-    for an_abs_result_path in metacp_result["status"].keys():
-      for rel_input_path in (classpath_rel + jvm_lib_jars):
-        if an_abs_result_path.endswith(rel_input_path):
-          prefix = an_abs_result_path[:-len(rel_input_path)]
-          break
-      if prefix:
-        break
-
-    def desandboxify_pantsd_loc(path):
-      if path.startswith(prefix):
-        return path[len(prefix):]
-      else:
-        return path
+    #    TODO remove this after https://github.com/scalameta/scalameta/issues/1791 is released
+    desandboxify = _create_desandboxify_fn(
+      [
+        os.path.join(relative_workdir, 'ivy', 'jars'),
+        os.path.join(relative_workdir, 'compile', 'rsc'),
+        os.path.join(relative_workdir, '.jdk'),
+        '.jdk'
+      ]
+      )
 
     status_elements = {
-      desandboxify_pantsd_loc(k): v
+      desandboxify(k): desandboxify(v)
       for k,v in metacp_result["status"].items()
     }
 
-    for cp_entry in classpath_rel:
-      metai_classpath.append(desandboxify_pantsd_loc(status_elements[cp_entry]))
-    for cp_entry in jvm_lib_jars:
-      metai_classpath.append(desandboxify_pantsd_loc(status_elements[cp_entry]))
+    for cp_entry in relative_input_paths:
+      metai_classpath.append(status_elements[cp_entry])
 
-    # TODO when these are generated once, we won't need to collect them here.
-    metai_classpath.append(desandboxify_pantsd_loc(metacp_result["scalaLibrarySynthetics"]))
+    scala_lib_synthetics = metacp_result["scalaLibrarySynthetics"]
+    if scala_lib_synthetics:
+      metai_classpath.append(desandboxify(scala_lib_synthetics))
 
     return metai_classpath
 
@@ -784,7 +884,7 @@ def _get_jvm_distribution(self):
     # See: https://github.com/pantsbuild/pants/issues/6416 for covering using
     #      different jdks in remote builds.
     local_distribution = JvmPlatform.preferred_jvm_distribution([], strict=True)
-    if self.execution_strategy == self.HERMETIC:
+    if self.execution_strategy == self.HERMETIC and self.get_options().remote_execution_server:
       class HermeticDistribution(object):
         def __init__(self, home_path, distribution):
           self._underlying = distribution


Test patch:
diff --git a/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile.py b/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile.py
--- a/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile.py
+++ b/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile.py
@@ -13,7 +13,7 @@
 from pants.backend.jvm.targets.scala_library import ScalaLibrary
 from pants.backend.jvm.tasks.classpath_products import ClasspathProducts
 from pants.backend.jvm.tasks.jvm_compile.execution_graph import ExecutionGraph
-from pants.backend.jvm.tasks.jvm_compile.rsc.rsc_compile import RscCompile
+from pants.backend.jvm.tasks.jvm_compile.rsc.rsc_compile import RscCompile, _create_desandboxify_fn
 from pants.java.jar.jar_dependency import JarDependency
 from pants.util.contextutil import temporary_dir
 from pants_test.subsystem.subsystem_util import init_subsystem
@@ -98,6 +98,11 @@ def test_metacp_job_scheduled_for_jar_library(self):
       dependee_graph = exec_graph.format_dependee_graph()
 
       self.assertEqual(dedent("""
+                     metacp(jdk) -> {
+                       rsc(java/classpath:scala_lib),
+                       compile_against_rsc(java/classpath:scala_lib),
+                       metacp(java/classpath:jar_lib)
+                     }
                      compile_against_rsc(java/classpath:java_lib) -> {}
                      rsc(java/classpath:scala_lib) -> {
                        compile_against_rsc(java/classpath:scala_lib)
@@ -107,3 +112,19 @@ def test_metacp_job_scheduled_for_jar_library(self):
                        rsc(java/classpath:scala_lib)
                      }""").strip(),
         dependee_graph)
+
+  def test_desandbox_fn(self):
+    # TODO remove this after https://github.com/scalameta/scalameta/issues/1791 is released
+    desandbox = _create_desandboxify_fn(['.pants.d/cool/beans', '.pants.d/c/r/c'])
+    self.assertEqual(desandbox('/some/path/.pants.d/cool/beans'), '.pants.d/cool/beans')
+    self.assertEqual(desandbox('/some/path/.pants.d/c/r/c/beans'), '.pants.d/c/r/c/beans')
+    self.assertEqual(desandbox(
+      '/some/path/.pants.d/exec-location/.pants.d/c/r/c/beans'),
+      '.pants.d/c/r/c/beans')
+    self.assertEqual(desandbox('/some/path/outside/workdir'), '/some/path/outside/workdir')
+    self.assertEqual(desandbox(None), None)
+    # ensure that temp workdirs are discovered relative to the buildroot
+    desandbox = _create_desandboxify_fn(['.pants.d/tmp.pants.d/cool/beans', '.pants.d/tmp.pants.d/c/r/c'])
+    self.assertEqual(desandbox('/some/path/.pants.d/tmp.pants.d/cool/beans'), '.pants.d/tmp.pants.d/cool/beans')
+    self.assertEqual(desandbox('/some/path/.pants.d/exec-location/.pants.d/tmp.pants.d/cool/beans'),
+                               '.pants.d/tmp.pants.d/cool/beans')
diff --git a/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile_integration.py b/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile_integration.py
--- a/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile_integration.py
+++ b/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile_integration.py
@@ -15,7 +15,8 @@ def test_basic_binary(self):
     with temporary_dir() as cache_dir:
       config = {
         'cache.compile.rsc': {'write_to': [cache_dir]},
-        'jvm-platform': {'compiler': 'rsc'}
+        'jvm-platform': {'compiler': 'rsc'},
+        'compile.rsc': {'execution_strategy': 'subprocess'},
       }
 
       pants_run = self.run_pants(