Repository: pantsbuild/pants

Issue:
[rsc-compile] break out java metacping into a separate job
Currently java targets are metacp'd as part of building each scala target. It would be better to metacp them once per java target.

There's a couple approaches here
1. create a new `Job` that
- depends on java zinc compiles
- is depended on by rsc scala compiles
- runs metacp on the result of a java zinc compile
2. wrap the existing java zinc work fn with a fn that runs metacp after invoking the zinc compile. This doesn't require any network changes.

This is very similar to https://github.com/pantsbuild/pants/commit/2be0b1d843e77c0fa6e74f00689ece1911211721#diff-074dd871ff62c835ab90d4a0a1264a63

- The remaining metacp portion of `work_for_vts_rsc` https://github.com/pantsbuild/pants/blob/master/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py#L435
- The jar_library metacp fn https://github.com/pantsbuild/pants/blob/master/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py#L520

I haven't spent much time refactoring this, so there is a lot of duplication.




Discussion:
Thank you @baroquebobcat... will tackle this this/next week.

Resolving patch:
diff --git a/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py b/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py
--- a/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py
+++ b/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py
@@ -129,7 +129,7 @@ def __init__(self, *args, **kwargs):
 
   @classmethod
   def implementation_version(cls):
-    return super(RscCompile, cls).implementation_version() + [('RscCompile', 170)]
+    return super(RscCompile, cls).implementation_version() + [('RscCompile', 171)]
 
   @classmethod
   def register_options(cls, register):
@@ -264,15 +264,18 @@ def select(self, target):
 
   def _rsc_key_for_target(self, compile_target):
     if self._only_zinc_compileable(compile_target):
-      # rsc outlining with java dependencies depend on the java's zinc compile
-      return self._compile_against_rsc_key_for_target(compile_target)
+      # rsc outlining with java dependencies depends on the output of a second metacp job.
+      return self._metacp_key_for_target(compile_target)
     elif self._rsc_compilable(compile_target):
       return "rsc({})".format(compile_target.address.spec)
     elif self._metacpable(compile_target):
-      return "metacp({})".format(compile_target.address.spec)
+      return self._metacp_key_for_target(compile_target)
     else:
       raise TaskError('unexpected target for compiling with rsc .... {}'.format(compile_target))
 
+  def _metacp_key_for_target(self, compile_target):
+    return "metacp({})".format(compile_target.address.spec)
+
   def _compile_against_rsc_key_for_target(self, compile_target):
     return "compile_against_rsc({})".format(compile_target.address.spec)
 
@@ -404,89 +407,26 @@ def is_java_compile_target(t):
                          if not (is_java_compile_target(t)) and not isinstance(t, JarLibrary)]
 
         metacped_jar_classpath_abs = _paths_from_classpath(
-          self._metacp_jars_classpath_product.get_for_targets(jar_deps)
+          self._metacp_jars_classpath_product.get_for_targets(jar_deps + java_deps)
         )
         metacped_jar_classpath_abs.extend(self._jvm_lib_metacp_classpath)
         metacped_jar_classpath_rel = fast_relpath_collection(metacped_jar_classpath_abs)
 
-        jar_rsc_classpath_paths = _paths_from_classpath(
-          self.context.products.get_data('rsc_classpath').get_for_targets(jar_deps),
-          collection_type=set)
-        jar_rsc_classpath_rel = fast_relpath_collection(jar_rsc_classpath_paths)
-
         non_java_paths = _paths_from_classpath(
           self.context.products.get_data('rsc_classpath').get_for_targets(non_java_deps),
           collection_type=set)
         non_java_rel = fast_relpath_collection(non_java_paths)
 
-        java_paths = _paths_from_classpath(
-          self.context.products.get_data('rsc_classpath').get_for_targets(java_deps),
-          collection_type=set)
-        java_rel = fast_relpath_collection(java_paths)
-
         ctx.ensure_output_dirs_exist()
 
         distribution = self._get_jvm_distribution()
         with Timer() as timer:
-          # Step 1: Convert classpath to SemanticDB
-          # ---------------------------------------
-          # If there are any as yet not metacp'd dependencies, metacp them so their indices can
-          # be passed to Rsc.
-          # TODO move these to their own jobs. https://github.com/pantsbuild/pants/issues/6754
-
-          # Inputs
-          # - Java dependencies jars
-          metacp_inputs = java_rel
-
-          # Dependencies
-          # - 3rdparty jars
-          # - non-java, ie scala, dependencies
-          # - jdk
-          snapshotable_metacp_dependencies = list(jar_rsc_classpath_rel) + \
-                                list(non_java_rel) + \
-                                fast_relpath_collection(
-                                  _paths_from_classpath(self._extra_compile_time_classpath))
-          metacp_dependencies = snapshotable_metacp_dependencies + self._jvm_lib_jars_abs
-
-          if metacp_inputs:
-            rsc_index_dir = fast_relpath(ctx.rsc_index_dir, get_buildroot())
-            args = [
-              '--verbose',
-              '--stub-broken-signatures',
-              '--dependency-classpath', os.pathsep.join(metacp_dependencies),
-              # NB: The directory to dump the semanticdb jars generated by metacp.
-              '--out', rsc_index_dir,
-              os.pathsep.join(metacp_inputs),
-            ]
-            metacp_wu = self._runtool(
-              'scala.meta.cli.Metacp',
-              'metacp',
-              args,
-              distribution,
-              tgt=tgt,
-              input_files=tuple(metacp_inputs + snapshotable_metacp_dependencies),
-              output_dir=rsc_index_dir)
-            metacp_stdout = stdout_contents(metacp_wu)
-            metacp_result = json.loads(metacp_stdout)
-
-            metacped_java_dependency_rel = self._collect_metai_classpath(metacp_result,
-              java_rel)
-
-            # Step 1.5: metai Index the semanticdbs
-            # -------------------------------------
-            self._run_metai_tool(distribution, metacped_java_dependency_rel, rsc_index_dir, tgt)
-          else:
-            # NB: there are no unmetacp'd dependencies
-            metacped_java_dependency_rel = []
-
-
-          # Step 2: Outline Scala sources into SemanticDB
+          # Outline Scala sources into SemanticDB
           # ---------------------------------------------
           rsc_mjar_file = fast_relpath(ctx.rsc_mjar_file, get_buildroot())
 
           # TODO remove non-rsc entries from non_java_rel in a better way
-          rsc_semanticdb_classpath = metacped_java_dependency_rel + \
-                                     metacped_jar_classpath_rel + \
+          rsc_semanticdb_classpath = metacped_jar_classpath_rel + \
                                      [j for j in non_java_rel if 'compile/rsc/' in j]
           target_sources = ctx.sources
           args = [
@@ -505,7 +445,7 @@ def is_java_compile_target(t):
             output_dir=os.path.dirname(rsc_mjar_file))
 
         self._record_target_stats(tgt,
-          len(metacp_inputs),
+          len(rsc_semanticdb_classpath),
           len(target_sources),
           timer.elapsed,
           False,
@@ -517,9 +457,9 @@ def is_java_compile_target(t):
       # Update the products with the latest classes.
       self.register_extra_products_from_contexts([ctx.target], compile_contexts)
 
-    def work_for_vts_rsc_jar_library(vts, ctx):
+    def work_for_vts_metacp(vts, ctx, classpath_product_key):
       metacp_dependencies_entries = self._zinc.compile_classpath_entries(
-        'compile_classpath',
+        classpath_product_key,
         ctx.target,
         extra_cp_entries=self._extra_compile_time_classpath)
 
@@ -533,7 +473,7 @@ def work_for_vts_rsc_jar_library(vts, ctx):
 
       classpath_entries = [
         cp_entry for (conf, cp_entry) in
-        self.context.products.get_data('compile_classpath').get_classpath_entries_for_targets(
+        self.context.products.get_data(classpath_product_key).get_classpath_entries_for_targets(
           [ctx.target])
       ]
       classpath_digests = [c.directory_digest for c in classpath_entries if c.directory_digest]
@@ -653,9 +593,10 @@ def work_for_vts_rsc_jar_library(vts, ctx):
         Job(
           rsc_key,
           functools.partial(
-            work_for_vts_rsc_jar_library,
+            work_for_vts_metacp,
             ivts,
-            compile_context_pair[0]),
+            compile_context_pair[0],
+            'compile_classpath'),
           [self._rsc_key_for_target(target) for target in invalid_dependencies] + ['metacp(jdk)'],
           self._size_estimator(compile_context_pair[0].sources),
           on_success=ivts.update,
@@ -712,10 +653,10 @@ def add_for_target(self, *args, **kwargs):
           self.runtime_classpath_product.add_for_target(*args, **kwargs)
           self.rsc_classpath_product.add_for_target(*args, **kwargs)
 
-      full_key = self._compile_against_rsc_key_for_target(compile_target)
+      zinc_key = self._compile_against_rsc_key_for_target(compile_target)
       zinc_jobs.append(
         Job(
-          full_key,
+          zinc_key,
           functools.partial(
             self._default_work_for_vts,
             ivts,
@@ -730,8 +671,24 @@ def add_for_target(self, *args, **kwargs):
             self._compile_against_rsc_key_for_target(target)
             for target in invalid_dependencies_without_jar_metacps],
           self._size_estimator(compile_context_pair[1].sources),
-          # NB: right now, only the last job will write to the cache, because we don't
-          #     do multiple cache entries per target-task tuple.
+          on_failure=ivts.force_invalidate,
+        )
+      )
+
+      metacp_key = self._metacp_key_for_target(compile_target)
+      rsc_jobs.append(
+        Job(
+          metacp_key,
+          functools.partial(
+            work_for_vts_metacp,
+            ivts,
+            compile_context_pair[0],
+            'runtime_classpath'),
+          [self._metacp_key_for_target(target) for target in invalid_dependencies] + [
+              'metacp(jdk)',
+              zinc_key,
+            ],
+          self._size_estimator(compile_context_pair[0].sources),
           on_success=ivts.update,
           on_failure=ivts.force_invalidate,
         )


Test patch:
diff --git a/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile.py b/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile.py
--- a/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile.py
+++ b/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile.py
@@ -99,16 +99,21 @@ def test_metacp_job_scheduled_for_jar_library(self):
 
       self.assertEqual(dedent("""
                      metacp(jdk) -> {
+                       metacp(java/classpath:java_lib),
                        rsc(java/classpath:scala_lib),
                        compile_against_rsc(java/classpath:scala_lib),
                        metacp(java/classpath:jar_lib)
                      }
-                     compile_against_rsc(java/classpath:java_lib) -> {}
+                     metacp(java/classpath:java_lib) -> {}
+                     compile_against_rsc(java/classpath:java_lib) -> {
+                       metacp(java/classpath:java_lib)
+                     }
                      rsc(java/classpath:scala_lib) -> {
                        compile_against_rsc(java/classpath:scala_lib)
                      }
                      compile_against_rsc(java/classpath:scala_lib) -> {}
                      metacp(java/classpath:jar_lib) -> {
+                       metacp(java/classpath:java_lib),
                        rsc(java/classpath:scala_lib)
                      }""").strip(),
         dependee_graph)
@@ -172,11 +177,22 @@ def test_scala_lib_with_java_sources_not_passed_to_rsc(self):
 
       self.assertEqual(dedent("""
                      metacp(jdk) -> {
+                       metacp(java/classpath:java_lib),
+                       metacp(java/classpath:scala_and_java_lib),
                        metacp(java/classpath:jar_lib)
                      }
-                     compile_against_rsc(java/classpath:java_lib) -> {}
-                     compile_against_rsc(java/classpath:scala_and_java_lib) -> {}
-                     metacp(java/classpath:jar_lib) -> {}""").strip(),
+                     metacp(java/classpath:java_lib) -> {}
+                     compile_against_rsc(java/classpath:java_lib) -> {
+                       metacp(java/classpath:java_lib)
+                     }
+                     metacp(java/classpath:scala_and_java_lib) -> {}
+                     compile_against_rsc(java/classpath:scala_and_java_lib) -> {
+                       metacp(java/classpath:scala_and_java_lib)
+                     }
+                     metacp(java/classpath:jar_lib) -> {
+                       metacp(java/classpath:java_lib),
+                       metacp(java/classpath:scala_and_java_lib)
+                     }""").strip(),
         dependee_graph)
 
   def test_desandbox_fn(self):