Repository: pantsbuild/pants

Issue:
runtime_classpath product is populated unconditionally for all zinc compiles
Currently we populate the `runtime_classpath` in a batch before doing all compiles: https://github.com/pantsbuild/pants/blob/d748a10adabca8bc710563a3903d1c92536de402/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py#L383-L386

I'm about to make it so that this adding sometimes also contains a `DirectoryDigest`. In order to do this, it would be useful if we did this adding closer to where the compile happens, rather than up-front. It would also be useful if the replacement at https://github.com/pantsbuild/pants/blob/d748a10adabca8bc710563a3903d1c92536de402/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py#L399-L406 happened close to where the compile happens (so that we can replace properly with the right `DirectoryDigest`), rather than in a blunt batch.

Ideally, this would happen inside the `Job` in the `ExecutionGraph` (https://github.com/pantsbuild/pants/blob/d748a10adabca8bc710563a3903d1c92536de402/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py#L667-L726), but it's possible that won't work for multiprocess reasons.

Note that for valid vts, no `Job` will actually be run, so we'll still need to fall back to registering the `runtime_classpath` path for valid vts somewhere - keeping on doing it where it's currently done is as good a place as any.


Discussion:

Resolving patch:
diff --git a/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py b/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py
--- a/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py
+++ b/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py
@@ -367,11 +367,6 @@ def execute(self):
     # Clone the compile_classpath to the runtime_classpath.
     classpath_product = self.create_runtime_classpath()
 
-    def classpath_for_context(context):
-      if self.get_options().use_classpath_jars:
-        return context.jar_file
-      return context.classes_dir
-
     fingerprint_strategy = DependencyContext.global_instance().create_fingerprint_strategy(
         classpath_product)
     # Note, JVM targets are validated (`vts.update()`) as they succeed.  As a result,
@@ -382,25 +377,14 @@ def classpath_for_context(context):
                           fingerprint_strategy=fingerprint_strategy,
                           topological_order=True) as invalidation_check:
 
-
-      # Register runtime classpath products for all targets.
       compile_contexts = {vt.target: self.create_compile_context(vt.target, vt.results_dir)
                           for vt in invalidation_check.all_vts}
-      for ccs in compile_contexts.values():
-        cc = self.select_runtime_context(ccs)
-        classpath_product.add_for_target(cc.target, [(conf, classpath_for_context(cc))
-                                                     for conf in self._confs])
-
-      # Register products for valid targets.
-      valid_targets = [vt.target for vt in invalidation_check.all_vts if vt.valid]
-      self.register_extra_products_from_contexts(valid_targets, compile_contexts)
-
-      # Build any invalid targets (which will register products in the background).
-      if invalidation_check.invalid_vts:
-        self.do_compile(
-          invalidation_check,
-          compile_contexts,
-        )
+
+      self.do_compile(
+        invalidation_check,
+        compile_contexts,
+        classpath_product,
+      )
 
       if not self.get_options().use_classpath_jars:
         # Once compilation has completed, replace the classpath entry for each target with
@@ -411,6 +395,11 @@ def classpath_for_context(context):
             classpath_product.remove_for_target(cc.target, [(conf, cc.classes_dir)])
             classpath_product.add_for_target(cc.target, [(conf, cc.jar_file)])
 
+  def _classpath_for_context(self, context):
+    if self.get_options().use_classpath_jars:
+      return context.jar_file
+    return context.classes_dir
+
   def create_runtime_classpath(self):
     compile_classpath = self.context.products.get_data('compile_classpath')
     classpath_product = self.context.products.get_data('runtime_classpath')
@@ -421,11 +410,23 @@ def create_runtime_classpath(self):
 
     return classpath_product
 
-  def do_compile(self, invalidation_check, compile_contexts):
+  def do_compile(self, invalidation_check, compile_contexts, classpath_product):
     """Executes compilations for the invalid targets contained in a single chunk."""
 
     invalid_targets = [vt.target for vt in invalidation_check.invalid_vts]
-    assert invalid_targets, "compile_chunk should only be invoked if there are invalid targets."
+    valid_targets = [vt.target for vt in invalidation_check.all_vts if vt.valid]
+
+    # Register classpaths and products for valid targets.
+    for valid_target in valid_targets:
+      cc = self.select_runtime_context(compile_contexts[valid_target])
+      classpath_product.add_for_target(
+        valid_target,
+        [(conf, self._classpath_for_context(cc)) for conf in self._confs],
+      )
+    self.register_extra_products_from_contexts(valid_targets, compile_contexts)
+
+    if not invalid_targets:
+      return
 
     # This ensures the workunit for the worker pool is set before attempting to compile.
     with self.context.new_workunit('isolation-{}-pool-bootstrap'.format(self.name())) \
@@ -446,7 +447,8 @@ def do_compile(self, invalidation_check, compile_contexts):
     # generated by upstream JVM tasks and our own prepare_compile().
     jobs = self._create_compile_jobs(compile_contexts,
                                      invalid_targets,
-                                     invalidation_check.invalid_vts)
+                                     invalidation_check.invalid_vts,
+                                     classpath_product)
 
     exec_graph = ExecutionGraph(jobs, self.get_options().print_exception_stacktrace)
     try:
@@ -640,7 +642,7 @@ def _upstream_analysis(self, compile_contexts, classpath_entries):
   def exec_graph_key_for_target(self, compile_target):
     return "compile({})".format(compile_target.address.spec)
 
-  def _create_compile_jobs(self, compile_contexts, invalid_targets, invalid_vts):
+  def _create_compile_jobs(self, compile_contexts, invalid_targets, invalid_vts, classpath_product):
     class Counter(object):
       def __init__(self, size, initial=0):
         self.size = size
@@ -664,11 +666,11 @@ def format_length(self):
 
       jobs.extend(
         self.create_compile_jobs(compile_target, compile_contexts, invalid_dependencies, ivts,
-          counter))
+          counter, classpath_product))
     return jobs
 
   def create_compile_jobs(self, compile_target, all_compile_contexts, invalid_dependencies, ivts,
-    counter):
+    counter, classpath_product):
 
     def work_for_vts(vts, ctx):
       progress_message = ctx.target.address.spec
@@ -717,6 +719,10 @@ def work_for_vts(vts, ctx):
         self._create_context_jar(ctx)
 
       # Update the products with the latest classes.
+      classpath_product.add_for_target(
+        ctx.target,
+        [(conf, self._classpath_for_context(ctx)) for conf in self._confs],
+      )
       self.register_extra_products_from_contexts([ctx.target], all_compile_contexts)
 
     context_for_target = all_compile_contexts[compile_target]


Test patch: