Repository: pantsbuild/pants

Issue:
ConsolidateClasspath assumes ClasspathEntries have a consistent ordering
This code: https://github.com/pantsbuild/pants/blob/670f8980650998e4c7f88610a294a8ec52858637/src/python/pants/backend/jvm/tasks/consolidate_classpath.py#L46-L66

assumes that across different runs of Pants, a single Target's ClasspathEntries will iterate in a consistent order. It creates its outputs based on that order.

With classpath jars enabled, I have an example where a JVM target has both a directory ClasspathEntry (because it has services) and a jar file ClasspathEntry (for its compiled code).

A bundle command will non-deterministically fail based on observed orderings, if the iteration order of ClasspathEntries is different between when the target was invalid and valid. Nothing currently attempts to keep that order consistent. I have observed the following sources of changes of order:

1. (The one specifically causing the bug I was investigating): The first run (when the target is invalid), the ClasspathEntry for the resources is added _before_ the compile is done (and the jar ClasspathEntry added), by ResourcesTask. The latter runs (when the target is valid), ResourcesTask skips itself, and the ClasspathEntry for the resources is added _after_ the compile is done (and the jar ClasspathEntry added), by the RscCompile task.
2. A handful of tasks call ClasspathProducts.remove_for_target then ClasspathProducts.add_for_target to replace one entry with another, but doesn't add the new entry at the same index as the removed entry. This is trivial to fix by adding an atomic replace_for_target function which preserves ordering.

Some possible fixes to 1:
* Stop using ordering to determine ConsolidateClasspath's output. Instead use a hash of the path of the directory relative to the workdir, or give names to the ClasspathEntries, or something similar.
* Specify some ordering for the different ClasspathEntry kinds (e.g. dir always before jar) and ensure this is upheld by anything which adds new ClasspathEntries.
* Make sure ResourcesTask consistently does the adding of resources, whether or not it generated them, rather than relying on RscCompile to clean up after the fact.
* For this specific case, but not the general case: Have service generation generate jar files, rather than directories.

Any thoughts?


Discussion:
Great investigation... argh, v1.

> Stop using ordering to determine ConsolidateClasspath's output. Instead use a hash of the path of the directory relative to the workdir, or give names to the ClasspathEntries, or something similar.

Hash should work. I think that the primary cause of this issue would be having multiple `Task`s installed, and the order of `Tasks` running not being stable (possibly due to running goals with different requirements one after the other...?).

Resolving patch:
diff --git a/src/python/pants/backend/jvm/tasks/consolidate_classpath.py b/src/python/pants/backend/jvm/tasks/consolidate_classpath.py
--- a/src/python/pants/backend/jvm/tasks/consolidate_classpath.py
+++ b/src/python/pants/backend/jvm/tasks/consolidate_classpath.py
@@ -6,7 +6,9 @@
 
 from pants.backend.jvm.tasks.classpath_util import ClasspathUtil
 from pants.backend.jvm.tasks.jvm_binary_task import JvmBinaryTask
+from pants.base.hash_utils import hash_all
 from pants.build_graph.target_scopes import Scopes
+from pants.util.dirutil import fast_relpath
 
 
 class ConsolidateClasspath(JvmBinaryTask):
@@ -17,7 +19,7 @@ class ConsolidateClasspath(JvmBinaryTask):
 
   @classmethod
   def implementation_version(cls):
-    return super().implementation_version() + [('ConsolidateClasspath', 1)]
+    return super().implementation_version() + [('ConsolidateClasspath', 2)]
 
   @classmethod
   def prepare(cls, options, round_manager):
@@ -56,9 +58,11 @@ def _consolidate_classpath(self, targets, classpath_products):
     with self.invalidated(targets=targets, invalidate_dependents=True) as invalidation:
       for vt in invalidation.all_vts:
         entries = entries_map.get(vt.target, [])
-        for index, (conf, entry) in enumerate(entries):
+        for conf, entry in entries:
+          relpath = fast_relpath(entry.path, self.get_options().pants_workdir)
+          suffix = hash_all([relpath])[:6]
           if ClasspathUtil.is_dir(entry.path):
-            jarpath = os.path.join(vt.results_dir, f'output-{index}.jar')
+            jarpath = os.path.join(vt.results_dir, f'output-{suffix}.jar')
 
             # Regenerate artifact for invalid vts.
             if not vt.valid:
diff --git a/src/python/pants/engine/round_manager.py b/src/python/pants/engine/round_manager.py
--- a/src/python/pants/engine/round_manager.py
+++ b/src/python/pants/engine/round_manager.py
@@ -3,6 +3,8 @@
 
 from collections import defaultdict, namedtuple
 
+from twitter.common.collections import OrderedSet
+
 from pants.goal.goal import Goal
 
 
@@ -20,7 +22,7 @@ class MissingProductError(KeyError):
 
   @staticmethod
   def _index_products():
-    producer_info_by_product_type = defaultdict(set)
+    producer_info_by_product_type = defaultdict(OrderedSet)
     for goal in Goal.all():
       for task_type in goal.task_types():
         for product_type in task_type.product_types():
@@ -29,8 +31,8 @@ def _index_products():
     return producer_info_by_product_type
 
   def __init__(self, context):
-    self._dependencies = set()
-    self._optional_dependencies = set()
+    self._dependencies = OrderedSet()
+    self._optional_dependencies = OrderedSet()
     self._context = context
     self._producer_infos_by_product_type = None
 
@@ -80,7 +82,7 @@ def optional_data(self, product_type):
 
   def get_dependencies(self):
     """Returns the set of data dependencies as producer infos corresponding to data requirements."""
-    producer_infos = set()
+    producer_infos = OrderedSet()
     for product_type in self._dependencies:
       producer_infos.update(self._get_producer_infos_by_product_type(product_type))
     return producer_infos


Test patch:
diff --git a/tests/python/pants_test/backend/jvm/tasks/test_consolidate_classpath.py b/tests/python/pants_test/backend/jvm/tasks/test_consolidate_classpath.py
--- a/tests/python/pants_test/backend/jvm/tasks/test_consolidate_classpath.py
+++ b/tests/python/pants_test/backend/jvm/tasks/test_consolidate_classpath.py
@@ -2,6 +2,8 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 import os
+import re
+from typing import List
 
 from pants.backend.jvm.targets.jar_library import JarLibrary
 from pants.backend.jvm.targets.java_library import JavaLibrary
@@ -85,13 +87,14 @@ def test_remove_raw_deps(self):
       'pants_backend_jvm_tasks_consolidate_classpath_ConsolidateClasspath'
     )
     found_files = [os.path.basename(f) for f in self.iter_files(task_dir)]
+    consolidate_classpath_jar = self.find_consolidate_classpath_jar(found_files)
     self.assertEqual(
-      sorted(['output-0.jar', 'Foo.class', 'foo.txt', 'file']),
+      sorted([consolidate_classpath_jar, 'Foo.class', 'foo.txt', 'file']),
       sorted(found_files)
     )
 
     # Confirm that we haven't destroyed deps.
-    expected_non_deps = {'output-0.jar', 'Foo.class', 'foo.txt', 'file'}
+    expected_non_deps = {consolidate_classpath_jar, 'Foo.class', 'foo.txt', 'file'}
     found = set(os.listdir(self.pants_workdir))
     print(expected_non_deps - found)
     self.assertTrue(expected_non_deps - found == expected_non_deps)
@@ -110,8 +113,9 @@ def test_consolidate_classpath(self):
       'pants_backend_jvm_tasks_consolidate_classpath_ConsolidateClasspath'
     )
     found_files = [os.path.basename(f) for f in self.iter_files(task_dir)]
+    consolidate_classpath_jar = self.find_consolidate_classpath_jar(found_files)
     self.assertEqual(
-      sorted(['output-0.jar', 'Foo.class', 'foo.txt', 'file']),
+      sorted([consolidate_classpath_jar, 'Foo.class', 'foo.txt', 'file']),
       sorted(found_files)
     )
 
@@ -122,3 +126,10 @@ def test_consolidate_classpath(self):
                      'org.pantsbuild-bar-2.0.0.zip'}
     found = set(os.listdir(self.pants_workdir))
     self.assertTrue(expected_deps - found == set())
+
+  @staticmethod
+  def find_consolidate_classpath_jar(files: List[str]) -> str:
+    matching = [f for f in files if re.match("output-[0-9a-f]{6}\.jar", f)]
+    if len(matching) == 1:
+      return matching[0]
+    raise ValueError(f"Could not find single consolidate classpath jar in {files}.")