Repository: pantsbuild/pants

Issue:
rsc-and-zinc hermetic on linux failing on globbing jdk
Consistently reproducible on ubuntu docker on master:
```
# ./pants --compile-rsc-worker-count=8 --process-execution-local-parallelism=8 --compile-rsc-execution-strategy=hermetic --no-compile-rsc-incremental --compile-rsc-workflow=rsc-and-zinc --cache-compile-rsc-ignore compile examples/src/scala/org/pantsbuild/example/hello/exe
Scrubbed PYTHONPATH=/root/workspace/pants/src/python: from the environment.

03:08:23 00:00 [main]
               (To run a reporting server: ./pants server)
03:08:24 00:01   [setup]
03:08:24 00:01     [parse]
               Executing tasks in goals: bootstrap -> imports -> unpack-jars -> unpack-wheels -> deferred-sources -> native-compile -> gen -> jvm-platform-validate -> resolve -> resources -> compile
03:08:25 00:02   [bootstrap]
03:08:25 00:02     [substitute-aliased-targets]
03:08:25 00:02     [jar-dependency-management]
03:08:25 00:02     [bootstrap-jvm-tools]
03:08:25 00:02     [provide-tools-jar]
03:08:25 00:02   [imports]
03:08:25 00:02     [ivy-imports]
03:08:25 00:02   [unpack-jars]
03:08:25 00:02     [unpack-jars]
03:08:25 00:02   [unpack-wheels]
03:08:25 00:02     [unpack-wheels]
03:08:25 00:02   [deferred-sources]
03:08:25 00:02     [deferred-sources]
03:08:25 00:02   [native-compile]
03:08:25 00:02     [conan-prep]
03:08:25 00:02     [conan-fetch]
03:08:25 00:02     [c-for-ctypes]
03:08:25 00:02     [cpp-for-ctypes]
03:08:25 00:02   [gen]
03:08:25 00:02     [antlr-java]
03:08:25 00:02     [antlr-py]
03:08:25 00:02     [jaxb]
03:08:25 00:02     [protoc]
03:08:25 00:02     [ragel]
03:08:25 00:02     [thrift-java]
03:08:25 00:02     [thrift-py]
03:08:25 00:02     [py-thrift-namespace-clash-check]
03:08:25 00:02     [grpcio-prep]
03:08:25 00:02     [grpcio-run]
03:08:25 00:02     [wire]
03:08:25 00:02     [avro-java]
03:08:25 00:02     [go-thrift]
03:08:25 00:02     [go-protobuf]
03:08:25 00:02     [jax-ws]
03:08:25 00:02     [scrooge]
03:08:25 00:02     [thrifty]
03:08:25 00:02   [jvm-platform-validate]
03:08:25 00:02     [jvm-platform-validate]
03:08:25 00:02   [resolve]
03:08:25 00:02     [ivy]03:08:25 [WARN] /root/workspace/pants/src/python/pants/engine/round_engine.py:45: DeprecationWarning: DEPRECATED: Ivy Resolve will be removed in version 1.27.0.dev0.
  resolve.ivy as well as --resolver-resolver=ivy will be removed.Please use --resolver-resolver=coursier instead and refer to https://www.pantsbuild.org/jvm_projects.html#coursier for setup.
  task.execute()


03:08:26 00:03     [coursier]
03:08:26 00:03     [go]
03:08:26 00:03     [scala-js-compile]
03:08:26 00:03     [scala-js-link]
03:08:26 00:03     [node]
03:08:26 00:03   [resources]
03:08:26 00:03     [prepare]
03:08:26 00:03     [services]
03:08:26 00:03   [compile]
03:08:26 00:03     [node]
03:08:26 00:03     [compile-jvm-prep-command]
03:08:26 00:03       [jvm_prep_command]
03:08:26 00:03     [compile-prep-command]
03:08:26 00:03     [compile]
03:08:26 00:03     [rsc]
                   Invalidated 3 targets.
03:08:26 00:03       [isolation-mixed-pool-bootstrap]
                   [1/5] Compiling 1 mixed source in 1 target (examples/src/java/org/pantsbuild/example/hello/greet:greet).
03:08:26 00:03       [compile]
                     
03:08:26 00:03         [mixed]
                       [info] Compiling 1 Java source to /tmp/process-executionolLSFj/./.pants.d/compile/rsc/c1e3836b60e5/examples.src.java.org.pantsbuild.example.hello.greet.greet/current/zinc/classes ...
                       [info] Done compiling.
                       [info] Compile success at Oct 11, 2019 9:33:04 PM [3.068s]
                       
                   [2/5] Rsc-ing 1 mixed source in 1 target (examples/src/scala/org/pantsbuild/example/hello/welcome:welcome).
                   [3/5] Compiling 1 mixed source in 1 target (examples/src/scala/org/pantsbuild/example/hello/welcome:welcome).
03:08:26 00:03       [compile]
                     
                   rsc(examples/src/scala/org/pantsbuild/example/hello/welcome:welcome) failed: Error expanding globs: Failed to scan directory "/usr/lib/jvm/java-8-openjdk-amd64/": No such file or directory (os error 2)
                   Traceback:
                     File "/root/workspace/pants/src/python/pants/backend/jvm/tasks/jvm_compile/execution_graph.py", line 281, in worker
                       work()
                   
                     File "/root/workspace/pants/src/python/pants/backend/jvm/tasks/jvm_compile/execution_graph.py", line 42, in __call__
                       self.fn()
                   
                     File "/root/workspace/pants/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py", line 380, in work_for_vts_rsc
                       self.NAILGUN: nonhermetic_digest_classpath,
                   
                     File "/root/workspace/pants/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py", line 367, in hermetic_digest_classpath
                       jdk_libs_rel, jdk_libs_digest = self._jdk_libs_paths_and_digest(distribution)
                   
                     File "/root/workspace/pants/src/python/pants/util/memo.py", line 115, in memoize
                       result = func(*args, **kwargs)
                   
                     File "/root/workspace/pants/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py", line 733, in _jdk_libs_paths_and_digest
                       (jdk_libs_globs,))[0].directory_digest
                   
                     File "/root/workspace/pants/src/python/pants/engine/scheduler.py", line 560, in capture_snapshots
                       return self._scheduler._raise_or_return(result)
                   
                     File "/root/workspace/pants/src/python/pants/engine/scheduler.py", line 175, in _raise_or_return
                       return self._native.context.raise_or_return(pyresult)
                   
                     File "/root/workspace/pants/src/python/pants/engine/native.py", line 592, in raise_or_return
                       raise value
                   
03:08:26 00:03         [mixed]
                       [info] Compiling 1 Scala source to /tmp/process-executionDYy8Z4/./.pants.d/compile/rsc/c1e3836b60e5/examples.src.scala.org.pantsbuild.example.hello.welcome.welcome/current/zinc/classes ...
                       [info] Done compiling.
                       [info] Compile success at Oct 11, 2019 9:33:10 PM [5.590s]
                       
FAILURE: Compilation failure: Failed jobs: rsc(examples/src/scala/org/pantsbuild/example/hello/welcome:welcome)


               Waiting for background workers to finish.
03:08:26 00:03   [complete]
               FAILURE
```
but the path actually exists
```
# ls -l /usr/lib/jvm/java-8-openjdk-amd64/
total 20
lrwxrwxrwx 1 root root   22 Jul 18 18:52 ASSEMBLY_EXCEPTION -> jre/ASSEMBLY_EXCEPTION
drwxr-xr-x 2 root root 4096 Aug 10 07:35 bin
lrwxrwxrwx 1 root root   41 Jul 18 18:52 docs -> ../../../share/doc/openjdk-8-jre-headless
drwxr-xr-x 3 root root 4096 Aug 10 07:35 include
drwxr-xr-x 5 root root 4096 Aug 10 07:35 jre
drwxr-xr-x 3 root root 4096 Aug 10 07:35 lib
drwxr-xr-x 4 root root 4096 Aug 10 07:35 man
lrwxrwxrwx 1 root root   20 Jul 18 18:52 src.zip -> ../openjdk-8/src.zip
lrwxrwxrwx 1 root root   22 Jul 18 18:52 THIRD_PARTY_README -> jre/THIRD_PARTY_README
```


Discussion:

Resolving patch:
diff --git a/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py b/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py
--- a/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py
+++ b/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py
@@ -908,12 +908,16 @@ def find_libs(self, names):
       return [self._rehome(l) for l in underlying_libs]
 
     def find_libs_path_globs(self, names):
-      libs_abs = self._underlying.find_libs(names)
-      libs_unrooted = [self._unroot_lib_path(l) for l in libs_abs]
-      path_globs = PathGlobsAndRoot(
-        PathGlobs(tuple(libs_unrooted)),
-        self._underlying.home)
-      return (libs_unrooted, path_globs)
+      path_globs = []
+      filenames = []
+      # We have to move the jars to top level directory because globbing jdk home with symlinks
+      # would cause failing to scan the directory. https://github.com/pantsbuild/pants/issues/8460
+      for lib_abs in self._underlying.find_libs(names):
+        root = os.path.dirname(lib_abs)
+        filename = os.path.basename(lib_abs)
+        filenames.append(filename)
+        path_globs.append(PathGlobsAndRoot(PathGlobs((filename,)), root))
+      return (filenames, path_globs)
 
     @property
     def java(self):
diff --git a/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py b/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py
--- a/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py
+++ b/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py
@@ -747,8 +747,9 @@ def _runtool(self, distribution, input_digest, ctx):
   @memoized_method
   def _jdk_libs_paths_and_digest(self, hermetic_dist):
     jdk_libs_rel, jdk_libs_globs = hermetic_dist.find_libs_path_globs(self._JDK_LIB_NAMES)
-    jdk_libs_digest = self.context._scheduler.capture_snapshots(
-      (jdk_libs_globs,))[0].directory_digest
+    jdk_libs_digest = self.context._scheduler.merge_directories([
+      snap.directory_digest for snap in (self.context._scheduler.capture_snapshots(jdk_libs_globs))
+    ])
     return (jdk_libs_rel, jdk_libs_digest)
 
   @memoized_method


Test patch:
diff --git a/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile_integration.py b/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile_integration.py
--- a/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile_integration.py
+++ b/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile_integration.py
@@ -3,66 +3,31 @@
 
 import os
 
-from pants.backend.jvm.subsystems.resolve_subsystem import JvmResolveSubsystem
 from pants.backend.jvm.tasks.jvm_compile.rsc.rsc_compile import RscCompile
-from pants.util.contextutil import temporary_dir
+from pants.util.contextutil import environment_as
 from pants_test.backend.jvm.tasks.jvm_compile.base_compile_integration_test import BaseCompileIT
-from pants_test.test_base import AbstractTestGenerator
 
 
-def _for_all_supported_execution_environments(func):
-  func._with_run_config = True
-  return func
+def ensure_compile_rsc_execution_strategy(f):
+  """A decorator for running an integration test with ivy and coursier as the resolver."""
 
+  def wrapper(self, *args, **kwargs):
+    for strategy in RscCompile.ExecutionStrategy.all_values():
+      with environment_as(
+        HERMETIC_ENV='PANTS_COMPILE_RSC_EXECUTION_STRATEGY',
+        PANTS_COMPILE_RSC_EXECUTION_STRATEGY=strategy.value,
+        PANTS_COMPILE_RSC_WORKFLOW='rsc-and-zinc',
+        PANTS_CACHE_COMPILE_RSC_IGNORE='True'):
+        f(self, *args, **kwargs)
 
-class RscCompileIntegration(BaseCompileIT, AbstractTestGenerator):
+  return wrapper
 
-  @classmethod
-  def generate_tests(cls):
-    tests_with_generated_config = {
-      name: func for name, func in cls.__dict__.items() if getattr(func, '_with_run_config', False)
-    }
 
-    for worker_count in [1, 2]:
-      for resolver in JvmResolveSubsystem.CHOICES:
-        for execution_strategy in RscCompile.ExecutionStrategy.all_values():
-          with temporary_dir() as cache_dir:
-            config = {
-              'cache.compile.rsc': {'write_to': [cache_dir]},
-              'jvm-platform': {'compiler': 'rsc'},
-              'compile.rsc': {
-                'workflow': 'rsc-and-zinc',
-                'execution_strategy': execution_strategy.value,
-                'worker_count': worker_count,
-              },
-              'resolver': {
-                'resolver': resolver,
-              }
-            }
+class RscCompileIntegration(BaseCompileIT):
 
-            def populate_necessary_hermetic_options():
-              config['compile.rsc'].update({
-                'incremental': False,
-                'use_classpath_jars': False,
-              })
-
-            execution_strategy.match({
-              RscCompile.ExecutionStrategy.nailgun: lambda: None,
-              RscCompile.ExecutionStrategy.subprocess: lambda: None,
-              RscCompile.ExecutionStrategy.hermetic: populate_necessary_hermetic_options,
-            })()
-
-            for name, test in tests_with_generated_config.items():
-              cls.add_test(
-                'test_{}_resolver_{}_strategy_{}_worker_{}'
-                .format(name, resolver, execution_strategy.value, worker_count),
-                lambda this: test(this, config=config.copy()))
-
-  @_for_all_supported_execution_environments
-  def basic_binary(self, config):
-    with self.do_command_yielding_workdir(
-        'compile', 'testprojects/src/scala/org/pantsbuild/testproject/mutual:bin',
-        config=config) as pants_run:
+  @ensure_compile_rsc_execution_strategy
+  def test_basic_binary(self):
+    with self.do_command_yielding_workdir('compile', 'testprojects/src/scala/org/pantsbuild/testproject/mutual:bin') as pants_run:
       zinc_compiled_classfile = os.path.join(
         pants_run.workdir,
         'compile/rsc/current/testprojects.src.scala.org.pantsbuild.testproject.mutual.mutual/current/zinc',
@@ -74,24 +39,37 @@ def basic_binary(self, config):
         'm.jar')
       self.assert_is_file(rsc_header_jar)
 
-  @_for_all_supported_execution_environments
-  def executing_multi_target_binary(self, config):
-    pants_run = self.do_command(
-      'run', 'examples/src/scala/org/pantsbuild/example/hello/exe',
-      config=config)
+  @ensure_compile_rsc_execution_strategy
+  def test_executing_multi_target_binary(self):
+    pants_run = self.do_command('run', 'examples/src/scala/org/pantsbuild/example/hello/exe')
     self.assertIn('Hello, Resource World!', pants_run.stdout_data)
 
-  @_for_all_supported_execution_environments
-  def java_with_transitive_exported_scala_dep(self, config):
-    self.do_command(
-      'compile', 'testprojects/src/scala/org/pantsbuild/testproject/javadepsonscalatransitive:java-in-different-package',
-      config=config)
-
-  @_for_all_supported_execution_environments
-  def java_sources(self, config):
-    self.do_command(
-      'compile', 'testprojects/src/scala/org/pantsbuild/testproject/javasources',
-      config=config)
-
-
-RscCompileIntegration.generate_tests()
+  @ensure_compile_rsc_execution_strategy
+  def test_java_with_transitive_exported_scala_dep(self):
+    self.do_command('compile', 'testprojects/src/scala/org/pantsbuild/testproject/javadepsonscalatransitive:java-in-different-package')
+
+  @ensure_compile_rsc_execution_strategy
+  def test_java_sources(self):
+    self.do_command('compile', 'testprojects/src/scala/org/pantsbuild/testproject/javasources')
+
+  def test_rsc_hermetic_jvm_options(self):
+    pants_run = self.run_pants(['compile', 'examples/src/scala/org/pantsbuild/example/hello/exe'],
+      config={
+        'cache.compile.rsc': {'ignore': True},
+        'jvm-platform': {'compiler': 'rsc'},
+        'compile.rsc': {
+          'workflow': 'rsc-and-zinc',
+          'execution_strategy': 'hermetic',
+        },
+        'rsc': {
+          'jvm_options': [
+            '-Djava.security.manager=java.util.Optional'
+          ]
+        }
+      })
+    self.assert_failure(pants_run)
+    self.assertIn(
+      'Could not create SecurityManager: java.util.Optional',
+      pants_run.stdout_data,
+      'Pants run is expected to fail and contain error about loading an invalid security '
+      'manager class, but it did not.')