Repository: pantsbuild/pants

Issue:
Bump Rsc version to 0.0.0-386-d79bf764 and Scalameta to 4.0.0-RC1
The newer versions of Rsc and Scalameta have bug fixes and additional features that'll help take care of some of the todos in #6408


Discussion:
The most recent versions are now: Rsc 0.0.0-388-2169dcf0 and Scalameta 4.0.0.
Thanks!
And another update of the Rsc version number. As discussed internally today, 0.0.0-406-6829ec44 can handle rewritten OSS twitter/util with a few caveats.
The recently published 0.0.0-432-20928bbd is a big update in stability and features:
  * The toolchain now handles OSS twitter/util without caveats (although Java outlining is still too immature for twitter/util).
  * Thanks to the stability updates, the toolchain now fully supports two-phase compilation.
  * Rsc now supports `-d` that works similarly to `-d` in Scala - if you pass a directory, Rsc will dump its results into that directory, if you pass a jar file, Rsc will dump its results into that jar file.
  * Rsc no longer supports `-out`, since this option has been superseded by `-d`.
  * Rsc now emits SemanticDB payloads exactly like semanticdb-scalac does. Instead of a single .semanticdb file, it produces one .semanticdb file per one source file in META-INF/semanticdb/. Moreover, it also produces META-INF/semanticdb.semanticidx. This means that outputs of Rsc compilations can now serve as inputs for other Rsc compilations.
  * Rsc now includes functionality that was previously provided by Mjar. Concretely, alongside the SemanticDB payloads it will now emit .class files with ScalaSignature annotations. This means that outputs of Rsc compilations can now serve as inputs for Scalac compilations.
Rsc 0.0.0-446-c64e6937 implements Java outlining to the degree that's sufficient to compile twitter/util.

Resolving patch:
diff --git a/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py b/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py
--- a/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py
+++ b/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py
@@ -62,15 +62,15 @@ def _create_desandboxify_fn(possible_path_patterns):
   # if it finds a matching prefix, strips the path prior to the prefix and returns it
   # if it doesn't it returns the original path
   # TODO remove this after https://github.com/scalameta/scalameta/issues/1791 is released
-  regexes = [re.compile(p) for p in possible_path_patterns]
+  regexes = [re.compile('/({})'.format(p)) for p in possible_path_patterns]
   def desandboxify(path):
     if not path:
       return path
     for r in regexes:
       match = r.search(path)
-      print('>>> matched {} with {} against {}'.format(match, r.pattern, path))
+      logger.debug('>>> matched {} with {} against {}'.format(match, r.pattern, path))
       if match:
-        return match.group(0)
+        return match.group(1)
     return path
   return desandboxify
 
@@ -85,18 +85,15 @@ def __init__(self,
                log_dir,
                zinc_args_file,
                sources,
-               rsc_index_dir,
-               rsc_outline_dir):
+               rsc_index_dir):
     super(RscCompileContext, self).__init__(target, analysis_file, classes_dir, jar_file,
                                                log_dir, zinc_args_file, sources)
     self.rsc_mjar_file = rsc_mjar_file
     self.rsc_index_dir = rsc_index_dir
-    self.rsc_outline_dir = rsc_outline_dir
 
   def ensure_output_dirs_exist(self):
     safe_mkdir(os.path.dirname(self.rsc_mjar_file))
     safe_mkdir(self.rsc_index_dir)
-    safe_mkdir(self.rsc_outline_dir)
 
 
 class RscCompile(ZincCompile):
@@ -117,8 +114,8 @@ def implementation_version(cls):
   def register_options(cls, register):
     super(RscCompile, cls).register_options(register)
 
-    rsc_toolchain_version = '0.0.0-294-d7114447'
-    scalameta_toolchain_version = '4.0.0-M10'
+    rsc_toolchain_version = '0.0.0-446-c64e6937'
+    scalameta_toolchain_version = '4.0.0'
 
     # TODO: it would be better to have a less adhoc approach to handling
     #       optional dependencies. See: https://github.com/pantsbuild/pants/issues/6390
@@ -156,19 +153,6 @@ def register_options(cls, register):
       custom_rules=[
         Shader.exclude_package('rsc', recursive=True),
       ])
-    cls.register_jvm_tool(
-      register,
-      'mjar',
-      classpath=[
-          JarDependency(
-              org='com.twitter',
-              name='mjar_2.11',
-              rev=rsc_toolchain_version,
-          ),
-      ],
-      custom_rules=[
-        Shader.exclude_package('scala', recursive=True),
-      ])
     cls.register_jvm_tool(
       register,
       'metacp',
@@ -431,15 +415,14 @@ def work_for_vts_rsc(vts, ctx):
             # NB: there are no unmetacp'd dependencies
             metai_classpath = []
 
+          rsc_mjar_file = fast_relpath(ctx.rsc_mjar_file, get_buildroot())
+
           # Step 2: Outline Scala sources into SemanticDB
           # ---------------------------------------------
-          rsc_outline_dir = fast_relpath(ctx.rsc_outline_dir, get_buildroot())
-          rsc_out = os.path.join(rsc_outline_dir, 'META-INF/semanticdb/out.semanticdb')
-          safe_mkdir(os.path.join(rsc_outline_dir, 'META-INF/semanticdb'))
           target_sources = ctx.sources
           args = [
             '-cp', os.pathsep.join(metai_classpath + metacp_jar_classpath_rel),
-            '-out', rsc_out,
+            '-d', rsc_mjar_file,
           ] + target_sources
           self._runtool(
             'rsc.cli.Main',
@@ -450,41 +433,7 @@ def work_for_vts_rsc(vts, ctx):
             # TODO pass the input files from the target snapshot instead of the below
             # input_snapshot = ctx.target.sources_snapshot(scheduler=self.context._scheduler)
             input_files=target_sources + metai_classpath + metacp_jar_classpath_rel,
-            output_dir=rsc_outline_dir)
-          rsc_classpath = [rsc_outline_dir]
-
-          # Step 2.5: Postprocess the rsc outputs
-          # TODO: This is only necessary as a workaround for https://github.com/twitter/rsc/issues/199.
-          # Ideally, Rsc would do this on its own.
-          self._run_metai_tool(distribution,
-            rsc_classpath,
-            rsc_outline_dir,
-            tgt,
-            extra_input_files=(rsc_out,))
-
-
-          # Step 3: Convert SemanticDB into an mjar
-          # ---------------------------------------
-          rsc_mjar_file = fast_relpath(ctx.rsc_mjar_file, get_buildroot())
-          args = [
-            '-out', rsc_mjar_file,
-            os.pathsep.join(rsc_classpath),
-          ]
-          self._runtool(
-            'scala.meta.cli.Mjar',
-            'mjar',
-            args,
-            distribution,
-            tgt=tgt,
-            input_files=(
-              rsc_out,
-            ),
-            output_dir=os.path.dirname(rsc_mjar_file)
-            )
-          self.context.products.get_data('rsc_classpath').add_for_target(
-            ctx.target,
-            [(conf, ctx.rsc_mjar_file) for conf in self._confs],
-          )
+            output_dir=os.path.dirname(rsc_mjar_file))
 
         self._record_target_stats(tgt,
                                   len(cp_entries),
@@ -738,7 +687,6 @@ def create_compile_context(self, target, target_workdir):
         log_dir=os.path.join(rsc_dir, 'logs'),
         sources=sources,
         rsc_index_dir=os.path.join(rsc_dir, 'index'),
-        rsc_outline_dir=os.path.join(rsc_dir, 'outline'),
       ),
       CompileContext(
         target=target,
@@ -860,9 +808,11 @@ def _collect_metai_classpath(self, metacp_result, relative_input_paths):
     #    TODO remove this after https://github.com/scalameta/scalameta/issues/1791 is released
     desandboxify = _create_desandboxify_fn(
       [
+        os.path.join(relative_workdir, 'resolve', 'coursier', '[^/]*', 'cache', '.*'),
         os.path.join(relative_workdir, 'resolve', 'ivy', '[^/]*', 'ivy', 'jars', '.*'),
         os.path.join(relative_workdir, 'compile', 'rsc', '.*'),
         os.path.join(relative_workdir, '\.jdk', '.*'),
+        os.path.join('\.jdk', '.*'),
       ]
       )
 


Test patch:
diff --git a/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile.py b/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile.py
--- a/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile.py
+++ b/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile.py
@@ -122,6 +122,8 @@ def test_desandbox_fn(self):
       '/some/path/.pants.d/exec-location/.pants.d/c/r/c/beans'),
       '.pants.d/c/r/c/beans')
     self.assertEqual(desandbox('/some/path/outside/workdir'), '/some/path/outside/workdir')
+    # NB ensure that a path outside the workdir that partially matches won't be truncated
+    self.assertEqual(desandbox('/some/path/outside/workdir.pants.d/cool/beans/etc'), '/some/path/outside/workdir.pants.d/cool/beans/etc')
     self.assertEqual(desandbox(None), None)
     # ensure that temp workdirs are discovered relative to the buildroot
     desandbox = _create_desandboxify_fn(['.pants.d/tmp.pants.d/cool/beans', '.pants.d/tmp.pants.d/c/r/c'])
diff --git a/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile_integration.py b/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile_integration.py
--- a/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile_integration.py
+++ b/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile_integration.py
@@ -52,7 +52,7 @@ def test_basic_binary_hermetic(self):
         path = os.path.join(
           workdir,
           'compile/rsc/current/testprojects.src.scala.org.pantsbuild.testproject.mutual.mutual/current/rsc',
-          'outline/META-INF/semanticdb/out.semanticdb')
+          'm.jar')
         self.assertTrue(os.path.exists(path))
         path = os.path.join(
           workdir,