Repository: bazelbuild/bazel
Issue: Bazel CI: Android build is failing on Windows
Labels: type: bug
P1
breakage
team-Android

https://buildkite.com/bazel/bazel-at-head-plus-downstream/builds/1128
```
(02:09:04) ERROR: D:/b/zqacmrtc/external/maven/BUILD:359:1: Couldn't build file external/maven/androidx_lifecycle_lifecycle_livedata_2_0_0_files/androidx_lifecycle_lifecycle_livedata_2_0_0_resources_aapt2-src.jar: Linking static android resource library for @maven//:androidx_lifecycle_lifecycle_livedata_2_0_0 failed (Exit 1)
--
  | Aug 07, 2019 2:09:04 AM com.google.devtools.build.android.ValidateAndLinkResourcesAction main
  | SEVERE: Error while validating and linking resources
  | com.google.devtools.build.android.aapt2.ResourceLinker$LinkError: java.nio.file.NoSuchFileException: bazel-out\android-armeabi-v7a-fastbuild\bin\external\maven\androidx_arch_core_core_runtime_2_0_0_symbols\symbols.zip;bazel-out\android-armeabi-v7a-fastbuild\bin\external\maven\androidx_lifecycle_lifecycle_livedata_core_2_0_0_symbols\symbols.zip
```

A bisect shows the culprit is https://github.com/bazelbuild/bazel/commit/b98ad776b4d79f41e59631ddf7e1e135f8566df5

/cc @jin 

Issue comments:
It looks like this is being treated as a single filename?

bazel-out\android-armeabi-v7a-fastbuild\bin\external\maven\androidx_arch_core_core_runtime_2_0_0_symbols\symbols.zip;bazel-out\android-armeabi-v7a-fastbuild\bin\external\maven\androidx_lifecycle_lifecycle_livedata_core_2_0_0_symbols\symbols.zip
I can reproduce this on Windows. Looking.

```
PS C:\Users\jingwen\Code\rules_jvm_external\examples\android_instrumentation_test> bazel build //src/test:greeter_test_app --an
droid_aapt=aapt2
INFO: Analyzed target //src/test:greeter_test_app (0 packages loaded, 0 targets configured).
INFO: Found 1 target...
INFO: From Linking external/bazel_tools/src/main/native/windows/windows_jni.dll [for host]:
   Creating library bazel-out/host/bin/external/bazel_tools/src/main/native/windows/windows_jni.dll.if.lib and object bazel-out/host/bin/external/bazel_tools/src/main/native/windows/windows_jni.dll.if.exp
ERROR: C:/users/jingwen/_bazel_jingwen/w7ee76y7/external/maven/BUILD:31:1: Linking static android resource library for @maven//:androidx_test_espresso_espresso_core failed (Exit 1)
Aug 07, 2019 12:24:23 PM com.google.devtools.build.android.ValidateAndLinkResourcesAction main
SEVERE: Error while validating and linking resources
com.google.devtools.build.android.aapt2.ResourceLinker$LinkError: java.nio.file.NoSuchFileException: bazel-out\android-armeabi-v7a-fastbuild\bin\external\maven\androidx_test_espresso_espresso_idling_resource_symbols\symbols.zip;bazel-out\android-armeabi-v7a-fastbuild\bin\external\maven\androidx_test_runner_symbols\symbols.zip;bazel-out\android-armeabi-v7a-fastbuild\bin\external\maven\androidx_test_monitor_symbols\symbols.zip
        at com.google.devtools.build.android.aapt2.ResourceLinker$LinkError.of(ResourceLinker.java:112)
        at com.google.devtools.build.android.aapt2.ResourceLinker.lambda$rethrowLinkError$8(ResourceLinker.java:338)
        at java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195)
        at java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195)
        at java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195)
        at java.base/java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1654)
        at java.base/java.util.stream.Streams$ConcatSpliterator.forEachRemaining(Streams.java:734)
        at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484)
        at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474)
        at java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:913)
        at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
        at java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:578)
        at com.google.devtools.build.android.aapt2.ResourceLinker.compiledResourcesToPaths(ResourceLinker.java:301)
        at com.google.devtools.build.android.aapt2.ResourceLinker.linkStatically(ResourceLinker.java:244)
        at com.google.devtools.build.android.ValidateAndLinkResourcesAction.main(ValidateAndLinkResourcesAction.java:195)
        at com.google.devtools.build.android.ResourceProcessorBusyBox$Tool$13.call(ResourceProcessorBusyBox.java:138)
        at com.google.devtools.build.android.ResourceProcessorBusyBox.processRequest(ResourceProcessorBusyBox.java:239)
        at com.google.devtools.build.android.ResourceProcessorBusyBox.main(ResourceProcessorBusyBox.java:203)
Caused by: java.nio.file.NoSuchFileException: bazel-out\android-armeabi-v7a-fastbuild\bin\external\maven\androidx_test_espresso_espresso_idling_resource_symbols\symbols.zip;bazel-out\android-armeabi-v7a-fastbuild\bin\external\maven\androidx_test_runner_symbols\symbols.zip;bazel-out\android-armeabi-v7a-fastbuild\bin\external\maven\androidx_test_monitor_symbols\symbols.zip
        at java.base/sun.nio.fs.WindowsException.translateToIOException(WindowsException.java:85)
        at java.base/sun.nio.fs.WindowsException.rethrowAsIOException(WindowsException.java:103)
        at java.base/sun.nio.fs.WindowsException.rethrowAsIOException(WindowsException.java:108)
        at java.base/sun.nio.fs.WindowsFileSystemProvider.newFileChannel(WindowsFileSystemProvider.java:116)
        at java.base/java.nio.channels.FileChannel.open(FileChannel.java:292)
        at java.base/java.nio.channels.FileChannel.open(FileChannel.java:345)
        at com.google.devtools.build.android.aapt2.ResourceLinker.filterZip(ResourceLinker.java:315)
        at com.google.devtools.build.android.aapt2.ResourceLinker.lambda$compiledResourcesToPaths$5(ResourceLinker.java:297)
        at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125)
        at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:57)
        at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base/java.lang.Thread.run(Thread.java:834)

Exception in thread "main" com.google.devtools.build.android.aapt2.ResourceLinker$LinkError: java.nio.file.NoSuchFileException: bazel-out\android-armeabi-v7a-fastbuild\bin\external\maven\androidx_test_espresso_espresso_idling_resource_symbols\symbols.zip;bazel-out\android-armeabi-v7a-fastbuild\bin\external\maven\androidx_test_runner_symbols\symbols.zip;bazel-out\android-armeabi-v7a-fastbuild\bin\external\maven\androidx_test_monitor_symbols\symbols.zip
        at com.google.devtools.build.android.aapt2.ResourceLinker$LinkError.of(ResourceLinker.java:112)
        at com.google.devtools.build.android.aapt2.ResourceLinker.lambda$rethrowLinkError$8(ResourceLinker.java:338)
        at java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195)
        at java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195)
        at java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195)
        at java.base/java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1654)
        at java.base/java.util.stream.Streams$ConcatSpliterator.forEachRemaining(Streams.java:734)
        at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484)
        at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474)
        at java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:913)
        at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
        at java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:578)
        at com.google.devtools.build.android.aapt2.ResourceLinker.compiledResourcesToPaths(ResourceLinker.java:301)
        at com.google.devtools.build.android.aapt2.ResourceLinker.linkStatically(ResourceLinker.java:244)
        at com.google.devtools.build.android.ValidateAndLinkResourcesAction.main(ValidateAndLinkResourcesAction.java:195)
        at com.google.devtools.build.android.ResourceProcessorBusyBox$Tool$13.call(ResourceProcessorBusyBox.java:138)
        at com.google.devtools.build.android.ResourceProcessorBusyBox.processRequest(ResourceProcessorBusyBox.java:239)
        at com.google.devtools.build.android.ResourceProcessorBusyBox.main(ResourceProcessorBusyBox.java:203)
Caused by: java.nio.file.NoSuchFileException: bazel-out\android-armeabi-v7a-fastbuild\bin\external\maven\androidx_test_espresso_espresso_idling_resource_symbols\symbols.zip;bazel-out\android-armeabi-v7a-fastbuild\bin\external\maven\androidx_test_runner_symbols\symbols.zip;bazel-out\android-armeabi-v7a-fastbuild\bin\external\maven\androidx_test_monitor_symbols\symbols.zip
        at java.base/sun.nio.fs.WindowsException.translateToIOException(WindowsException.java:85)
        at java.base/sun.nio.fs.WindowsException.rethrowAsIOException(WindowsException.java:103)
        at java.base/sun.nio.fs.WindowsException.rethrowAsIOException(WindowsException.java:108)
        at java.base/sun.nio.fs.WindowsFileSystemProvider.newFileChannel(WindowsFileSystemProvider.java:116)
        at java.base/java.nio.channels.FileChannel.open(FileChannel.java:292)
        at java.base/java.nio.channels.FileChannel.open(FileChannel.java:345)
        at com.google.devtools.build.android.aapt2.ResourceLinker.filterZip(ResourceLinker.java:315)
        at com.google.devtools.build.android.aapt2.ResourceLinker.lambda$compiledResourcesToPaths$5(ResourceLinker.java:297)
        at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125)
        at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:57)
        at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base/java.lang.Thread.run(Thread.java:834)
Target //src/test:greeter_test_app failed to build
Use --verbose_failures to see the command lines of failed build steps.
INFO: Elapsed time: 11.788s, Critical Path: 9.73s
INFO: 72 processes: 43 local, 29 worker.
FAILED: Build did NOT complete successfully
```
Figured out the root cause. The params file contains:

```
--compiledDep
'bazel-out/android-armeabi-v7a-fastbuild/bin/external/maven/androidx_test_monitor_symbols/symbols.zip;bazel-out/android-armeabi-v7a-fastbuild/bin/external/maven/androidx_test_core_symbols/symbols.zip'
```

Note the use of `;` as a delimiter. `--compiledDep` is a list of paths converted with `PathListConverter`

https://github.com/bazelbuild/bazel/blob/13dc4f7d10b4f70a2c2ff9fe068e9c2e73f20dbc/src/tools/android/java/com/google/devtools/build/android/ValidateAndLinkResourcesAction.java#L55-L64

which is delimited with `:` instead `;` 

https://github.com/bazelbuild/bazel/blob/a6e3e875c55b1a6d101c79ee5d171a96edb0caa6/src/tools/android/java/com/google/devtools/build/android/Converters.java#L302

But the `compiledDeps` execution paths were joined with `getHostPathSeparator()`, which is `;` for Windows only. 

https://github.com/bazelbuild/bazel/blob/d0bd3c8273e2a782d7e9b05a8b1fbab368589c9a/src/main/java/com/google/devtools/build/lib/analysis/config/OutputDirectories.java#L279
Now I'm running into another error:

```
ERROR: C:/users/jingwen/code/rules_jvm_external/examples/android_instrumentation_test/src/main/BUILD:1:1: Processing Android resources for //src/main:greeter_app failed (Exit 1)
Aug 07, 2019 4:24:18 PM com.google.devtools.build.android.ResourceProcessorBusyBox processRequest
SEVERE: Error during processing
java.nio.file.InvalidPathException: Illegal character [:] in path at index 4: ///C:/Users/jingwen/AppData/Local/Temp/android_resources_tmp14123356876479215233/linked/bin.-pb.apk
        at java.base/sun.nio.fs.WindowsPathParser.nextSlash(WindowsPathParser.java:212)
        at java.base/sun.nio.fs.WindowsPathParser.parse(WindowsPathParser.java:111)
        at java.base/sun.nio.fs.WindowsPathParser.parse(WindowsPathParser.java:77)
        at java.base/sun.nio.fs.WindowsPath.parse(WindowsPath.java:92)
        at java.base/sun.nio.fs.WindowsFileSystem.getPath(WindowsFileSystem.java:229)
        at java.base/java.nio.file.Path.of(Path.java:147)
        at java.base/java.nio.file.Paths.get(Paths.java:69)
        at com.google.devtools.build.android.aapt2.ProtoApk.asApkPath(ProtoApk.java:229)
        at com.google.devtools.build.android.aapt2.ResourceLinker.link(ResourceLinker.java:565)
        at com.google.devtools.build.android.aapt2.ResourceLinker.link(ResourceLinker.java:546)
        at com.google.devtools.build.android.Aapt2ResourcePackagingAction.main(Aapt2ResourcePackagingAction.java:190)
        at com.google.devtools.build.android.ResourceProcessorBusyBox$Tool$14.call(ResourceProcessorBusyBox.java:144)
        at com.google.devtools.build.android.ResourceProcessorBusyBox.processRequest(ResourceProcessorBusyBox.java:242)
        at com.google.devtools.build.android.ResourceProcessorBusyBox.main(ResourceProcessorBusyBox.java:203)

Exception in thread "main" java.nio.file.InvalidPathException: Illegal character [:] in path at index 4: ///C:/Users/jingwen/AppData/Local/Temp/android_resources_tmp14123356876479215233/linked/bin.-pb.apk
        at java.base/sun.nio.fs.WindowsPathParser.nextSlash(WindowsPathParser.java:212)
        at java.base/sun.nio.fs.WindowsPathParser.parse(WindowsPathParser.java:111)
        at java.base/sun.nio.fs.WindowsPathParser.parse(WindowsPathParser.java:77)
        at java.base/sun.nio.fs.WindowsPath.parse(WindowsPath.java:92)
        at java.base/sun.nio.fs.WindowsFileSystem.getPath(WindowsFileSystem.java:229)
        at java.base/java.nio.file.Path.of(Path.java:147)
        at java.base/java.nio.file.Paths.get(Paths.java:69)
        at com.google.devtools.build.android.aapt2.ProtoApk.asApkPath(ProtoApk.java:229)
        at com.google.devtools.build.android.aapt2.ResourceLinker.link(ResourceLinker.java:565)
        at com.google.devtools.build.android.aapt2.ResourceLinker.link(ResourceLinker.java:546)
        at com.google.devtools.build.android.Aapt2ResourcePackagingAction.main(Aapt2ResourcePackagingAction.java:190)
        at com.google.devtools.build.android.ResourceProcessorBusyBox$Tool$14.call(ResourceProcessorBusyBox.java:144)
        at com.google.devtools.build.android.ResourceProcessorBusyBox.processRequest(ResourceProcessorBusyBox.java:242)
        at com.google.devtools.build.android.ResourceProcessorBusyBox.main(ResourceProcessorBusyBox.java:203)
Target //src/test:greeter_test_app failed to build
Use --verbose_failures to see the command lines of failed build steps.
```

Pull request review comments:

Patch that resolved the issue:
diff --git a/src/main/java/com/google/devtools/build/lib/rules/android/BusyBoxActionBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/android/BusyBoxActionBuilder.java
index eb61855b672152..bbb6579df4b34a 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/android/BusyBoxActionBuilder.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/android/BusyBoxActionBuilder.java
@@ -223,21 +223,15 @@ public <T> BusyBoxActionBuilder addTransitiveFlagForEach(
   /**
    * Adds an efficient flag and inputs based on transitive values.
    *
-   * <p>Each value will be separated on the command line by the host-specific path separator.
+   * <p>Each value will be separated on the command line by the ':' character, the option parser's
+   * PathListConverter delimiter.
    *
    * <p>Unlike other transitive input methods in this class, this method adds the values to both the
    * command line and the list of inputs.
    */
   public BusyBoxActionBuilder addTransitiveVectoredInput(
       @CompileTimeConstant String arg, NestedSet<Artifact> values) {
-    commandLine.addExecPaths(
-        arg,
-        VectorArg.join(
-                dataContext
-                    .getActionConstructionContext()
-                    .getConfiguration()
-                    .getHostPathSeparator())
-            .each(values));
+    commandLine.addExecPaths(arg, VectorArg.join(":").each(values));
     inputs.addTransitive(values);
     return this;
   }
diff --git a/src/tools/android/java/com/google/devtools/build/android/aapt2/ProtoApk.java b/src/tools/android/java/com/google/devtools/build/android/aapt2/ProtoApk.java
index 2d2c94d23b6ea1..efc8401adf07ba 100644
--- a/src/tools/android/java/com/google/devtools/build/android/aapt2/ProtoApk.java
+++ b/src/tools/android/java/com/google/devtools/build/android/aapt2/ProtoApk.java
@@ -226,7 +226,17 @@ public Path writeManifestAsXmlTo(Path directory) {
 
   /** The apk as path. */
   public Path asApkPath() {
-    return Paths.get(uri.toString().substring("jar:".length() + 1));
+    String pathString = uri.toString().substring("jar:".length() + 1);
+
+    // On Windows, the fully qualified URI looks like:
+    // jar:////C:/Users/user/AppData/Local/Temp/android_resources_tmp12318742536955859612/linked/bin.-pb.apk
+    //
+    // We want "C:/Users...".
+    if (pathString.startsWith("///")) {
+      pathString = pathString.substring("///".length());
+    }
+
+    return Paths.get(pathString);
   }
 
   /** Thrown when errors occur during proto apk processing. */


Test patch: