Repository: bazelbuild/bazel
Issue: support run command for experimental_remote_download_outputs
Labels: type: feature request
team-Remote-Exec

### Description of the problem / feature request:

experimental_remote_download_outputs does not properly download all runtime dependencies.

### Feature requests: what underlying problem are you trying to solve with this feature?
Attempting to use the new remote builds without the bytes (#6862)


### Bugs: what's the simplest, easiest way to reproduce this bug? Please provide a minimal example if possible.
https://github.com/phb/bazel_minimal_missingrunfiles
and specifically see 
https://github.com/phb/bazel_minimal_missingrunfiles/blob/master/EXAMPLE_OUTPUT

### What operating system are you running Bazel on?

macosx.

### What's the output of `bazel info release`?

Tried various versions including latest HEAD as of today.

This was initially reported in #6862 and split into it's own ticket. CC @buchgr 

Issue comments:
Thanks for filing the issue. I see what you mean now. You are indeed correct the run command is not supported (yet).
Hi @buchgr,

I just wanted to give a quick ping on this issue. Can you think of any workaround? We have a deploy target that we generally run from our CI, but we are unable to run it without turning off minimal downloads and trigger a full rebuild.


I wrote a patch that should make it work. Can you give it a try please: https://github.com/bazelbuild/bazel/pull/9298?

Pull request review comments:

Patch that resolved the issue:
diff --git a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java
index b03b2c136d2374..7f3167c55e72fc 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java
@@ -235,7 +235,8 @@ public final class RemoteOptions extends OptionsBase {
   public boolean allowSymlinkUpload;
 
   @Option(
-      name = "experimental_remote_download_outputs",
+      name = "remote_download_outputs",
+      oldName = "experimental_remote_download_outputs",
       defaultValue = "all",
       category = "remote",
       documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS,
@@ -257,7 +258,8 @@ public RemoteOutputsStrategyConverter() {
   }
 
   @Option(
-      name = "experimental_remote_download_minimal",
+      name = "remote_download_minimal",
+      oldName = "experimental_remote_download_minimal",
       defaultValue = "null",
       expansion = {
         "--experimental_inmemory_jdeps_files",
@@ -275,7 +277,8 @@ public RemoteOutputsStrategyConverter() {
   public Void remoteOutputsMinimal;
 
   @Option(
-      name = "experimental_remote_download_toplevel",
+      name = "remote_download_toplevel",
+      oldName = "experimental_remote_download_toplevel",
       defaultValue = "null",
       expansion = {
         "--experimental_inmemory_jdeps_files",


Test patch:
diff --git a/src/test/shell/bazel/remote/remote_execution_test.sh b/src/test/shell/bazel/remote/remote_execution_test.sh
index 2f0e0021f01530..7c781193b2d69e 100755
--- a/src/test/shell/bazel/remote/remote_execution_test.sh
+++ b/src/test/shell/bazel/remote/remote_execution_test.sh
@@ -805,7 +805,7 @@ EOF
 
 function test_downloads_minimal() {
   # Test that genrule outputs are not downloaded when using
-  # --experimental_remote_download_outputs=minimal
+  # --remote_download_minimal
   mkdir -p a
   cat > a/BUILD <<'EOF'
 genrule(
@@ -826,7 +826,7 @@ EOF
   bazel build \
     --genrule_strategy=remote \
     --remote_executor=grpc://localhost:${worker_port} \
-    --experimental_remote_download_minimal \
+    --remote_download_minimal \
     //a:foobar || fail "Failed to build //a:foobar"
 
   (! [[ -f bazel-bin/a/foo.txt ]] && ! [[ -f bazel-bin/a/foobar.txt ]]) \
@@ -835,7 +835,7 @@ EOF
 
 function test_downloads_minimal_failure() {
   # Test that outputs of failing actions are downloaded when using
-  # --experimental_remote_download_outputs=minimal
+  # --remote_download_minimal
   mkdir -p a
   cat > a/BUILD <<'EOF'
 genrule(
@@ -849,7 +849,7 @@ EOF
   bazel build \
     --spawn_strategy=remote \
     --remote_executor=grpc://localhost:${worker_port} \
-    --experimental_remote_download_minimal \
+    --remote_download_minimal \
     //a:fail && fail "Expected test failure" || true
 
   [[ -f bazel-bin/a/fail.txt ]] \
@@ -857,7 +857,7 @@ EOF
 }
 
 function test_downloads_minimal_prefetch() {
-  # Test that when using --experimental_remote_download_outputs=minimal a remote-only output that's
+  # Test that when using --remote_download_minimal a remote-only output that's
   # an input to a local action is downloaded lazily before executing the local action.
   mkdir -p a
   cat > a/BUILD <<'EOF'
@@ -880,7 +880,7 @@ EOF
   bazel build \
     --genrule_strategy=remote \
     --remote_executor=grpc://localhost:${worker_port} \
-    --experimental_remote_download_minimal \
+    --remote_download_minimal \
     //a:remote || fail "Failed to build //a:remote"
 
   (! [[ -f bazel-bin/a/remote.txt ]]) \
@@ -889,7 +889,7 @@ EOF
   bazel build \
     --genrule_strategy=remote,local \
     --remote_executor=grpc://localhost:${worker_port} \
-    --experimental_remote_download_minimal \
+    --remote_download_minimal \
     //a:local || fail "Failed to build //a:local"
 
   localtxt="bazel-bin/a/local.txt"
@@ -901,7 +901,7 @@ EOF
 }
 
 function test_download_outputs_invalidation() {
-  # Test that when changing values of --experimental_remote_download_outputs all actions are
+  # Test that when changing values of --remote_download_minimal all actions are
   # invalidated.
   mkdir -p a
   cat > a/BUILD <<'EOF'
@@ -916,7 +916,7 @@ EOF
   bazel build \
     --genrule_strategy=remote \
     --remote_executor=grpc://localhost:${worker_port} \
-    --experimental_remote_download_minimal \
+    --remote_download_minimal \
     //a:remote >& $TEST_log || fail "Failed to build //a:remote"
 
   expect_log "1 process: 1 remote"
@@ -924,16 +924,16 @@ EOF
   bazel build \
     --genrule_strategy=remote \
     --remote_executor=grpc://localhost:${worker_port} \
-    --experimental_remote_download_outputs=all \
+    --remote_download_outputs=all \
     //a:remote >& $TEST_log || fail "Failed to build //a:remote"
 
-  # Changing --experimental_remote_download_outputs to "all" should invalidate SkyFrames in-memory
+  # Changing --remote_download_outputs to "all" should invalidate SkyFrames in-memory
   # caching and make it re-run the action.
   expect_log "1 process: 1 remote"
 }
 
 function test_downloads_minimal_native_prefetch() {
-  # Test that when using --experimental_remote_download_outputs=minimal a remotely stored output
+  # Test that when using --remote_download_outputs=minimal a remotely stored output
   # that's an input to a native action (ctx.actions.expand_template) is staged lazily for action
   # execution.
   mkdir -p a
@@ -979,7 +979,7 @@ EOF
   bazel build \
     --genrule_strategy=remote \
     --remote_executor=grpc://localhost:${worker_port} \
-    --experimental_remote_download_minimal \
+    --remote_download_minimal \
     //a:substitute-buchgr >& $TEST_log || fail "Failed to build //a:substitute-buchgr"
 
   # The genrule //a:generate-template should run remotely and //a:substitute-buchgr
@@ -995,7 +995,7 @@ EOF
 }
 
 function test_downloads_toplevel() {
-  # Test that when using --experimental_remote_download_outputs=toplevel only the output of the
+  # Test that when using --remote_download_outputs=toplevel only the output of the
   # toplevel target is being downloaded.
   mkdir -p a
   cat > a/BUILD <<'EOF'
@@ -1017,7 +1017,7 @@ EOF
   bazel build \
     --genrule_strategy=remote \
     --remote_executor=grpc://localhost:${worker_port} \
-    --experimental_remote_download_toplevel \
+    --remote_download_toplevel \
     //a:foobar || fail "Failed to build //a:foobar"
 
   (! [[ -f bazel-bin/a/foo.txt ]]) \
@@ -1033,7 +1033,7 @@ EOF
   bazel build \
     --genrule_strategy=remote \
     --remote_executor=grpc://localhost:${worker_port} \
-    --experimental_remote_download_toplevel \
+    --remote_download_toplevel \
     //a:foobar >& $TEST_log || fail "Failed to build //a:foobar"
 
   expect_log "1 process: 1 remote cache hit"
@@ -1043,7 +1043,7 @@ EOF
 }
 
 function test_download_toplevel_test_rule() {
-  # Test that when using --experimental_remote_download_outputs=toplevel with bazel test only
+  # Test that when using --remote_download_toplevel with bazel test only
   # the test.log and test.xml file are downloaded but not the test binary. However when building
   # a test then the test binary should be downloaded.
 
@@ -1070,9 +1070,7 @@ EOF
   # When invoking bazel test only test.log and test.xml should be downloaded.
   bazel test \
     --remote_executor=grpc://localhost:${worker_port} \
-    --experimental_inmemory_jdeps_files \
-    --experimental_inmemory_dotd_files \
-    --experimental_remote_download_outputs=toplevel \
+    --remote_download_toplevel \
     //a:test >& $TEST_log || fail "Failed to test //a:test with remote execution"
 
   (! [[ -f bazel-bin/a/test ]]) \
@@ -1089,9 +1087,7 @@ EOF
   # When invoking bazel build the test binary should be downloaded.
   bazel build \
     --remote_executor=grpc://localhost:${worker_port} \
-    --experimental_inmemory_jdeps_files \
-    --experimental_inmemory_dotd_files \
-    --experimental_remote_download_outputs=toplevel \
+    --remote_download_toplevel \
     //a:test >& $TEST_log || fail "Failed to build //a:test with remote execution"
 
   ([[ -f bazel-bin/a/test ]]) \
@@ -1099,7 +1095,7 @@ EOF
 }
 
 function test_downloads_minimal_bep() {
-  # Test that when using --experimental_remote_download_outputs=minimal all URI's in the BEP
+  # Test that when using --remote_download_minimal all URI's in the BEP
   # are rewritten as bytestream://..
   mkdir -p a
   cat > a/success.sh <<'EOF'
@@ -1123,7 +1119,7 @@ EOF
 
   bazel test \
     --remote_executor=grpc://localhost:${worker_port} \
-    --experimental_remote_download_minimal \
+    --remote_download_minimal \
     --build_event_text_file=$TEST_log \
     //a:foo //a:success_test || fail "Failed to test //a:foo //a:success_test"
 
@@ -1179,7 +1175,7 @@ chmod +x status.sh
 
   bazel build \
     --remote_executor=grpc://localhost:${worker_port} \
-    --experimental_remote_download_minimal \
+    --remote_download_minimal \
     --workspace_status_command=status.sh \
     //a:foo || "Failed to build //a:foo"
 
@@ -1190,7 +1186,7 @@ EOF
 
   bazel build \
     --remote_executor=grpc://localhost:${worker_port} \
-    --experimental_remote_download_minimal \
+    --remote_download_minimal \
     --workspace_status_command=status.sh \
     //a:foo || "Failed to build //a:foo"
 }