Repository: bazelbuild/bazel
Issue: Implement the credential helpers proposal
Labels: type: feature request
P2
team-Remote-Exec

### Description of the feature request:


Tracking issue for implementing https://github.com/bazelbuild/proposals/blob/main/designs/2022-06-07-bazel-credential-helpers.md.

### What underlying problem are you trying to solve with this feature?


N/A

### Which operating system are you running Bazel on?


N/A

### What is the output of `bazel info release`?


N/A

### If `bazel info release` returns `development version` or `(@non-git)`, tell us how you built Bazel.


N/A

### What's the output of `git remote get-url origin; git rev-parse master; git rev-parse HEAD` ?


```text
N/A
```


### Have you found anything relevant by searching the web?

_No response_

### Any other information, logs, or outputs that you want to share?

_No response_

Issue comments:
@bazel-io flag
@bazel-io fork 5.3.0
Since this feature is implemented and cherry picked to 5.3.0, should we close this one?
The proposal hasn't been fully implemented yet (e.g. we're missing support for repository fetching).
Marking this as a release blocker since we want the repository fetching support to make it into 6.0.0 if possible.
@tjgq What's the ETA for implementing this? Ideally, we don't want to delay the 6.0 release further more, and since this should be a backwards compatible feature, we can cherry pick it in 6.x releases?
I was planning to get this submitted in time for 6.0.0, but other things got in the way. Let's do it for a 6.x release instead.
This has been fully implemented in 7.x and backported to 6.x.

Pull request review comments:

Patch that resolved the issue:
diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD
index 745e6eaebae2b7..8b11bce467a160 100644
--- a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD
@@ -14,6 +14,8 @@ java_library(
     name = "credentialhelper",
     srcs = glob(["*.java"]),
     deps = [
+        "//src/main/java/com/google/devtools/build/lib/events",
+        "//src/main/java/com/google/devtools/build/lib/shell",
         "//src/main/java/com/google/devtools/build/lib/vfs",
         "//third_party:auto_value",
         "//third_party:error_prone_annotations",
diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelper.java b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelper.java
index c6b60b1fb1a0b4..c82417b0034383 100644
--- a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelper.java
+++ b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelper.java
@@ -14,14 +14,32 @@
 
 package com.google.devtools.build.lib.authandtls.credentialhelper;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.io.CharStreams;
+import com.google.devtools.build.lib.shell.Subprocess;
+import com.google.devtools.build.lib.shell.SubprocessBuilder;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.errorprone.annotations.Immutable;
+import com.google.gson.Gson;
+import com.google.gson.JsonSyntaxException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Reader;
+import java.io.Writer;
+import java.net.URI;
+import java.util.Locale;
+import java.util.Objects;
 
 /** Wraps an external tool used to obtain credentials. */
 @Immutable
 public final class CredentialHelper {
+  private static final Gson GSON = new Gson();
+
   // `Path` is immutable, but not annotated.
   @SuppressWarnings("Immutable")
   private final Path path;
@@ -35,5 +53,101 @@ Path getPath() {
     return path;
   }
 
-  // TODO(yannic): Implement running the helper subprocess.
+  /**
+   * Fetches credentials for the specified {@link URI} by invoking the credential helper as
+   * subprocess according to the <a
+   * href="https://github.com/bazelbuild/proposals/blob/main/designs/2022-06-07-bazel-credential-helpers.md">credential
+   * helper protocol</a>.
+   *
+   * @param environment The environment to run the subprocess in.
+   * @param uri The {@link URI} to fetch credentials for.
+   * @return The response from the subprocess.
+   */
+  public GetCredentialsResponse getCredentials(CredentialHelperEnvironment environment, URI uri)
+      throws InterruptedException, IOException {
+    Preconditions.checkNotNull(environment);
+    Preconditions.checkNotNull(uri);
+
+    Subprocess process = spawnSubprocess(environment, "get");
+    try (Reader stdout = new InputStreamReader(process.getInputStream(), UTF_8);
+        Reader stderr = new InputStreamReader(process.getErrorStream(), UTF_8)) {
+      try (Writer stdin = new OutputStreamWriter(process.getOutputStream(), UTF_8)) {
+        GSON.toJson(GetCredentialsRequest.newBuilder().setUri(uri).build(), stdin);
+      }
+
+      process.waitFor();
+      if (process.timedout()) {
+        throw new IOException(
+            String.format(
+                Locale.US,
+                "Failed to get credentials for '%s' from helper '%s': process timed out",
+                uri,
+                path));
+      }
+      if (process.exitValue() != 0) {
+        throw new IOException(
+            String.format(
+                Locale.US,
+                "Failed to get credentials for '%s' from helper '%s': process exited with code %d."
+                    + " stderr: %s",
+                uri,
+                path,
+                process.exitValue(),
+                CharStreams.toString(stderr)));
+      }
+
+      try {
+        GetCredentialsResponse response = GSON.fromJson(stdout, GetCredentialsResponse.class);
+        if (response == null) {
+          throw new IOException(
+              String.format(
+                  Locale.US,
+                  "Failed to get credentials for '%s' from helper '%s': process exited without"
+                      + " output. stderr: %s",
+                  uri,
+                  path,
+                  CharStreams.toString(stderr)));
+        }
+        return response;
+      } catch (JsonSyntaxException e) {
+        throw new IOException(
+            String.format(
+                Locale.US,
+                "Failed to get credentials for '%s' from helper '%s': error parsing output. stderr:"
+                    + " %s",
+                uri,
+                path,
+                CharStreams.toString(stderr)),
+            e);
+      }
+    }
+  }
+
+  private Subprocess spawnSubprocess(CredentialHelperEnvironment environment, String... args)
+      throws IOException {
+    Preconditions.checkNotNull(environment);
+    Preconditions.checkNotNull(args);
+
+    return new SubprocessBuilder()
+        .setArgv(ImmutableList.<String>builder().add(path.getPathString()).add(args).build())
+        .setWorkingDirectory(environment.getWorkspacePath().getPathFile())
+        .setEnv(environment.getClientEnvironment())
+        .setTimeoutMillis(environment.getHelperExecutionTimeout().toMillis())
+        .start();
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (o instanceof CredentialHelper) {
+      CredentialHelper that = (CredentialHelper) o;
+      return Objects.equals(this.getPath(), that.getPath());
+    }
+
+    return false;
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hashCode(getPath());
+  }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperEnvironment.java b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperEnvironment.java
new file mode 100644
index 00000000000000..e2ae01c190b3da
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperEnvironment.java
@@ -0,0 +1,75 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.authandtls.credentialhelper;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.vfs.Path;
+import java.time.Duration;
+
+/** Environment for running {@link CredentialHelper}s in. */
+@AutoValue
+public abstract class CredentialHelperEnvironment {
+  /** Returns the reporter for reporting events related to {@link CredentialHelper}s. */
+  public abstract Reporter getEventReporter();
+
+  /**
+   * Returns the (absolute) path to the workspace.
+   *
+   * <p>Used as working directory when invoking the subprocess.
+   */
+  public abstract Path getWorkspacePath();
+
+  /**
+   * Returns the environment from the Bazel client.
+   *
+   * <p>Passed as environment variables to the subprocess.
+   */
+  public abstract ImmutableMap<String, String> getClientEnvironment();
+
+  /** Returns the execution timeout for the helper subprocess. */
+  public abstract Duration getHelperExecutionTimeout();
+
+  /** Returns a new builder for {@link CredentialHelperEnvironment}. */
+  public static CredentialHelperEnvironment.Builder newBuilder() {
+    return new AutoValue_CredentialHelperEnvironment.Builder();
+  }
+
+  /** Builder for {@link CredentialHelperEnvironment}. */
+  @AutoValue.Builder
+  public abstract static class Builder {
+    /** Sets the reporter for reporting events related to {@link CredentialHelper}s. */
+    public abstract Builder setEventReporter(Reporter reporter);
+
+    /**
+     * Sets the (absolute) path to the workspace to use as working directory when invoking the
+     * subprocess.
+     */
+    public abstract Builder setWorkspacePath(Path path);
+
+    /**
+     * Sets the environment from the Bazel client to pass as environment variables to the
+     * subprocess.
+     */
+    public abstract Builder setClientEnvironment(ImmutableMap<String, String> environment);
+
+    /** Sets the execution timeout for the helper subprocess. */
+    public abstract Builder setHelperExecutionTimeout(Duration timeout);
+
+    /** Returns the newly constructed {@link CredentialHelperEnvironment}. */
+    public abstract CredentialHelperEnvironment build();
+  }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/GetCredentialsResponse.java b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/GetCredentialsResponse.java
index 72f25cf4ddf196..6d450e001fb682 100644
--- a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/GetCredentialsResponse.java
+++ b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/GetCredentialsResponse.java
@@ -50,7 +50,7 @@ public static Builder newBuilder() {
   /** Builder for {@link GetCredentialsResponse}. */
   @AutoValue.Builder
   public abstract static class Builder {
-    protected abstract ImmutableMap.Builder<String, ImmutableList<String>> headersBuilder();
+    public abstract ImmutableMap.Builder<String, ImmutableList<String>> headersBuilder();
 
     /** Returns the newly constructed {@link GetCredentialsResponse}. */
     public abstract GetCredentialsResponse build();


Test patch:
diff --git a/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD b/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD
index 95b3ee4483cdc9..024763f7c1dd63 100644
--- a/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD
@@ -17,12 +17,17 @@ filegroup(
 java_test(
     name = "credentialhelper",
     srcs = glob(["*.java"]),
+    data = [
+        ":test_credential_helper",
+    ],
     test_class = "com.google.devtools.build.lib.AllTests",
     runtime_deps = [
         "//src/test/java/com/google/devtools/build/lib:test_runner",
     ],
     deps = [
         "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper",
+        "//src/main/java/com/google/devtools/build/lib/events",
+        "//src/main/java/com/google/devtools/build/lib/util:os",
         "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
         "//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs",
@@ -30,5 +35,11 @@ java_test(
         "//third_party:guava",
         "//third_party:junit4",
         "//third_party:truth",
+        "@bazel_tools//tools/java/runfiles",
     ],
 )
+
+py_binary(
+    name = "test_credential_helper",
+    srcs = ["test_credential_helper.py"],
+)
diff --git a/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperTest.java b/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperTest.java
new file mode 100644
index 00000000000000..4aeb4595b084d0
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperTest.java
@@ -0,0 +1,143 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.authandtls.credentialhelper;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertThrows;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.util.OS;
+import com.google.devtools.build.lib.vfs.DigestHashFunction;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+import com.google.devtools.build.runfiles.Runfiles;
+import java.io.IOException;
+import java.net.URI;
+import java.time.Duration;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class CredentialHelperTest {
+  private static final PathFragment TEST_WORKSPACE_PATH =
+      PathFragment.create(System.getenv("TEST_TMPDIR"));
+  private static final PathFragment TEST_CREDENTIAL_HELPER_PATH =
+      PathFragment.create(
+          "io_bazel/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/test_credential_helper"
+              + (OS.getCurrent() == OS.WINDOWS ? ".exe" : ""));
+
+  private static final Reporter reporter = new Reporter(new EventBus());
+
+  private GetCredentialsResponse getCredentialsFromHelper(
+      String uri, ImmutableMap<String, String> env) throws Exception {
+    Preconditions.checkNotNull(uri);
+    Preconditions.checkNotNull(env);
+
+    FileSystem fs = new InMemoryFileSystem(DigestHashFunction.SHA256);
+
+    CredentialHelper credentialHelper =
+        new CredentialHelper(
+            fs.getPath(
+                Runfiles.create().rlocation(TEST_CREDENTIAL_HELPER_PATH.getSafePathString())));
+    return credentialHelper.getCredentials(
+        CredentialHelperEnvironment.newBuilder()
+            .setEventReporter(reporter)
+            .setWorkspacePath(fs.getPath(TEST_WORKSPACE_PATH))
+            .setClientEnvironment(env)
+            .setHelperExecutionTimeout(Duration.ofSeconds(5))
+            .build(),
+        URI.create(uri));
+  }
+
+  private GetCredentialsResponse getCredentialsFromHelper(String uri) throws Exception {
+    Preconditions.checkNotNull(uri);
+
+    return getCredentialsFromHelper(uri, ImmutableMap.of());
+  }
+
+  @Test
+  public void knownUriWithSingleHeader() throws Exception {
+    GetCredentialsResponse response = getCredentialsFromHelper("https://singleheader.example.com");
+    assertThat(response.getHeaders()).containsExactly("header1", ImmutableList.of("value1"));
+  }
+
+  @Test
+  public void knownUriWithMultipleHeaders() throws Exception {
+    GetCredentialsResponse response =
+        getCredentialsFromHelper("https://multipleheaders.example.com");
+    assertThat(response.getHeaders())
+        .containsExactly(
+            "header1",
+            ImmutableList.of("value1"),
+            "header2",
+            ImmutableList.of("value1", "value2"),
+            "header3",
+            ImmutableList.of("value1", "value2", "value3"));
+  }
+
+  @Test
+  public void unknownUri() {
+    IOException ioException =
+        assertThrows(
+            IOException.class, () -> getCredentialsFromHelper("https://unknown.example.com"));
+    assertThat(ioException).hasMessageThat().contains("Unknown uri 'https://unknown.example.com'");
+  }
+
+  @Test
+  public void credentialHelperOutputsNothing() throws Exception {
+    IOException ioException =
+        assertThrows(
+            IOException.class, () -> getCredentialsFromHelper("https://printnothing.example.com"));
+    assertThat(ioException).hasMessageThat().contains("exited without output");
+  }
+
+  @Test
+  public void credentialHelperOutputsExtraFields() throws Exception {
+    GetCredentialsResponse response = getCredentialsFromHelper("https://extrafields.example.com");
+    assertThat(response.getHeaders()).containsExactly("header1", ImmutableList.of("value1"));
+  }
+
+  @Test
+  public void helperRunsInWorkspace() throws Exception {
+    GetCredentialsResponse response = getCredentialsFromHelper("https://cwd.example.com");
+    ImmutableMap<String, ImmutableList<String>> headers = response.getHeaders();
+    assertThat(PathFragment.create(headers.get("cwd").get(0))).isEqualTo(TEST_WORKSPACE_PATH);
+  }
+
+  @Test
+  public void helperGetEnvironment() throws Exception {
+    GetCredentialsResponse response =
+        getCredentialsFromHelper(
+            "https://env.example.com", ImmutableMap.of("FOO", "BAR!", "BAR", "123"));
+    assertThat(response.getHeaders())
+        .containsExactly(
+            "foo", ImmutableList.of("BAR!"),
+            "bar", ImmutableList.of("123"));
+  }
+
+  @Test
+  public void helperTimeout() throws Exception {
+    IOException ioException =
+        assertThrows(
+            IOException.class, () -> getCredentialsFromHelper("https://timeout.example.com"));
+    assertThat(ioException).hasMessageThat().contains("process timed out");
+  }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/test_credential_helper.py b/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/test_credential_helper.py
new file mode 100644
index 00000000000000..c21fd7b22524e6
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/test_credential_helper.py
@@ -0,0 +1,92 @@
+# Copyright 2022 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Credential helper for testing."""
+
+import json
+import os
+import sys
+import time
+
+
+def eprint(*args, **kargs):
+  print(*args, file=sys.stderr, **kargs)
+
+
+def main(argv):
+  if len(argv) != 2:
+    eprint("Usage: test_credential_helper <command>")
+    return 1
+
+  if argv[1] != "get":
+    eprint("Unknown command '{}'".format(argv[1]))
+    return 1
+
+  request = json.load(sys.stdin)
+  if request["uri"] == "https://singleheader.example.com":
+    response = {
+        "headers": {
+            "header1": ["value1"],
+        },
+    }
+  elif request["uri"] == "https://multipleheaders.example.com":
+    response = {
+        "headers": {
+            "header1": ["value1"],
+            "header2": ["value1", "value2"],
+            "header3": ["value1", "value2", "value3"],
+        },
+    }
+  elif request["uri"] == "https://extrafields.example.com":
+    response = {
+        "foo": "YES",
+        "headers": {
+            "header1": ["value1"],
+        },
+        "umlaut": [
+            "ß",
+            "å",
+        ],
+    }
+  elif request["uri"] == "https://printnothing.example.com":
+    return 0
+  elif request["uri"] == "https://cwd.example.com":
+    response = {
+        "headers": {
+            "cwd": [os.getcwd()],
+        },
+    }
+  elif request["uri"] == "https://env.example.com":
+    response = {
+        "headers": {
+            "foo": [os.getenv("FOO")],
+            "bar": [os.getenv("BAR")],
+        },
+    }
+  elif request["uri"] == "https://timeout.example.com":
+    # We expect the subprocess to be killed after 5s.
+    time.sleep(10)
+    response = {
+        "headers": {
+            "header1": ["value1"],
+        },
+    }
+  else:
+    eprint("Unknown uri '{}'".format(request["uri"]))
+    return 1
+  json.dump(response, sys.stdout)
+  return 0
+
+
+if __name__ == "__main__":
+  sys.exit(main(sys.argv))