Repository: bazelbuild/bazel
Issue: Support querying repo mappings
Labels: type: feature request
P1
team-ExternalDeps

### Description of the feature request:


Add a way of querying for repo mappings, for usage in external tools (such as language servers). The repo mapping is necessary for these tools to resolve labels to files on disk, since the files live under the canonical repo name.

### Which category does this issue belong to?


External Dependency

### What underlying problem are you trying to solve with this feature?


Facebook have a [starlark parser & lsp](https://github.com/facebookexperimental/starlark-rust) which works pretty great with Bazel. However, it doesn't support bzlmod because it assumes the repo name of a label is the canonical name, i.e. it looks up `@foo` in `$(bazel info output_base)/external/foo`.

According to @fmeum, this is needed for intellij too.

### Which operating system are you running Bazel on?


_No response_

### What is the output of `bazel info release`?


_No response_

### If `bazel info release` returns `development version` or `(@non-git)`, tell us how you built Bazel.


_No response_

### What's the output of `git remote get-url origin; git rev-parse master; git rev-parse HEAD` ?


_No response_

### Have you found anything relevant by searching the web?

_No response_

### Any other information, logs, or outputs that you want to share?

Slack discussion: https://bazelbuild.slack.com/archives/C014RARENH0/p1703107424515999

Issue comments:
I started working on this in the past in https://github.com/bazelbuild/bazel/pull/18990, but noticed a problems with my implementation: All repositories generated by a module extension share the same repository mapping containing all these repos. If stored naively, this results in memory usage that is quadratic in the number of extension repos, which can easily be in the 1,000s. Thus, I think it's necessary to "intern" the repo mappings by having one array containing the deduplicated mappings and a map that references them by ID, e.g.:
```json
{
  "mappings": [{"foo": "foo~1.2.3, ...}, {"bar": "bar~4.5.6", ...}, ...],
  "repoToMappingIndex": {"foo~1.2.3": 0, ...}
}
```

Happy to support anyone who wants to pick this up!
CC @mai93 as this format will also be needed to make IntelliJ's "Go to Definition" features work for labels with Bzlmod.
Looks like this one is done! \o/
A fix for this issue has been included in [Bazel 7.1.0 RC1](https://releases.bazel.build/7.1.0/rc1/index.html). Please test out the release candidate and report any issues as soon as possible. Thanks!

Pull request review comments:

Patch that resolved the issue:
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModOptions.java
index ab45297ae423fc..bac6b42d6b9c48 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModOptions.java
@@ -172,7 +172,8 @@ public enum ModSubcommand {
     PATH(true),
     EXPLAIN(true),
     SHOW_REPO(false),
-    SHOW_EXTENSION(false);
+    SHOW_EXTENSION(false),
+    DUMP_REPO_MAPPING(false);
 
     /** Whether this subcommand produces a graph output. */
     private final boolean isGraph;
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD
index d4bd139ed31e78..2ce21351e194f2 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD
@@ -65,6 +65,7 @@ java_library(
         "//src/main/java/com/google/devtools/common/options",
         "//src/main/java/net/starlark/java/eval",
         "//src/main/protobuf:failure_details_java_proto",
+        "//third_party:gson",
         "//third_party:guava",
     ],
 )
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java
index 77c0be124615c8..ec6d9d3caa8158 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java
@@ -13,6 +13,7 @@
 // limitations under the License.
 package com.google.devtools.build.lib.bazel.commands;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModOptions.Charset.UTF8;
@@ -44,6 +45,7 @@
 import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModOptions.ModSubcommandConverter;
 import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModuleArg;
 import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModuleArg.ModuleArgConverter;
+import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
 import com.google.devtools.build.lib.cmdline.RepositoryMapping;
 import com.google.devtools.build.lib.cmdline.RepositoryName;
 import com.google.devtools.build.lib.events.Event;
@@ -57,6 +59,7 @@
 import com.google.devtools.build.lib.server.FailureDetails;
 import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
 import com.google.devtools.build.lib.server.FailureDetails.ModCommand.Code;
+import com.google.devtools.build.lib.skyframe.RepositoryMappingValue;
 import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
 import com.google.devtools.build.lib.util.AbruptExitException;
 import com.google.devtools.build.lib.util.DetailedExitCode;
@@ -70,11 +73,17 @@
 import com.google.devtools.common.options.OptionsParser;
 import com.google.devtools.common.options.OptionsParsingException;
 import com.google.devtools.common.options.OptionsParsingResult;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.stream.JsonWriter;
+import java.io.IOException;
 import java.io.OutputStreamWriter;
+import java.io.Writer;
 import java.util.List;
 import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.stream.IntStream;
 
 /** Queries the Bzlmod external dependency graph. */
 @Command(
@@ -125,8 +134,51 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti
   }
 
   private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingResult options) {
+    ModOptions modOptions = options.getOptions(ModOptions.class);
+    Preconditions.checkArgument(modOptions != null);
+
+    if (options.getResidue().isEmpty()) {
+      String errorMessage =
+          String.format(
+              "No subcommand specified, choose one of : %s.", ModSubcommand.printValues());
+      return reportAndCreateFailureResult(env, errorMessage, Code.MOD_COMMAND_UNKNOWN);
+    }
+
+    // The first element in the residue must be the subcommand, and then comes a list of arguments.
+    String subcommandStr = options.getResidue().get(0);
+    ModSubcommand subcommand;
+    try {
+      subcommand = new ModSubcommandConverter().convert(subcommandStr);
+    } catch (OptionsParsingException e) {
+      String errorMessage =
+          String.format("Invalid subcommand, choose one from : %s.", ModSubcommand.printValues());
+      return reportAndCreateFailureResult(env, errorMessage, Code.MOD_COMMAND_UNKNOWN);
+    }
+    List<String> args = options.getResidue().subList(1, options.getResidue().size());
+
+    ImmutableList.Builder<RepositoryMappingValue.Key> repositoryMappingKeysBuilder =
+        ImmutableList.builder();
+    if (subcommand.equals(ModSubcommand.DUMP_REPO_MAPPING)) {
+      if (args.isEmpty()) {
+        // Make this case an error so that we are free to add a mode that emits all mappings in a
+        // single JSON object later.
+        return reportAndCreateFailureResult(
+            env, "No repository name(s) specified", Code.INVALID_ARGUMENTS);
+      }
+      for (String arg : args) {
+        try {
+          repositoryMappingKeysBuilder.add(RepositoryMappingValue.key(RepositoryName.create(arg)));
+        } catch (LabelSyntaxException e) {
+          return reportAndCreateFailureResult(env, e.getMessage(), Code.INVALID_ARGUMENTS);
+        }
+      }
+    }
+    ImmutableList<RepositoryMappingValue.Key> repoMappingKeys =
+        repositoryMappingKeysBuilder.build();
+
     BazelDepGraphValue depGraphValue;
     BazelModuleInspectorValue moduleInspector;
+    ImmutableList<RepositoryMappingValue> repoMappingValues;
 
     SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor();
     LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class);
@@ -140,10 +192,14 @@ private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingRe
     try {
       env.syncPackageLoading(options);
 
+      ImmutableSet.Builder<SkyKey> keys = ImmutableSet.builder();
+      if (subcommand.equals(ModSubcommand.DUMP_REPO_MAPPING)) {
+        keys.addAll(repoMappingKeys);
+      } else {
+        keys.add(BazelDepGraphValue.KEY, BazelModuleInspectorValue.KEY);
+      }
       EvaluationResult<SkyValue> evaluationResult =
-          skyframeExecutor.prepareAndGet(
-              ImmutableSet.of(BazelDepGraphValue.KEY, BazelModuleInspectorValue.KEY),
-              evaluationContext);
+          skyframeExecutor.prepareAndGet(keys.build(), evaluationContext);
 
       if (evaluationResult.hasError()) {
         Exception e = evaluationResult.getError().getException();
@@ -159,6 +215,11 @@ private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingRe
       moduleInspector =
           (BazelModuleInspectorValue) evaluationResult.get(BazelModuleInspectorValue.KEY);
 
+      repoMappingValues =
+          repoMappingKeys.stream()
+              .map(evaluationResult::get)
+              .map(RepositoryMappingValue.class::cast)
+              .collect(toImmutableList());
     } catch (InterruptedException e) {
       String errorMessage = "mod command interrupted: " + e.getMessage();
       env.getReporter().handle(Event.error(errorMessage));
@@ -169,27 +230,29 @@ private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingRe
       return BlazeCommandResult.detailedExitCode(e.getDetailedExitCode());
     }
 
-    ModOptions modOptions = options.getOptions(ModOptions.class);
-    Preconditions.checkArgument(modOptions != null);
-
-    if (options.getResidue().isEmpty()) {
-      String errorMessage =
-          String.format(
-              "No subcommand specified, choose one of : %s.", ModSubcommand.printValues());
-      return reportAndCreateFailureResult(env, errorMessage, Code.MOD_COMMAND_UNKNOWN);
-    }
-
-    // The first element in the residue must be the subcommand, and then comes a list of arguments.
-    String subcommandStr = options.getResidue().get(0);
-    ModSubcommand subcommand;
-    try {
-      subcommand = new ModSubcommandConverter().convert(subcommandStr);
-    } catch (OptionsParsingException e) {
-      String errorMessage =
-          String.format("Invalid subcommand, choose one from : %s.", ModSubcommand.printValues());
-      return reportAndCreateFailureResult(env, errorMessage, Code.MOD_COMMAND_UNKNOWN);
+    if (subcommand.equals(ModSubcommand.DUMP_REPO_MAPPING)) {
+      String missingRepos =
+          IntStream.range(0, repoMappingKeys.size())
+              .filter(i -> repoMappingValues.get(i) == RepositoryMappingValue.NOT_FOUND_VALUE)
+              .mapToObj(repoMappingKeys::get)
+              .map(RepositoryMappingValue.Key::repoName)
+              .map(RepositoryName::getName)
+              .collect(joining(", "));
+      if (!missingRepos.isEmpty()) {
+        return reportAndCreateFailureResult(
+            env, "Repositories not found: " + missingRepos, Code.INVALID_ARGUMENTS);
+      }
+      try {
+        dumpRepoMappings(
+            repoMappingValues,
+            new OutputStreamWriter(
+                env.getReporter().getOutErr().getOutputStream(),
+                modOptions.charset == UTF8 ? UTF_8 : US_ASCII));
+      } catch (IOException e) {
+        throw new IllegalStateException(e);
+      }
+      return BlazeCommandResult.success();
     }
-    List<String> args = options.getResidue().subList(1, options.getResidue().size());
 
     // Extract and check the --base_module argument first to use it when parsing the other args.
     // Can only be a TargetModule or a repoName relative to the ROOT.
@@ -453,6 +516,8 @@ private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingRe
       case SHOW_EXTENSION:
         modExecutor.showExtension(argsAsExtensions, usageKeys);
         break;
+      default:
+        throw new IllegalStateException("Unexpected subcommand: " + subcommand);
     }
 
     return BlazeCommandResult.success();
@@ -510,4 +575,21 @@ private static BlazeCommandResult createFailureResult(String message, Code detai
                 .setMessage(message)
                 .build()));
   }
+
+  public static void dumpRepoMappings(List<RepositoryMappingValue> repoMappings, Writer writer)
+      throws IOException {
+    Gson gson = new GsonBuilder().disableHtmlEscaping().create();
+    for (RepositoryMappingValue repoMapping : repoMappings) {
+      JsonWriter jsonWriter = gson.newJsonWriter(writer);
+      jsonWriter.beginObject();
+      for (Entry<String, RepositoryName> entry :
+          repoMapping.getRepositoryMapping().entries().entrySet()) {
+        jsonWriter.name(entry.getKey());
+        jsonWriter.value(entry.getValue().getName());
+      }
+      jsonWriter.endObject();
+      writer.write('\n');
+    }
+    writer.flush();
+  }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/mod.txt b/src/main/java/com/google/devtools/build/lib/bazel/commands/mod.txt
index 01d576972cf61d..8cdc0fdc1aa1ab 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/commands/mod.txt
+++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/mod.txt
@@ -12,6 +12,7 @@ The command will display the external dependency graph or parts thereof, structu
     - explain <module>...: Prints all the places where the module is (or was) requested as a direct dependency, along with the reason why the respective final version was selected. It will display a pruned version of the `all_paths <module>...` command which only contains the direct deps of the root, the <module(s)> leaves, along with their dependants (can be modified with --depth).
     - show_repo <module>...: Prints the rule that generated the specified repos (i.e. http_archive()). The arguments may refer to extension-generated repos.
     - show_extension <extension>...: Prints information about the given extension(s). Usages can be filtered down to only those from modules in --extension_usage.
+    - dump_repo_mapping <canonical_repo_name>...: Prints the mappings from apparent repo names to canonical repo names for the given repos in NDJSON format. The order of entries within each JSON object is unspecified. This command is intended for use by tools such as IDEs and Starlark language servers.
 
 
 <module> arguments must be one of the following:
@@ -25,4 +26,6 @@ The command will display the external dependency graph or parts thereof, structu
 
 <extension> arguments must be of the form <module><label_to_bzl_file>%<extension_name>. For example, both rules_java//java:extensions.bzl%toolchains and @rules_java//java:extensions.bzl%toolchains are valid specifications of extensions.
 
+<canonical_repo_name> arguments are canonical repo names without any leading @ characters. The canonical repo name of the root module repository is the empty string.
+
 %{options}
\ No newline at end of file


Test patch:
diff --git a/src/test/py/bazel/bzlmod/mod_command_test.py b/src/test/py/bazel/bzlmod/mod_command_test.py
index d17264ed2f7923..74172b0c765e7d 100644
--- a/src/test/py/bazel/bzlmod/mod_command_test.py
+++ b/src/test/py/bazel/bzlmod/mod_command_test.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 """Tests the mod command."""
 
+import json
 import os
 import tempfile
 from absl.testing import absltest
@@ -454,6 +455,72 @@ def testShowRepoThrowsUnusedModule(self):
         stderr,
     )
 
+  def testDumpRepoMapping(self):
+    _, stdout, _ = self.RunBazel(
+        [
+            'mod',
+            'dump_repo_mapping',
+            '',
+            'foo~2.0',
+        ],
+    )
+    root_mapping, foo_mapping = [json.loads(l) for l in stdout]
+
+    self.assertContainsSubset(
+        {
+            'my_project': '',
+            'foo1': 'foo~1.0',
+            'foo2': 'foo~2.0',
+            'myrepo2': 'ext2~1.0~ext~repo1',
+            'bazel_tools': 'bazel_tools',
+        }.items(),
+        root_mapping.items(),
+    )
+
+    self.assertContainsSubset(
+        {
+            'foo': 'foo~2.0',
+            'ext_mod': 'ext~1.0',
+            'my_repo3': 'ext~1.0~ext~repo3',
+            'bazel_tools': 'bazel_tools',
+        }.items(),
+        foo_mapping.items(),
+    )
+
+  def testDumpRepoMappingThrowsNoRepos(self):
+    _, _, stderr = self.RunBazel(
+        ['mod', 'dump_repo_mapping'],
+        allow_failure=True,
+    )
+    self.assertIn(
+        "ERROR: No repository name(s) specified. Type 'bazel help mod' for"
+        ' syntax and help.',
+        stderr,
+    )
+
+  def testDumpRepoMappingThrowsInvalidRepoName(self):
+    _, _, stderr = self.RunBazel(
+        ['mod', 'dump_repo_mapping', '{}'],
+        allow_failure=True,
+    )
+    self.assertIn(
+        "ERROR: invalid repository name '{}': repo names may contain only A-Z,"
+        " a-z, 0-9, '-', '_', '.' and '~' and must not start with '~'. Type"
+        " 'bazel help mod' for syntax and help.",
+        stderr,
+    )
+
+  def testDumpRepoMappingThrowsUnknownRepoName(self):
+    _, _, stderr = self.RunBazel(
+        ['mod', 'dump_repo_mapping', 'does_not_exist'],
+        allow_failure=True,
+    )
+    self.assertIn(
+        "ERROR: Repositories not found: does_not_exist. Type 'bazel help mod'"
+        ' for syntax and help.',
+        stderr,
+    )
+
 
 if __name__ == '__main__':
   absltest.main()