Repository: bazelbuild/bazel
Issue: Please add `repository_ctx.extract()` function
Labels: type: feature request
P2

The existing `repository_ctx.download_and_extract()` will, as it says on the tin, download an archive and then extract it to the local filesystem. There's also a function `repository_ctx.download()`, which lets the user handle the extraction part themselves.

This request is for a function `repository_ctx.extract()`, so that I can write custom Skylark to download an archive via a non-HTTP protocol, and then let Bazel handle extraction. Such a function is especially useful for XZ archives, which Bazel can handle natively but many OSes (e.g. MacOS) don't bundle tools for.

Issue comments:
PR: https://github.com/bazelbuild/bazel/pull/7207

Pull request review comments:

Patch that resolved the issue:
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/debug/WorkspaceRuleEvent.java b/src/main/java/com/google/devtools/build/lib/bazel/debug/WorkspaceRuleEvent.java
index ae2115374f5ab0..6f62d31a6a61cb 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/debug/WorkspaceRuleEvent.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/debug/WorkspaceRuleEvent.java
@@ -107,7 +107,32 @@ public static WorkspaceRuleEvent newDownloadEvent(
     return new WorkspaceRuleEvent(result.build());
   }
 
-  /** Creates a new WorkspaceRuleEvent for a download and excract event. */
+  /** Creates a new WorkspaceRuleEvent for an extract event. */
+  public static WorkspaceRuleEvent newExtractEvent(
+      String archive,
+      String output,
+      String stripPrefix,
+      String ruleLabel,
+      Location location) {
+    WorkspaceLogProtos.ExtractEvent.Builder e =
+        WorkspaceLogProtos.ExtractEvent.newBuilder()
+            .setArchive(archive)
+            .setOutput(output)
+            .setStripPrefix(stripPrefix);
+
+    WorkspaceLogProtos.WorkspaceEvent.Builder result =
+        WorkspaceLogProtos.WorkspaceEvent.newBuilder();
+    result = result.setExtractEvent(e.build());
+    if (location != null) {
+      result = result.setLocation(location.print());
+    }
+    if (ruleLabel != null) {
+      result = result.setRule(ruleLabel);
+    }
+    return new WorkspaceRuleEvent(result.build());
+  }
+
+  /** Creates a new WorkspaceRuleEvent for a download and extract event. */
   public static WorkspaceRuleEvent newDownloadAndExtractEvent(
       List<URL> urls,
       String output,
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/debug/workspace_log.proto b/src/main/java/com/google/devtools/build/lib/bazel/debug/workspace_log.proto
index 874ddc652fa44d..4248911024640e 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/debug/workspace_log.proto
+++ b/src/main/java/com/google/devtools/build/lib/bazel/debug/workspace_log.proto
@@ -50,6 +50,15 @@ message DownloadEvent {
   bool executable = 4;
 }
 
+message ExtractEvent {
+  // Path to the archive file
+  string archive = 1;
+  // Path to the output directory
+  string output = 2;
+  // A directory prefix to strip from extracted files.
+  string strip_prefix = 3;
+}
+
 message DownloadAndExtractEvent {
   // Url(s) to download from
   repeated string url = 1;
@@ -120,5 +129,6 @@ message WorkspaceEvent {
     SymlinkEvent symlink_event = 8;
     TemplateEvent template_event = 9;
     WhichEvent which_event = 10;
+    ExtractEvent extract_event = 11;
   }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java
index b4c581b3f02f20..24bde5c29b8e22 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java
@@ -404,6 +404,28 @@ public StructImpl download(
     return StructProvider.STRUCT.createStruct(dict, null);
   }
 
+  @Override
+  public void extract(
+      Object archive, Object output, String stripPrefix, Location location)
+      throws RepositoryFunctionException, InterruptedException, EvalException {
+    SkylarkPath archivePath = getPath("extract()", archive);
+    SkylarkPath outputPath = getPath("extract()", output);
+
+    WorkspaceRuleEvent w =
+        WorkspaceRuleEvent.newExtractEvent(
+            archive.toString(), output.toString(), stripPrefix, rule.getLabel().toString(), location);
+    env.getListener().post(w);
+
+    DecompressorValue.decompress(
+        DecompressorDescriptor.builder()
+            .setTargetKind(rule.getTargetKind())
+            .setTargetName(rule.getName())
+            .setArchivePath(archivePath.getPath())
+            .setRepositoryPath(outputPath.getPath())
+            .setPrefix(stripPrefix)
+            .build());
+  }
+
   @Override
   public StructImpl downloadAndExtract(
       Object url, Object output, String sha256, String type, String stripPrefix, Location location)
diff --git a/src/main/java/com/google/devtools/build/lib/skylarkbuildapi/repository/SkylarkRepositoryContextApi.java b/src/main/java/com/google/devtools/build/lib/skylarkbuildapi/repository/SkylarkRepositoryContextApi.java
index ebdcd84c136839..ad7f961810edf4 100644
--- a/src/main/java/com/google/devtools/build/lib/skylarkbuildapi/repository/SkylarkRepositoryContextApi.java
+++ b/src/main/java/com/google/devtools/build/lib/skylarkbuildapi/repository/SkylarkRepositoryContextApi.java
@@ -302,6 +302,50 @@ public StructApi download(
       Object url, Object output, String sha256, Boolean executable, Location location)
       throws RepositoryFunctionExceptionT, EvalException, InterruptedException;
 
+  @SkylarkCallable(
+      name = "extract",
+      doc = "Extract an archive to the repository directory.",
+      useLocation = true,
+      parameters = {
+        @Param(
+            name = "archive",
+            allowedTypes = {
+              @ParamType(type = String.class),
+              @ParamType(type = Label.class),
+              @ParamType(type = RepositoryPathApi.class)
+            },
+            named = true,
+            doc =
+                "path to the archive that will be unpacked,"
+                    + " relative to the repository directory."),
+        @Param(
+            name = "output",
+            allowedTypes = {
+              @ParamType(type = String.class),
+              @ParamType(type = Label.class),
+              @ParamType(type = RepositoryPathApi.class)
+            },
+            defaultValue = "''",
+            named = true,
+            doc =
+                "path to the directory where the archive will be unpacked,"
+                    + " relative to the repository directory."),
+        @Param(
+            name = "stripPrefix",
+            type = String.class,
+            defaultValue = "''",
+            named = true,
+            doc =
+                "a directory prefix to strip from the extracted files."
+                    + "\nMany archives contain a top-level directory that contains all files in the"
+                    + " archive. Instead of needing to specify this prefix over and over in the"
+                    + " <code>build_file</code>, this field can be used to strip it from extracted"
+                    + " files."),
+      })
+  public void extract(
+      Object archive, Object output, String stripPrefix, Location location)
+      throws RepositoryFunctionExceptionT, InterruptedException, EvalException;
+
   @SkylarkCallable(
       name = "download_and_extract",
       doc =


Test patch:
diff --git a/src/test/shell/bazel/bazel_workspaces_test.sh b/src/test/shell/bazel/bazel_workspaces_test.sh
index 00b698eb48792d..8d21cfa74880a2 100755
--- a/src/test/shell/bazel/bazel_workspaces_test.sh
+++ b/src/test/shell/bazel/bazel_workspaces_test.sh
@@ -214,6 +214,40 @@ function test_download_multiple() {
   ensure_contains_exactly 'output: "out_for_list.txt"' 1
 }
 
+function test_download_then_extract() {
+  # Prepare HTTP server with Python
+  local server_dir="${TEST_TMPDIR}/server_dir"
+  mkdir -p "${server_dir}"
+  local file_prefix="${server_dir}/download_then_extract"
+
+  pushd ${TEST_TMPDIR}
+  echo "This is one file" > server_dir/download_then_extract.txt
+  zip -r server_dir/download_then_extract.zip server_dir
+  file_sha256="$(sha256sum server_dir/download_then_extract.zip | head -c 64)"
+  popd
+
+  # Start HTTP server with Python
+  startup_server "${server_dir}"
+
+  set_workspace_command "
+  repository_ctx.download(\"http://localhost:${fileserver_port}/download_then_extract.zip\", \"downloaded_file.zip\", \"${file_sha256}\")
+  repository_ctx.extract(\"downloaded_file.zip\", \"out_dir\", \"server_dir/\")"
+
+  build_and_process_log --exclude_rule "//external:local_config_cc"
+
+  ensure_contains_exactly 'location: .*repos.bzl:3:3' 1
+  ensure_contains_exactly 'location: .*repos.bzl:4:3' 1
+  ensure_contains_atleast 'rule: "//external:repo"' 2
+  ensure_contains_exactly 'download_event' 1
+  ensure_contains_exactly "url: \"http://localhost:${fileserver_port}/download_then_extract.zip\"" 1
+  ensure_contains_exactly 'output: "downloaded_file.zip"' 1
+  ensure_contains_exactly "sha256: \"${file_sha256}\"" 1
+  ensure_contains_exactly 'extract_event' 1
+  ensure_contains_exactly 'archive: "downloaded_file.zip"' 1
+  ensure_contains_exactly 'output: "out_dir"' 1
+  ensure_contains_exactly 'strip_prefix: "server_dir/"' 1
+}
+
 function test_download_and_extract() {
   # Prepare HTTP server with Python
   local server_dir="${TEST_TMPDIR}/server_dir"