Repository: bazelbuild/bazel
Issue: Embed a release of @platforms into the Bazel binary
Labels: type: feature request
P1
team-Configurability

This is a tracking issue implementing the effort described at [Migrating @bazel_tools to @platforms](https://docs.google.com/document/d/1EArrWYUDugqJzBcb0-OxY5BH1FFPYV3jxLIXbJpD9RY/edit?pli=1#heading=h.5mcn15i0e1ch) design doc.

The main goal is to bundle a snapshot of [@platforms](https://github.com/bazelbuild/platforms) into Bazel binary into implicit external repository named `@platforms`. This effort unblocks:

* Migration of rules to Starlark (e.g. [rules_cc](https://github.com/bazelbuild/bazel/issues/7643)).
* Migration of rules into toolchains/platforms (e.g. [rules_cc](https://github.com/bazelbuild/bazel/issues/6516)).

Issue comments:
CC @katre, @gregce, @aiuto.

Pull request review comments:
Is this deterministic? (IIRC we set the timestamps to some constant on the final zip call, but it would be nice to get change pruning in this step, too)
This looks somewhat complicated... (I'm not sure if there is a simpler way to cut the first few path segments, though)
+1!

Which part worries you? That files coming from the http_archive will change timestamps but not their content? Let's address that on bazelbuild/platforms releases pipeline then?
I was shooting for the simplest possible thing that could work. I can remove external/platforms in the genrule above creating the archive, but this looked simpler to me.
I assume this is a comment to simplify the test, right?
Okie. Greenlight then.
No, this is a comment expressing my enthusiasm that you do test this. Move on!
Does the filename need to be opaque?
Nope, but I didn't care here, the filename will make sense once we have an actual release archive of bazelbuild/platforms.
SGTM.
Is it worth doing a quick 0.1.0 release of platforms to clean the name up now.
If not, it makes it harder to find and update line 174 (  "d91995f1a9497602a02d11db41a7ffa5551147be.zip": ) than it would be if it were "platforms-0.1.0.tar.gz"
and you won't need the strip prefix at line 514
I think it's deterministic, see https://source.bazel.build/bazel/+/master:src/main/java/com/google/devtools/build/lib/bazel/repository/ZipDecompressor.java;l=165
nit: last time I checked, it already was 2019
:)
I decided not to block submission of this PR on setting up release pipeline for platforms. We definitely have to set up releases for platforms repository. I filed https://github.com/bazelbuild/continuous-integration/issues/715.

Patch that resolved the issue:
diff --git a/WORKSPACE b/WORKSPACE
index 18156c2ca601a7..05b16806e0b133 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -117,6 +117,8 @@ distdir_tar(
         "8ccf4f1c351928b55d5dddf3672e3667f6978d60.tar.gz",
         "0.16.2.zip",
         "android_tools_pkg-0.5.tar.gz",
+        # bazelbuild/platforms
+        "441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip"
     ],
     dirname = "derived/distdir",
     sha256 = {
@@ -130,6 +132,8 @@ distdir_tar(
         "8ccf4f1c351928b55d5dddf3672e3667f6978d60.tar.gz": "d868ce50d592ef4aad7dec4dd32ae68d2151261913450fac8390b3fd474bb898",
         "0.16.2.zip": "9b72bb0aea72d7cbcfc82a01b1e25bf3d85f791e790ddec16c65e2d906382ee0",
         "android_tools_pkg-0.5.tar.gz": "b3aaa5c4544b9da8d226be3f5103e4469bd8cc9be07f93cdfabcfe5249d11378", # built at 86955316258c883daf0c135c8bc09ba8415f7ee7
+        # bazelbuild/platforms
+        "441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip": "c36e154b40b2495c7d82dfae5977aa46ad0a20628659e136f7a511504cee7b14",
     },
     urls = {
         "e0b0291b2c51fbe5a7cfa14473a1ae850f94f021.zip": [
@@ -167,6 +171,10 @@ distdir_tar(
         "android_tools_pkg-0.5.tar.gz": [
             "https://mirror.bazel.build/bazel_android_tools/android_tools_pkg-0.5.tar.gz",
         ],
+        "441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip": [
+            "https://mirror.bazel.build/github.com/bazelbuild/platforms/archive/441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip",
+            "https://github.com/bazelbuild/platforms/archive/441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip",
+        ],
     },
 )
 
@@ -496,6 +504,16 @@ http_archive(
     ],
 )
 
+http_archive(
+    name = "platforms",
+    sha256 = "a07fe5e75964361885db725039c2ba673f0ee0313d971ae4f50c9b18cd28b0b5",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/platforms/archive/441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip",
+        "https://github.com/bazelbuild/platforms/archive/441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip",
+    ],
+    strip_prefix = "platforms-441afe1bfdadd6236988e9cac159df6b5a9f5a98"
+)
+
 load("@io_bazel_skydoc//:setup.bzl", "skydoc_repositories")
 
 skydoc_repositories()
diff --git a/src/BUILD b/src/BUILD
index c20d6505259a19..164a8f66b98c37 100644
--- a/src/BUILD
+++ b/src/BUILD
@@ -328,10 +328,11 @@ filegroup(
     name = "package-zip" + suffix,
     srcs = ([":embedded_tools" + suffix + ".zip"] if embed else []) + [
         # The script assumes that the embedded tools zip (if exists) is the
-        # first item here, the deploy jar the second and install base key is the
-        # third
+        # first item here, the deploy jar the second, install base key is the
+        # third, and platforms archive is the fourth.
         "//src/main/java/com/google/devtools/build/lib:bazel/BazelServer_deploy.jar",
         "install_base_key" + suffix,
+        ":platforms_archive",
         ":libunix",
         "//src/main/tools:build-runfiles",
         "//src/main/tools:process-wrapper",
@@ -355,6 +356,13 @@ filegroup(
     ("_nojdk", True),
 ]]
 
+genrule(
+    name = "platforms_archive",
+    srcs = ["@platforms//:srcs"],
+    outs = ["platforms.zip"],
+    cmd = "zip -qX $@ $$(echo $(SRCS) | sort)",
+)
+
 [genrule(
     name = "bazel-bin" + suffix,
     srcs = [
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/local_config_platform.WORKSPACE b/src/main/java/com/google/devtools/build/lib/bazel/repository/local_config_platform.WORKSPACE
index d0ce5b75dba567..d345d0b521578a 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/local_config_platform.WORKSPACE
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/local_config_platform.WORKSPACE
@@ -1,1 +1,6 @@
+local_repository(
+    name = "platforms",
+    path = __embedded_dir__ + "/platforms",
+)
+
 local_config_platform(name = "local_config_platform")
diff --git a/src/package-bazel.sh b/src/package-bazel.sh
index 632b1335404130..0bdfd8e00855db 100755
--- a/src/package-bazel.sh
+++ b/src/package-bazel.sh
@@ -16,16 +16,15 @@
 
 set -euo pipefail
 
-# This script bootstraps building a Bazel binary without Bazel then
-# use this compiled Bazel to bootstrap Bazel itself. It can also
-# be provided with a previous version of Bazel to bootstrap Bazel
-# itself.
+# This script creates the Bazel archive that Bazel client unpacks and then
+# starts the server from.
 
 WORKDIR=$(pwd)
 OUT=$1
 EMBEDDED_TOOLS=$2
 DEPLOY_JAR=$3
 INSTALL_BASE_KEY=$4
+PLATFORMS_ARCHIVE=$5
 shift 4
 
 TMP_DIR=${TMPDIR:-/tmp}
@@ -66,4 +65,15 @@ if [ -n "${EMBEDDED_TOOLS}" ]; then
   (cd ${PACKAGE_DIR}/embedded_tools && unzip -q "${WORKDIR}/${EMBEDDED_TOOLS}")
 fi
 
+# Unzip platforms.zip into platforms/, move files up from external/platforms
+# subdirectory, and cleanup after itself
+( \
+  cd ${PACKAGE_DIR} && \
+    unzip -q -d platforms platforms.zip && \
+    rm platforms.zip && \
+    cd platforms && \
+    mv external/platforms/* . && \
+    rmdir -p external/platforms \
+)
+
 (cd ${PACKAGE_DIR} && find . -type f | sort | zip -q9DX@ "${WORKDIR}/${OUT}")


Test patch:
diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD
index 86868870eb1b78..391bcb4f397c1d 100644
--- a/src/test/shell/bazel/BUILD
+++ b/src/test/shell/bazel/BUILD
@@ -1080,6 +1080,12 @@ sh_test(
     deps = ["@bazel_tools//tools/bash/runfiles"],
 )
 
+sh_test(
+    name = "platforms_test",
+    srcs = ["platforms_test.sh"],
+    data = [":test-deps"],
+)
+
 sh_test(
     name = "platform_mapping_test",
     srcs = ["platform_mapping_test.sh"],
diff --git a/src/test/shell/bazel/platforms_test.sh b/src/test/shell/bazel/platforms_test.sh
new file mode 100755
index 00000000000000..585f490c604f33
--- /dev/null
+++ b/src/test/shell/bazel/platforms_test.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+#
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Test related to @platforms embedded repository
+#
+
+set -euo pipefail
+
+# Load the test setup defined in the parent directory
+CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${CURRENT_DIR}/../integration_test_setup.sh" \
+  || { echo "integration_test_setup.sh not found!" >&2; exit 1; }
+
+function test_platforms_repository_builds_itself() {
+  # We test that a built-in @platforms repository is buildable.
+  bazel build @platforms//:all &> $TEST_log \
+      || fail "Build failed unexpectedly"
+}
+
+function test_platforms_can_be_overridden() {
+  # We test that a custom repository can override @platforms in their
+  # WORKSPACE file.
+  mkdir -p platforms_can_be_overridden || fail "couldn't create directory"
+  touch platforms_can_be_overridden/BUILD || \ fail "couldn't touch BUILD file"
+  cat > platforms_can_be_overridden/WORKSPACE <<EOF
+local_repository(
+  name = 'platforms',
+  path = '../override',
+)
+EOF
+
+  mkdir -p override || fail "couldn't create override directory"
+  touch override/WORKSPACE || fail "couldn't touch override/WORKSPACE"
+  cat > override/BUILD <<EOF
+filegroup(name = 'yolo')
+EOF
+
+  cd platforms_can_be_overridden || fail "couldn't cd into workspace"
+  bazel build @platforms//:yolo &> $TEST_log || \
+    fail "Bazel failed to build @platforms"
+}
+
+run_suite "platform mapping test"
+