Repository: kythe/kythe
Pull request #4857 (2021-03-09 22:00:54)

fix(docs): Use rules_ruby to hermetically generated the web site 
Let's try this again ...

shahms: Not a huge fan of the rules_ruby dependency, but it's a more accurate reflection of our actual dependencies unless/until we move off of asciidoc.
shahms: PTAL?

Patch:
diff --git a/.gitignore b/.gitignore
index 0d939c6bc..d673acf35 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,10 +31,6 @@ third_party/openssl
 third_party/ossp-uuid
 tools/node
 
-# Kythe website artifacts
-kythe/web/site/.bundle/
-kythe/web/site/_vendor/
-
 # User bazelrc overrides
 /user.bazelrc
 
diff --git a/WORKSPACE b/WORKSPACE
index 3edba04c2..3577dbf10 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -67,3 +67,15 @@ rbe_autoconfig(
     toolchain_config_suite_spec = DEFAULT_TOOLCHAIN_CONFIG_SUITE_SPEC,
     use_legacy_platform_definition = False,
 )
+
+load(
+    "@bazelruby_rules_ruby//ruby:defs.bzl",
+    "ruby_bundle",
+)
+
+ruby_bundle(
+    name = "website_bundle",
+    bundler_version = "2.1.4",
+    gemfile = "//kythe/web/site:Gemfile",
+    gemfile_lock = "//kythe/web/site:Gemfile.lock",
+)
diff --git a/external.bzl b/external.bzl
index bde1a21d5..389dfcc75 100644
--- a/external.bzl
+++ b/external.bzl
@@ -19,6 +19,11 @@ load("@bazel_toolchains//repositories:repositories.bzl", bazel_toolchains_reposi
 load("@io_bazel_rules_rust//rust:repositories.bzl", "rust_repositories")
 load("@io_bazel_rules_rust//proto:repositories.bzl", "rust_proto_repositories")
 load("@build_bazel_rules_nodejs//:index.bzl", "npm_install")
+load(
+    "@bazelruby_rules_ruby//ruby:deps.bzl",
+    "rules_ruby_dependencies",
+    "rules_ruby_select_sdk",
+)
 
 # The raze macros automatically check for duplicated dependencies so we can
 # simply load each macro here.
@@ -34,6 +39,8 @@ def _rule_dependencies():
     bazel_toolchains_repositories()
     rust_repositories(version = "nightly", iso_date = "2020-06-23", dev_components = True)
     rust_proto_repositories()
+    rules_ruby_dependencies()
+    rules_ruby_select_sdk(version = "host")
 
 def _gazelle_ignore(**kwargs):
     """Dummy macro which causes gazelle to see a repository as already defined."""
diff --git a/kythe/docs/BUILD b/kythe/docs/BUILD
index fdb984290..a57578a5f 100644
--- a/kythe/docs/BUILD
+++ b/kythe/docs/BUILD
@@ -66,4 +66,8 @@ asciidoc(
 bzl_library(
     name = "asciidoc_bzl",
     srcs = ["asciidoc.bzl"],
+    deps = [
+        "@bazel_skylib//lib:paths",
+        "@bazel_skylib//lib:shell",
+    ],
 )
diff --git a/kythe/docs/asciidoc.bzl b/kythe/docs/asciidoc.bzl
index a8e513cc0..369b75e63 100644
--- a/kythe/docs/asciidoc.bzl
+++ b/kythe/docs/asciidoc.bzl
@@ -1,12 +1,25 @@
 load("@bazel_skylib//lib:shell.bzl", "shell")
+load("@bazel_skylib//lib:paths.bzl", "paths")
+
+AsciidocInfo = provider(
+    doc = "Information about the asciidoc-generated files.",
+    fields = {
+        "primary_output_path": "Path of the primary output file beneath {resource_dir}.",
+        "resource_dir": "File for the directory containing all of the generated resources.",
+    },
+)
 
 _toolchain_type = "//tools/build_rules/external_tools:external_tools_toolchain_type"
 
 def _asciidoc_impl(ctx):
+    resource_dir = ctx.actions.declare_directory(ctx.label.name + ".d")
+    primary_output = "{name}.html".format(name = ctx.label.name)
+
+    # Declared as an output, but not saved as part of the default output group.
+    # Build with --output_groups=+asciidoc_logfile to retain.
+    logfile = ctx.actions.declare_file(ctx.label.name + ".logfile")
+
     # Locate the asciidoc binary from the toolchain and construct its args.
-    # Because an asciidoc run produces an unknown number of files from the
-    # execution of filters, the output is staged to a temporary directory
-    # and packaged into a .zip file.
     asciidoc = ctx.toolchains[_toolchain_type].asciidoc
     args = ["--backend", "html", "--no-header-footer"]
     for key, value in ctx.attr.attrs.items():
@@ -17,17 +30,13 @@ def _asciidoc_impl(ctx):
     if ctx.attr.example_script:
         args.append("--attribute=example_script=" + ctx.file.example_script.path)
     args += ["--conf-file=%s" % c.path for c in ctx.files.confs]
-    args += ["-o", "out/" + ctx.attr.name + ".html"]
+    args += ["-o", paths.join(resource_dir.path, primary_output)]
     args.append(ctx.file.src.path)
 
     # Get the path where all our necessary tools are located so it can be set
     # to PATH in our run_shell command.
     tool_path = ctx.toolchains[_toolchain_type].path
 
-    # Declare the logfile as an output so that it can be read if something goes
-    # awry (otherwise Bazel will clean it up).
-    logfile = ctx.actions.declare_file(ctx.attr.name + ".logfile")
-
     # Resolve data targets to get input files and runfiles manifests.
     data, _, manifests = ctx.resolve_command(tools = ctx.attr.data)
 
@@ -39,41 +48,34 @@ def _asciidoc_impl(ctx):
                   ([ctx.file.example_script] if ctx.file.example_script else []) +
                   data),
         input_manifests = manifests,
-        outputs = [ctx.outputs.out, logfile],
+        outputs = [resource_dir, logfile],
+        arguments = args,
         command = "\n".join([
+            "set -e",
+            "mkdir -p {resource_dir}".format(resource_dir = shell.quote(resource_dir.path)),
             # so we can locate the binaries asciidoc needs
-            'export PATH="$PATH:' + tool_path + '"',
-
-            # Create the temporary staging directory.
-            "mkdir out",
-
+            'export PATH="$PATH:{tool_path}"'.format(tool_path = tool_path),
             # Run asciidoc itself, and fail if it returns nonzero.
-            "%s %s 2> %s" % (
-                shell.quote(asciidoc),
-                " ".join([shell.quote(arg) for arg in args]),
-                shell.quote(logfile.path),
+            "{asciidoc} \"$@\" 2> >(tee -a {logfile} >&2)".format(
+                logfile = shell.quote(logfile.path),
+                asciidoc = shell.quote(asciidoc),
             ),
-            "if [[ $? -ne 0 ]]; then",
-            "exit 1",
-            "fi",
-
             # The tool succeeded, but now check for error diagnostics.
-            "cat %s" % (shell.quote(logfile.path)),
-            'grep -q -e "filter non-zero exit code" -e "no output from filter" %s' % (
-                shell.quote(logfile.path)
+            'if grep -q -e "filter non-zero exit code" -e "no output from filter" {logfile}; then'.format(
+                logfile = shell.quote(logfile.path),
             ),
-            "if [[ $? -ne 1 ]]; then",
             "exit 1",
             "fi",
-
-            # Move any generated images to the out directory.
-            "find . -name '*.svg' -maxdepth 1 -exec mv '{}' out/ \\;",
-
-            # Package up the outputs into the zip file.
-            "(cd out; zip -9qr ../%s *)" % ctx.outputs.out.path,
+            # Move SVGs to the appropriate directory.
+            "find . -name '*.svg' -maxdepth 1 -exec mv '{{}}' {out}/ \\;".format(out = shell.quote(resource_dir.path)),
         ]),
         mnemonic = "RunAsciidoc",
     )
+    return [
+        DefaultInfo(files = depset([resource_dir])),
+        OutputGroupInfo(asciidoc_logfile = depset([logfile])),
+        AsciidocInfo(primary_output_path = primary_output, resource_dir = resource_dir),
+    ]
 
 asciidoc = rule(
     implementation = _asciidoc_impl,
@@ -100,7 +102,4 @@ asciidoc = rule(
         ),
     },
     doc = "Generate asciidoc",
-    outputs = {
-        "out": "%{name}.zip",
-    },
 )
diff --git a/kythe/docs/schema/BUILD b/kythe/docs/schema/BUILD
index 6be25eee8..f7060c900 100644
--- a/kythe/docs/schema/BUILD
+++ b/kythe/docs/schema/BUILD
@@ -49,4 +49,5 @@ asciidoc_with_verifier(
 bzl_library(
     name = "asciidoc_with_verifier_bzl",
     srcs = ["asciidoc_with_verifier.bzl"],
+    deps = ["//kythe/docs:asciidoc_bzl"],
 )
diff --git a/kythe/release/appengine/default/deploy.sh b/kythe/release/appengine/default/deploy.sh
index a80b1147d..672c7db39 100755
--- a/kythe/release/appengine/default/deploy.sh
+++ b/kythe/release/appengine/default/deploy.sh
@@ -21,7 +21,8 @@ cd "$(dirname "$0")"
 COMMIT="$(git rev-parse HEAD)"
 echo "Deploying Kythe website version $COMMIT" >&2
 
-../../../web/site/build.sh -d "$PWD/site"
+bazel build //kythe/web/site
+rsync -Lr --delete "$(bazel info workspace)/bazel-bin/kythe/web/site/_site/" "$PWD/site/"
 gcloud app deploy --no-promote --version "$COMMIT" app.yaml
 
 echo >&2
diff --git a/kythe/web/site/.gitignore b/kythe/web/site/.gitignore
index 2a0107d5c..5162e2e81 100644
--- a/kythe/web/site/.gitignore
+++ b/kythe/web/site/.gitignore
@@ -1,5 +1,7 @@
-_site
-.sass-cache
-.jekyll-cache
-_docs/
 asciidoc-pygments.css
+_docs/
+_site/
+_vendor/
+.bundle/
+.jekyll-cache/
+.sass-cache/
diff --git a/kythe/web/site/BUILD b/kythe/web/site/BUILD
new file mode 100644
index 000000000..f0c45f32d
--- /dev/null
+++ b/kythe/web/site/BUILD
@@ -0,0 +1,123 @@
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+load("@bazelruby_rules_ruby//ruby:defs.bzl", "ruby_binary")
+load(":site_docs.bzl", "jekyll_build", "site_docs")
+
+ruby_binary(
+    name = "doc_header",
+    srcs = ["doc_header.rb"],
+    tags = ["manual"],
+    visibility = ["//visibility:public"],
+    deps = ["@website_bundle//:asciidoctor"],
+)
+
+site_docs(
+    name = "_docs",
+    srcs = [
+        "//kythe/docs:exploring-kythe-sample-web-ui",
+        "//kythe/docs:kythe-compatible-compilers",
+        "//kythe/docs:kythe-compilation-database",
+        "//kythe/docs:kythe-configurable-extraction",
+        "//kythe/docs:kythe-kzip",
+        "//kythe/docs:kythe-overview",
+        "//kythe/docs:kythe-storage",
+        "//kythe/docs:kythe-uri-spec",
+        "//kythe/docs:kythe-verifier",
+        "//kythe/docs:kythes-command-line-tool",
+        "//kythe/docs:modeling-libraries",
+        "//kythe/docs:schema-overview",
+    ],
+    tags = ["manual"],
+    deps = [":schema"],
+)
+
+site_docs(
+    name = "schema",
+    srcs = [
+        "//kythe/docs/schema",
+        "//kythe/docs/schema:callgraph",
+        "//kythe/docs/schema:indexing-generated-code",
+        "//kythe/docs/schema:influences-relation",
+        "//kythe/docs/schema:marked-source",
+        "//kythe/docs/schema:modules",
+        "//kythe/docs/schema:verifierstyle",
+        "//kythe/docs/schema:writing-an-indexer",
+    ],
+    rename_files = {
+        "schema.html": "index.html",
+    },
+    tags = ["manual"],
+)
+
+jekyll_build(
+    name = "site",
+    srcs = [":site_sources"],
+    tags = ["manual"],
+)
+
+filegroup(
+    name = "site_sources",
+    srcs = glob(
+        ["**"],
+        exclude = [
+            "README.adoc",
+            "Gemfile.lock",
+            "Gemfile",
+            "build.sh",
+            "sync_docs.sh",
+            "check_links*.sh",
+            "site_docs.bzl",
+            "BUILD",
+            "doc_header.rb",
+            "_site*/**",
+            "_vendor/**",
+            "_docs/**",
+            ".bundle/**",
+            "asciidoc-pygments.css",
+        ],
+    ) + [":_docs"],
+    tags = ["manual"],
+)
+
+filegroup(
+    name = "site_files",
+    srcs = [
+        ":site",
+        ":site_sources",
+    ],
+    tags = ["manual"],
+)
+
+sh_test(
+    name = "check_links_test",
+    srcs = ["check_links_test.sh"],
+    data = [
+        # runfiles, $(execpath) and directories interact poorly.
+        # so include both the :site itself and each of the files as
+        # discovered via :site_files.
+        ":site",
+        ":site_files",
+        # Similarly :bin/jekyll is a plain file, :jekyll is a ruby library
+        # and we need the toolchain.
+        "@org_ruby_lang_ruby_toolchain//:ruby_bin",
+        "@website_bundle//:bin/jekyll",
+        "@website_bundle//:jekyll",
+    ],
+    env = {
+        "JEKYLL_BIN": "$(location @website_bundle//:bin/jekyll)",
+    },
+    tags = [
+        "arc-ignore",
+        "local",
+        "manual",
+    ],
+)
+
+bzl_library(
+    name = "site_docs_bzl",
+    srcs = ["site_docs.bzl"],
+    deps = [
+        "//kythe/docs:asciidoc_bzl",
+        "@bazel_skylib//lib:paths",
+        "@bazel_skylib//lib:shell",
+    ],
+)
diff --git a/kythe/web/site/Gemfile b/kythe/web/site/Gemfile
index fea2c7160..880689d40 100644
--- a/kythe/web/site/Gemfile
+++ b/kythe/web/site/Gemfile
@@ -3,6 +3,7 @@ gem 'jekyll'
 gem 'jekyll-redirect-from'
 gem 'asciidoctor'
 gem 'nokogiri'
+gem 'pygments.rb'
 group :jekyll_plugins do
   gem "jekyll-asciidoc"
 end
diff --git a/kythe/web/site/Gemfile.lock b/kythe/web/site/Gemfile.lock
index 8075f4c91..3c1abbf76 100644
--- a/kythe/web/site/Gemfile.lock
+++ b/kythe/web/site/Gemfile.lock
@@ -55,6 +55,7 @@ GEM
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
     public_suffix (4.0.6)
+    pygments.rb (2.1.0)
     racc (1.5.2)
     rb-fsevent (0.10.4)
     rb-inotify (0.10.1)
@@ -77,6 +78,7 @@ DEPENDENCIES
   jekyll-asciidoc
   jekyll-redirect-from
   nokogiri
+  pygments.rb
 
 BUNDLED WITH
    2.1.4
diff --git a/kythe/web/site/_config.yml b/kythe/web/site/_config.yml
index fd07448c2..f5afc8a06 100644
--- a/kythe/web/site/_config.yml
+++ b/kythe/web/site/_config.yml
@@ -5,7 +5,7 @@ description: >
   with code.
 permalink: pretty
 future: false
-exclude: ['README.adoc', 'Gemfile.lock', 'Gemfile', 'build.sh', 'sync_docs.sh', 'check_links.sh']
+exclude: ['README.adoc', 'Gemfile.lock', 'Gemfile', 'build.sh', 'check_links_test.sh', 'site_docs.bzl', 'BUILD', 'doc_header.rb']
 collections:
   docs:
     output: true
@@ -17,3 +17,7 @@ asciidoctor:
   attributes:
     - hardbreaks!
     - source-highlighter=pygments
+sass:
+  # Paths are CWD relative, not --source relative.
+  # https://github.com/jekyll/jekyll-sass-converter/issues/115
+  sourcemap: never
diff --git a/kythe/web/site/build.sh b/kythe/web/site/build.sh
index e1daebad8..e1011e7e9 100755
--- a/kythe/web/site/build.sh
+++ b/kythe/web/site/build.sh
@@ -15,7 +15,4 @@
 # limitations under the License.
 
 cd "$(dirname "$0")"
-
-bundle install --path _vendor/bundle
-./sync_docs.sh
-bundle exec jekyll build "$@"
+bazel build :site
diff --git a/kythe/web/site/check_links.sh b/kythe/web/site/check_links.sh
deleted file mode 100755
index 586929567..000000000
--- a/kythe/web/site/check_links.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash -e
-#
-# Copyright 2014 The Kythe Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Builds website, serves it locally, and checks it for any broken links.
-
-cd "$(dirname "$0")"
-
-./build.sh
-bundle exec jekyll serve --skip-initial-build &
-server_pid=$!
-trap 'kill "$server_pid"' EXIT ERR INT
-
-while ! curl -s localhost:4000 >/dev/null; do
-  echo 'Waiting for server...' >&2
-  sleep 1s
-done
-
-echo
-set -o pipefail
-wget --spider -nv -e robots=off -r -p http://localhost:4000 2>&1 | \
-  grep -v -e 'unlink: ' -e ' URL:'
diff --git a/kythe/web/site/doc_header.rb b/kythe/web/site/doc_header.rb
new file mode 100644
index 000000000..f09225afc
--- /dev/null
+++ b/kythe/web/site/doc_header.rb
@@ -0,0 +1,15 @@
+#!/usr/bin/ruby
+require 'asciidoctor'
+
+input = Asciidoctor.load_file(ARGV[0])
+File.open(ARGV[1], "w") { |out|
+  out.puts "---"
+  out.puts "layout: page"
+  out.puts ["title: ", input.doctitle].join
+  out.puts ["priority: ", input.attributes["priority"]].join
+  out.puts ["toclevels: ", input.attributes["toclevels"]].join
+  if input.attributes["toc"] != nil || input.attributes["toc2"] != nil
+    out.puts "toc: true"
+  end
+  out.puts "---"
+}
diff --git a/kythe/web/site/site_docs.bzl b/kythe/web/site/site_docs.bzl
new file mode 100644
index 000000000..b31be634c
--- /dev/null
+++ b/kythe/web/site/site_docs.bzl
@@ -0,0 +1,161 @@
+load("//kythe/docs:asciidoc.bzl", "AsciidocInfo")
+load("@bazel_skylib//lib:paths.bzl", "paths")
+load("@bazel_skylib//lib:shell.bzl", "shell")
+
+_AsciidocHeaderInfo = provider(
+    fields = {"header": "File with the asciidoc header."},
+)
+
+_SiteDocsInfo = provider()
+
+def _header_impl(target, ctx):
+    src = ctx.rule.file.src
+    header = ctx.actions.declare_file(paths.replace_extension(src.path, "head." + src.extension))
+    ctx.actions.run(
+        inputs = [src],
+        outputs = [header],
+        tools = [ctx.executable._docheader],
+        executable = ctx.executable._docheader,
+        arguments = [src.path, header.path],
+        mnemonic = "JekyllHeader",
+    )
+
+    return [
+        _AsciidocHeaderInfo(header = header),
+    ]
+
+_header_aspect = aspect(
+    implementation = _header_impl,
+    attrs = {
+        "_docheader": attr.label(
+            default = Label("//kythe/web/site:doc_header"),
+            executable = True,
+            cfg = "exec",
+        ),
+    },
+)
+
+def _impl(ctx):
+    outdir = ctx.actions.declare_directory(ctx.label.name)
+    commands = []
+    inputs = []
+    for src in ctx.attr.srcs:
+        header = src[_AsciidocHeaderInfo].header
+        html = src[AsciidocInfo].primary_output_path
+        resources = src[AsciidocInfo].resource_dir
+        inputs += [resources, header]
+        commands += [
+            # Copy only the files from the resource dir, omitting the html file itself
+            # or we will get subsequent permissions problems.
+            "find {resource_dir} -mindepth 1 -maxdepth 1 -depth -not -path {html} -exec cp -L -r {{}} {outdir} \\;".format(
+                resource_dir = shell.quote(resources.path),
+                outdir = shell.quote(outdir.path),
+                html = shell.quote(paths.join(resources.path, html)),
+            ),
+            "cat {header} {html} > {output}".format(
+                header = shell.quote(header.path),
+                html = shell.quote(paths.join(resources.path, html)),
+                output = shell.quote(paths.join(outdir.path, html)),
+            ),
+        ]
+    for dep in ctx.attr.deps:
+        files = dep.files.to_list()
+        inputs += files
+        commands += [
+            "cp -L -r {file} {outdir}".format(
+                file = shell.quote(file.path),
+                outdir = shell.quote(outdir.path),
+            )
+            for file in files
+        ]
+
+    commands.append("pushd {outdir}".format(outdir = shell.quote(outdir.path)))
+    for src, dest in ctx.attr.rename_files.items():
+        commands.append("mv {src} {dest}".format(
+            src = shell.quote(src),
+            dest = shell.quote(dest),
+        ))
+    commands.append("popd")
+
+    ctx.actions.run_shell(
+        mnemonic = "BuildDocs",
+        inputs = inputs,
+        outputs = [outdir],
+        command = "\n".join([
+            "set -e",
+            "mkdir -p {outdir}".format(outdir = shell.quote(outdir.path)),
+        ] + commands),
+    )
+
+    return [
+        # Only include the root directory in our declared outputs.
+        # This ensure that downstream rules don't see files listed twice if the expand tree artifacts.
+        DefaultInfo(files = depset([outdir])),
+        _SiteDocsInfo(),
+    ]
+
+site_docs = rule(
+    implementation = _impl,
+    attrs = {
+        "srcs": attr.label_list(
+            aspects = [_header_aspect],
+            providers = [AsciidocInfo],
+        ),
+        "deps": attr.label_list(
+            providers = [_SiteDocsInfo],
+        ),
+        "rename_files": attr.string_dict(),
+    },
+)
+
+def _package_path(file):
+    pkgroot = paths.join(file.root.path, file.owner.package)
+    return paths.relativize(file.path, pkgroot)
+
+def _jekyll_impl(ctx):
+    input_root = ctx.label.name + ".staging.d"
+    symlinks = []
+    for src in ctx.files.srcs:
+        symlink = ctx.actions.declare_file(paths.join(input_root, _package_path(src)))
+        ctx.actions.symlink(output = symlink, target_file = src)
+        symlinks.append(symlink)
+    input_dir = paths.join(symlinks[0].root.path, symlinks[0].owner.package, input_root)
+
+    outdir = ctx.outputs.out
+    if not outdir:
+        outdir = ctx.actions.declare_directory("_site")
+
+    # Dummy command for now.
+    args = ctx.actions.args()
+    args.add("build")
+    args.add("-s", input_dir)
+    args.add("-d", outdir.path)
+    ctx.actions.run(
+        outputs = [outdir],
+        inputs = symlinks,
+        arguments = [args],
+        executable = ctx.executable._jekyll,
+        # TODO(shahms): We don't currently have a Ruby toolchain in the RBE environment.
+        execution_requirements = {"local": ""},
+        mnemonic = "JekyllBuild",
+    )
+
+    return [
+        DefaultInfo(files = depset([outdir])),
+    ]
+
+jekyll_build = rule(
+    implementation = _jekyll_impl,
+    attrs = {
+        "out": attr.output(),
+        "srcs": attr.label_list(
+            allow_files = True,
+        ),
+        "_jekyll": attr.label(
+            default = "@website_bundle//:bin/jekyll",
+            executable = True,
+            cfg = "exec",
+            allow_files = True,
+        ),
+    },
+)
diff --git a/kythe/web/site/sync_docs.sh b/kythe/web/site/sync_docs.sh
deleted file mode 100755
index fb1a14581..000000000
--- a/kythe/web/site/sync_docs.sh
+++ /dev/null
@@ -1,92 +0,0 @@
-#!/bin/bash -e
-#
-# Copyright 2014 The Kythe Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-#
-# Script to sync Kythe documentation into /_docs/ for kythe.io.
-#
-# Usage: ./sync_docs.sh
-
-# Make sure the user has installed the asciidoc gem, otherwise the website will
-# generate successfully but it will be missing tocs, titles, and other
-# attributes.
-if ! gem list -i asciidoctor &> /dev/null; then
-  echo "You don't have the asciidoctor gem installed."
-  echo "Please run 'gem install --user asciidoctor' before executing this script."
-  exit 1
-fi
-
-export SHELL=/bin/bash
-
-DIR="$(readlink -e "$(dirname "$0")")"
-cd "$DIR/../../.."
-
-# Use the project's bazelrc so we pick up the default options that Kythe needs
-# to build correctly.
-bazel --bazelrc=.bazelrc build //kythe/docs/... \
-  //kythe/docs:schema-overview \
-  //kythe/docs/schema \
-  //kythe/docs/schema:callgraph \
-  //kythe/docs/schema:verifierstyle \
-  //kythe/docs/schema:writing-an-indexer \
-  //kythe/docs/schema:indexing-generated-code \
-  //kythe/docs/schema:marked-source
-# Copy the zipped asciidoc outputs into the staging directory, unpack the
-# archives, then remove them. We do this to ensure the output retains the
-# directory structure of the source tree.
-rsync -Lr --chmod=a+w --delete "bazel-bin/kythe/docs/" "$DIR"/_docs
-find "$DIR"/_docs -type f -name '*.zip' -execdir unzip -q {} ';' -delete
-
-# shellcheck disable=SC2016
-readarray -t DOCS < <(bazel query 'kind("source file", deps(//kythe/docs/..., 1))' |
-  grep -E '\.(txt|adoc|ad)$' |
-  parallel --gnu -L1 'x() { file="$(tr : / <<<"$1")"; echo ${file#//kythe/docs/}; }; x')
-
-asciidoc_query() {
-  bundle exec ruby -r asciidoctor -e 'puts Asciidoctor.load_file(ARGV[0]).'"$2" "$1" 2> /dev/null
-}
-
-asciidoc_attribute_presence() {
-  [[ "$(asciidoc_query "$1" "attributes[\"$2\"] != nil")" == "true" ]]
-}
-
-doc_header() {
-  echo "---
-layout: page
-title: $(asciidoc_query "$1" doctitle)
-priority: $(asciidoc_query "$1" 'attributes["priority"]')
-toclevels: $(asciidoc_query "$1" 'attributes["toclevels"]')"
-  if asciidoc_attribute_presence "$1" toc || asciidoc_attribute_presence "$1" toc2; then
-    echo "toc: true"
-  fi
-  echo "---"
-}
-
-TMP="$(mktemp)"
-trap 'rm -rf "$TMP"' EXIT ERR INT
-
-cd "$DIR"
-for doc in "${DOCS[@]}"; do
-  html=${doc%%.*}.html
-  echo "Processing $html"
-  abs_path="../../../kythe/docs/$doc"
-  cp "_docs/$html" "$TMP"
-  {
-    doc_header "$abs_path"
-    cat "$TMP"
-  } > "_docs/$html"
-done
-
-mv _docs/schema/{schema,index}.html
diff --git a/setup.bzl b/setup.bzl
index 36c71aacf..a864602f5 100644
--- a/setup.bzl
+++ b/setup.bzl
@@ -116,3 +116,17 @@ def kythe_rule_repositories():
             "https://github.com/bazelbuild/rules_rust/archive/997a8a19a6893026a94bee5d1a775c2ba3d75a42.tar.gz",
         ],
     )
+
+    maybe(
+        http_archive,
+        name = "bazelruby_rules_ruby",
+        strip_prefix = "rules_ruby-0.4.1",
+        sha256 = "abfc2758cc379e0ff0eb9824e3b507c1633d4c8f99f24735aef63c7180be50f0",
+        urls = [
+            "https://github.com/bazelruby/rules_ruby/archive/v0.4.1.zip",
+        ],
+        patches = [
+            "//third_party:rules_ruby_allow_empty.patch",
+        ],
+        patch_args = ["-p1"],
+    )
diff --git a/third_party/rules_ruby_allow_empty.patch b/third_party/rules_ruby_allow_empty.patch
new file mode 100644
index 000000000..57be8cfa5
--- /dev/null
+++ b/third_party/rules_ruby_allow_empty.patch
@@ -0,0 +1,48 @@
+diff --git a/ruby/private/bundle/create_bundle_build_file.rb b/ruby/private/bundle/create_bundle_build_file.rb
+index b283b40..10dce4e 100755
+--- a/ruby/private/bundle/create_bundle_build_file.rb
++++ b/ruby/private/bundle/create_bundle_build_file.rb
+@@ -38,6 +38,7 @@ GEM_TEMPLATE = <<~GEM_TEMPLATE
+         {gem_binaries}
+       ],
+       exclude = {exclude},
++      allow_empty = True,
+     ),
+     deps = {deps},
+     includes = ["lib/ruby/{ruby_version}/gems/{name}-{version}/lib"],
+@@ -47,7 +48,7 @@ GEM_TEMPLATE
+ ALL_GEMS = <<~ALL_GEMS
+   ruby_library(
+     name = "gems",
+-    srcs = glob([{bundle_lib_files}]) + glob(["bin/*"]),
++    srcs = glob([{bundle_lib_files}], allow_empty = True) + glob(["bin/*"]),
+     includes = {bundle_lib_paths},
+   )
+ 
+diff --git a/ruby/private/toolchains/BUILD.runtime.tpl b/ruby/private/toolchains/BUILD.runtime.tpl
+index fbb3e64..18fb412 100644
+--- a/ruby/private/toolchains/BUILD.runtime.tpl
++++ b/ruby/private/toolchains/BUILD.runtime.tpl
+@@ -24,14 +24,20 @@ sh_binary(
+ 
+ cc_import(
+     name = "libruby",
+-    hdrs = glob({hdrs}),
++    hdrs = glob(
++        include = {hdrs},
++        allow_empty = True,
++    ),
+     shared_library = {shared_library},
+     static_library = {static_library},
+ )
+ 
+ cc_library(
+     name = "headers",
+-    hdrs = glob({hdrs}),
++    hdrs = glob(
++        include = {hdrs},
++        allow_empty = True,
++    ),
+     includes = {includes},
+ )
+ 
diff --git a/tools/platforms/BUILD b/tools/platforms/BUILD
index ffd0fb0cd..1c7af2cba 100644
--- a/tools/platforms/BUILD
+++ b/tools/platforms/BUILD
@@ -1,1 +1,6 @@
 package(default_visibility = ["//visibility:public"])
+
+platform(
+    name = "rbe",
+    parents = ["@rbe_default//config:platform"],
+)
diff --git a/tools/platforms/Dockerfile b/tools/platforms/Dockerfile
index 1bce96a50..0c206d9d7 100644
--- a/tools/platforms/Dockerfile
+++ b/tools/platforms/Dockerfile
@@ -13,12 +13,12 @@
 # limitations under the License.
 
 # docker build -t gcr.io/kythe-repo/kythe-builder .
-FROM gcr.io/cloud-marketplace/google/rbe-ubuntu18-04@sha256:5703843282226b2f7db86afc0b1535f46d411fa72c1d540e1bd0f8064adb0bdb
+FROM gcr.io/cloud-marketplace/google/rbe-ubuntu18-04@sha256:48b67b41118dbcdfc265e7335f454fbefa62681ab8d47200971fc7a52fb32054
 
 RUN apt-get update && \
     apt-get install -y \
       # Kythe C++ dependencies
       uuid-dev flex bison g++ \
       # Kythe misc dependencies
-      asciidoc source-highlight graphviz && \
+      asciidoc ruby-dev source-highlight graphviz && \
     apt-get clean
diff --git a/tools/platforms/configs/default_toolchain_config_spec_name/bazel_4.0.0/cc/BUILD b/tools/platforms/configs/default_toolchain_config_spec_name/bazel_4.0.0/cc/BUILD
index b552706a1..62974e601 100755
--- a/tools/platforms/configs/default_toolchain_config_spec_name/bazel_4.0.0/cc/BUILD
+++ b/tools/platforms/configs/default_toolchain_config_spec_name/bazel_4.0.0/cc/BUILD
@@ -14,12 +14,12 @@
 
 # This becomes the BUILD file for @local_config_cc// under non-BSD unixes.
 
+package(default_visibility = ["//visibility:public"])
+
 load(":cc_toolchain_config.bzl", "cc_toolchain_config")
 load(":armeabi_cc_toolchain_config.bzl", "armeabi_cc_toolchain_config")
 load("@rules_cc//cc:defs.bzl", "cc_toolchain", "cc_toolchain_suite")
 
-package(default_visibility = ["//visibility:public"])
-
 licenses(["notice"])  # Apache 2.0
 
 cc_library(
@@ -38,10 +38,7 @@ filegroup(
 
 filegroup(
     name = "compiler_deps",
-    srcs = glob(
-        ["extra_tools/**"],
-        allow_empty = True,
-    ) + [":builtin_include_directory_paths"],
+    srcs = glob(["extra_tools/**"], allow_empty = True) + [":builtin_include_directory_paths"],
 )
 
 # This is the entry point for --crosstool_top.  Toolchains are found
@@ -59,74 +56,39 @@ cc_toolchain_suite(
 
 cc_toolchain(
     name = "cc-compiler-k8",
+    toolchain_identifier = "linux_gnu_x86",
+    toolchain_config = ":linux_gnu_x86",
     all_files = ":compiler_deps",
     ar_files = ":compiler_deps",
     as_files = ":compiler_deps",
     compiler_files = ":compiler_deps",
     dwp_files = ":empty",
     linker_files = ":compiler_deps",
-    module_map = ":module.modulemap",
     objcopy_files = ":empty",
     strip_files = ":empty",
     supports_param_files = 1,
-    toolchain_config = ":linux_gnu_x86",
-    toolchain_identifier = "linux_gnu_x86",
+    module_map = ":module.modulemap",
 )
 
 cc_toolchain_config(
     name = "linux_gnu_x86",
-    abi_libc_version = "glibc_2.19",
-    abi_version = "clang",
-    compile_flags = [
-        "-U_FORTIFY_SOURCE",
-        "-fstack-protector",
-        "-Wall",
-        "-Wthread-safety",
-        "-Wself-assign",
-        "-fcolor-diagnostics",
-        "-fno-omit-frame-pointer",
-    ],
-    compiler = "clang",
-    coverage_compile_flags = ["--coverage"],
-    coverage_link_flags = ["--coverage"],
     cpu = "k8",
-    cxx_builtin_include_directories = [
-        "/usr/local/include",
-        "/usr/local/lib/clang/10.0.0/include",
-        "/usr/include/x86_64-linux-gnu",
-        "/usr/include",
-        "/usr/local/lib/clang/10.0.0/share",
-        "/usr/include/c++/7.4.0",
-        "/usr/include/x86_64-linux-gnu/c++/7.4.0",
-        "/usr/include/c++/7.4.0/backward",
-    ],
-    cxx_flags = ["-std=c++0x"],
-    dbg_compile_flags = ["-g"],
+    compiler = "clang",
+    toolchain_identifier = "linux_gnu_x86",
     host_system_name = "i686-unknown-linux-gnu",
-    link_flags = [
-        "-fuse-ld=/usr/bin/ld.gold",
-        "-Wl,-no-as-needed",
-        "-Wl,-z,relro,-z,now",
-        "-B/usr/local/bin",
-    ],
-    link_libs = [
-        "-lstdc++",
-        "-lm",
-    ],
-    opt_compile_flags = [
-        "-g0",
-        "-O2",
-        "-D_FORTIFY_SOURCE=1",
-        "-DNDEBUG",
-        "-ffunction-sections",
-        "-fdata-sections",
-    ],
-    opt_link_flags = ["-Wl,--gc-sections"],
-    supports_start_end_lib = True,
-    target_libc = "glibc_2.19",
     target_system_name = "x86_64-unknown-linux-gnu",
-    tool_paths = {
-        "ar": "/usr/bin/ar",
+    target_libc = "glibc_2.19",
+    abi_version = "clang",
+    abi_libc_version = "glibc_2.19",
+    cxx_builtin_include_directories = ["/usr/local/include",
+    "/usr/local/lib/clang/10.0.0/include",
+    "/usr/include/x86_64-linux-gnu",
+    "/usr/include",
+    "/usr/local/lib/clang/10.0.0/share",
+    "/usr/include/c++/7.4.0",
+    "/usr/include/x86_64-linux-gnu/c++/7.4.0",
+    "/usr/include/c++/7.4.0/backward"],
+    tool_paths = {"ar": "/usr/bin/ar",
         "ld": "/usr/bin/ld",
         "llvm-cov": "/usr/local/bin/llvm-cov",
         "cpp": "/usr/bin/cpp",
@@ -136,21 +98,44 @@ cc_toolchain_config(
         "nm": "/usr/bin/nm",
         "objcopy": "/usr/bin/objcopy",
         "objdump": "/usr/bin/objdump",
-        "strip": "/usr/bin/strip",
-    },
-    toolchain_identifier = "linux_gnu_x86",
-    unfiltered_compile_flags = [
-        "-no-canonical-prefixes",
-        "-Wno-builtin-macro-redefined",
-        "-D__DATE__=\"redacted\"",
-        "-D__TIMESTAMP__=\"redacted\"",
-        "-D__TIME__=\"redacted\"",
-    ],
+        "strip": "/usr/bin/strip"},
+    compile_flags = ["-U_FORTIFY_SOURCE",
+    "-fstack-protector",
+    "-Wall",
+    "-Wthread-safety",
+    "-Wself-assign",
+    "-fcolor-diagnostics",
+    "-fno-omit-frame-pointer"],
+    opt_compile_flags = ["-g0",
+    "-O2",
+    "-D_FORTIFY_SOURCE=1",
+    "-DNDEBUG",
+    "-ffunction-sections",
+    "-fdata-sections"],
+    dbg_compile_flags = ["-g"],
+    cxx_flags = ["-std=c++0x"],
+    link_flags = ["-fuse-ld=/usr/bin/ld.gold",
+    "-Wl,-no-as-needed",
+    "-Wl,-z,relro,-z,now",
+    "-B/usr/local/bin"],
+    link_libs = ["-lstdc++",
+    "-lm"],
+    opt_link_flags = ["-Wl,--gc-sections"],
+    unfiltered_compile_flags = ["-no-canonical-prefixes",
+    "-Wno-builtin-macro-redefined",
+    "-D__DATE__=\"redacted\"",
+    "-D__TIMESTAMP__=\"redacted\"",
+    "-D__TIME__=\"redacted\""],
+    coverage_compile_flags = ["--coverage"],
+    coverage_link_flags = ["--coverage"],
+    supports_start_end_lib = True,
 )
 
 # Android tooling requires a default toolchain for the armeabi-v7a cpu.
 cc_toolchain(
     name = "cc-compiler-armeabi-v7a",
+    toolchain_identifier = "stub_armeabi-v7a",
+    toolchain_config = ":stub_armeabi-v7a",
     all_files = ":empty",
     ar_files = ":empty",
     as_files = ":empty",
@@ -160,8 +145,6 @@ cc_toolchain(
     objcopy_files = ":empty",
     strip_files = ":empty",
     supports_param_files = 1,
-    toolchain_config = ":stub_armeabi-v7a",
-    toolchain_identifier = "stub_armeabi-v7a",
 )
 
 armeabi_cc_toolchain_config(name = "stub_armeabi-v7a")
diff --git a/tools/platforms/configs/default_toolchain_config_spec_name/bazel_4.0.0/config/BUILD b/tools/platforms/configs/default_toolchain_config_spec_name/bazel_4.0.0/config/BUILD
index 69654f8ee..9040b543b 100644
--- a/tools/platforms/configs/default_toolchain_config_spec_name/bazel_4.0.0/config/BUILD
+++ b/tools/platforms/configs/default_toolchain_config_spec_name/bazel_4.0.0/config/BUILD
@@ -40,8 +40,5 @@ platform(
         "@bazel_tools//platforms:x86_64",
         "@bazel_tools//tools/cpp:clang",
     ],
-    exec_properties = {
-        "container-image": "docker://gcr.io/kythe-repo/kythe-builder@sha256:affd4b4d05144cb524db0d4f4639db74942b53ff499a1dfce23ce0592654e259",
-        "OSFamily": "Linux",
-    },
+    exec_properties = {"container-image": "docker://gcr.io/kythe-repo/kythe-builder@sha256:affd4b4d05144cb524db0d4f4639db74942b53ff499a1dfce23ce0592654e259", "OSFamily": "Linux"},
 )


Test patch:
diff --git a/kythe/web/site/check_links_test.sh b/kythe/web/site/check_links_test.sh
new file mode 100755
index 000000000..2e74c26a8
--- /dev/null
+++ b/kythe/web/site/check_links_test.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+set -e
+JEKYLL_BIN="$(realpath "$JEKYLL_BIN")"
+cd "kythe/web/site"
+"${JEKYLL_BIN}" serve --skip-initial-build --no-watch &
+server_pid=$!
+trap 'kill -INT "$server_pid"' EXIT ERR INT
+
+for ((i=0; i < 10; i++)); do
+  if curl -s localhost:4000 >/dev/null; then
+    break
+  fi
+  echo 'Waiting for server...' >&2
+  sleep 1s
+done
+
+set -o pipefail
+wget --spider -nv -e robots=off -r -p http://localhost:4000 2>&1 | \
+  grep -v -e 'unlink: ' -e ' URL:'
+
+# If grep finds a match, exit with a failure.
+if [[ "${PIPESTATUS[1]}" -ne 1 ]]; then
+  exit 1
+fi