Change:
We already had at least two different CI issues caused by the Android SDK's sdkmanager: a new license was and sdkmanager failed to accept it with the old command the emulator was updated to a broken (at least for us) version, causing all builds to fail The root cause of those issues is that sdkmanager doesn't support any kind of pinning (as far as I can tell), and any change Google makes can break our CI. Should we develop a pinning mechanism and use it for all the Android SDK components? cc

Discussion:
RUN . /scripts/android-ndk.sh &&  download_and_make_toolchain android-ndk-r15c-linux-x86_64.zip arm 14 <del> # Note: # Do not upgrade to `openjdk-9-jre-headless`, as it will cause certificate error # when installing the Android SDK (see PR #45193). This is unfortunate, but # every search result suggested either disabling HTTPS or replacing JDK 9 by # JDK 8 as the solution (e.g. https://stackoverflow.com/q/41421340). :| </del> RUN dpkg --add-architecture i386 &&  apt-get update &&  apt-get install -y --no-install-recommends  libgl1-mesa-glx  libpulse0  libstdc++6:i386  <del> openjdk-8-jre-headless  tzdata </del> <ins> openjdk-9-jre-headless  tzdata  wget  python3 </ins> COPY scripts/android-sdk.sh /scripts/ <del> RUN . /scripts/android-sdk.sh &&  download_and_create_avd 4333796 armeabi-v7a 18 5264690 </del> <ins> COPY scripts/android-sdk-manager.py /scripts/ COPY arm-android/android-sdk.lock /android/sdk/android-sdk.lock RUN /scripts/android-sdk.sh </ins> ENV PATH=$PATH:/android/sdk/emulator ENV PATH=$PATH:/android/sdk/tools

<ins> emulator emulator-linux-5264690.zip 48c1cda2bdf3095d9d9d5c010fbfb3d6d673e3ea patcher;v4 3534162-studio.sdk-patcher.zip 046699c5e2716ae11d77e0bad814f7f33fab261e platform-tools platform-tools_r28.0.2-linux.zip 46a4c02a9b8e4e2121eddf6025da3c979bf02e28 platforms;android-18 android-18_r03.zip e6b09b3505754cbbeb4a5622008b907262ee91cb system-images;android-18;default;armeabi-v7a sys-img/android/armeabi-v7a-18_r05.zip 580b583720f7de671040d5917c8c9db0c7aa03fd tools sdk-tools-linux-4333796.zip 8c7c28554a32318461802c1291d76fccfafde054 </ins>

<ins> #!/usr/bin/env python3 # Simpler reimplementation of Android's sdkmanager # Extra features of this implementation are pinning and mirroring # These URLs are the Google repositories containing the list of available # packages and their versions. The list has been generated by listing the URLs # fetched while executing `tools/bin/sdkmanager --list` BASE_REPOSITORY = "https://dl.google.com/android/repository/" REPOSITORIES = [ "sys-img/android/sys-img2-1.xml", "sys-img/android-wear/sys-img2-1.xml", "sys-img/android-wear-cn/sys-img2-1.xml", "sys-img/android-tv/sys-img2-1.xml", "sys-img/google_apis/sys-img2-1.xml", "sys-img/google_apis_playstore/sys-img2-1.xml", "addon2-1.xml", "glass/addon2-1.xml", "extras/intel/addon2-1.xml", "repository2-1.xml", ] # Available hosts: linux, macosx and windows HOST_OS = "linux" # Mirroring options MIRROR_BUCKET = "rust-lang-ci2" MIRROR_BASE_DIR = "rust-ci-mirror/android/" import argparse import hashlib import os import subprocess import sys import tempfile import urllib.request import xml.etree.ElementTree as ET class Package: def __init__(self, path, url, sha1, deps=None): if deps is None: deps = [] self.path = path.strip() self.url = url.strip() self.sha1 = sha1.strip() self.deps = deps def download(self, base_url): _, file = tempfile.mkstemp() url = base_url + self.url subprocess.run(["curl", "-o", file, url], check=True) # Ensure there are no hash mismatches with open(file, "rb") as f: sha1 = hashlib.sha1(f.read()).hexdigest() if sha1 != self.sha1: raise RuntimeError( "hash mismatch for package " + self.path + ": " + sha1 + " vs " + self.sha1 + " (known good)" ) return file def __repr__(self): return "<Package "+self.path+" at "+self.url+" (sha1="+self.sha1+")" def fetch_url(url): page = urllib.request.urlopen(url) return page.read() def fetch_repository(base, repo_url): packages = {} root = ET.fromstring(fetch_url(base + repo_url)) for package in root: if package.tag != "remotePackage": continue path = package.attrib["path"] for archive in package.find("archives"): host_os = archive.find("host-os") if host_os is not None and host_os.text != HOST_OS: continue complete = archive.find("complete") url = os.path.join(os.path.dirname(repo_url), complete.find("url").text) sha1 = complete.find("checksum").text deps = [] dependencies = package.find("dependencies") if dependencies is not None: for dep in dependencies: deps.append(dep.attrib["path"]) packages[path] = Package(path, url, sha1, deps) break return packages def fetch_repositories(): packages = {} for repo in REPOSITORIES: packages.update(fetch_repository(BASE_REPOSITORY, repo)) return packages class Lockfile: def __init__(self, path): self.path = path self.packages = {} if os.path.exists(path): with open(path) as f: for line in f: path, url, sha1 = line.split(" ") self.packages[path] = Package(path, url, sha1) def add(self, packages, name, *, update=True): if name not in packages: raise NameError("package not found: " + name) if not update and name in self.packages: return self.packages[name] = packages[name] for dep in packages[name].deps: self.add(packages, dep, update=False) def save(self): packages = list(sorted(self.packages.values(), key=lambda p: p.path)) with open(self.path, "w") as f: for package in packages: f.write(package.path + " " + package.url + " " + package.sha1 + "n") def cli_add_to_lockfile(args): lockfile = Lockfile(args.lockfile) packages = fetch_repositories() for package in args.packages: lockfile.add(packages, package) lockfile.save() def cli_update_mirror(args): lockfile = Lockfile(args.lockfile) for package in lockfile.packages.values(): path = package.download(BASE_REPOSITORY) subprocess.run([ "aws", "s3", "mv", path, "s3://" + MIRROR_BUCKET + "/" + MIRROR_BASE_DIR + package.url, "--profile=" + args.awscli_profile, ], check=True) def cli_install(args): lockfile = Lockfile(args.lockfile) for package in lockfile.packages.values(): # Download the file from the mirror into a temp file url = "https://" + MIRROR_BUCKET + ".s3.amazonaws.com/" + MIRROR_BASE_DIR downloaded = package.download(url) # Extract the file in a temporary directory extract_dir = tempfile.mkdtemp() subprocess.run([ "unzip", "-q", downloaded, "-d", extract_dir, ], check=True) # Figure out the prefix used in the zip subdirs = [d for d in os.listdir(extract_dir) if not d.startswith(".")] if len(subdirs) != 1: raise RuntimeError("extracted directory contains more than one dir") # Move the extracted files in the proper directory dest = os.path.join(args.dest, package.path.replace(";", "/")) os.makedirs("/".join(dest.split("/")[:-1]), exist_ok=True) os.rename(os.path.join(extract_dir, subdirs[0]), dest) os.unlink(downloaded) def cli(): parser = argparse.ArgumentParser() subparsers = parser.add_subparsers() add_to_lockfile = subparsers.add_parser("add-to-lockfile") add_to_lockfile.add_argument("lockfile") add_to_lockfile.add_argument("packages", nargs="+") add_to_lockfile.set_defaults(func=cli_add_to_lockfile) update_mirror = subparsers.add_parser("update-mirror") update_mirror.add_argument("lockfile") update_mirror.add_argument("--awscli-profile", default="default") update_mirror.set_defaults(func=cli_update_mirror) install = subparsers.add_parser("install") install.add_argument("lockfile") install.add_argument("dest") install.set_defaults(func=cli_install) args = parser.parse_args() if not hasattr(args, "func"): print("error: a subcommand is required (see --help)") exit(1) args.func(args) if __name__ == "__main__": cli() </ins>

export ANDROID_HOME=/android/sdk PATH=$PATH:"${ANDROID_HOME}/tools/bin" <ins> LOCKFILE="${ANDROID_HOME}/android-sdk.lock" </ins> <del> download_sdk() { mkdir -p /android curl -fo sdk.zip "https://dl.google.com/android/repository/sdk-tools-linux-$1.zip" unzip -q sdk.zip -d "$ANDROID_HOME" rm -f sdk.zip } download_sysimage() { abi=$1 api=$2 # See https://developer.android.com/studio/command-line/sdkmanager.html for # usage of `sdkmanager`. # # The output from sdkmanager is so noisy that it will occupy all of the 4 MB # log extremely quickly. Thus we must silence all output. yes | sdkmanager --licenses > /dev/null yes | sdkmanager platform-tools  "platforms;android-$api"  "system-images;android-$api;default;$abi" > /dev/null } download_emulator() { # Download a pinned version of the emulator since upgrades can cause issues curl -fo emulator.zip "https://dl.google.com/android/repository/emulator-linux-$1.zip" rm -rf "${ANDROID_HOME}/emulator" unzip -q emulator.zip -d "${ANDROID_HOME}" rm -f emulator.zip } create_avd() { abi=$1 api=$2 </del> <ins> # To add a new packages to the SDK or to update an existing one you need to # run the command: # #    android-sdk-manager.py add-to-lockfile $LOCKFILE <package-name> # # Then, after every lockfile update the mirror has to be synchronized as well: # #    android-sdk-manager.py update-mirror $LOCKFILE # /scripts/android-sdk-manager.py install "${LOCKFILE}" "${ANDROID_HOME}" </ins> <del> # See https://developer.android.com/studio/command-line/avdmanager.html for # usage of `avdmanager`. echo no | avdmanager create avd  -n "$abi-$api"  -k "system-images;android-$api;default;$abi" } </del> <ins> details=$(cat "${LOCKFILE}"  | grep system-images  | sed 's/^system-images;android-([0-9]+);default;([a-z0-9-]+) /1 2 /g') api="$(echo "${details}" | awk '{print($1)}')" abi="$(echo "${details}" | awk '{print($2)}')" </ins> <del> download_and_create_avd() { download_sdk $1 download_sysimage $2 $3 create_avd $2 $3 download_emulator $4 } </del> <ins> # See https://developer.android.com/studio/command-line/avdmanager.html for # usage of `avdmanager`. echo no | avdmanager create avd  -n "$abi-$api"  -k "system-images;android-$api;default;$abi" </ins> <del> # Usage: # #       download_and_create_avd 4333796 armeabi-v7a 18 5264690 # # 4333796 => #   SDK tool version. #   Copy from https://developer.android.com/studio/index.html#command-tools # armeabi-v7a => #   System image ABI # 18 => #   Android API Level (18 = Android 4.3 = Jelly Bean MR2) # 5264690 => #   Android Emulator version. #   Copy from the "build_id" in the `/android/sdk/emulator/emulator -version` output </del>