1 files changed, 192 insertions, 0 deletions
diff --git a/src/ci/docker/scripts/android-sdk-manager.py b/src/ci/docker/scripts/android-sdk-manager.py
new file mode 100755
index 000000000..c9e2961f6
--- /dev/null
+++ b/src/ci/docker/scripts/android-sdk-manager.py
@@ -0,0 +1,192 @@
+#!/usr/bin/env python3
+# Simpler reimplementation of Android's sdkmanager
+# Extra features of this implementation are pinning and mirroring
+
+# These URLs are the Google repositories containing the list of available
+# packages and their versions. The list has been generated by listing the URLs
+# fetched while executing `tools/bin/sdkmanager --list`
+BASE_REPOSITORY = "https://dl.google.com/android/repository/"
+REPOSITORIES = [
+    "sys-img/android/sys-img2-1.xml",
+    "sys-img/android-wear/sys-img2-1.xml",
+    "sys-img/android-wear-cn/sys-img2-1.xml",
+    "sys-img/android-tv/sys-img2-1.xml",
+    "sys-img/google_apis/sys-img2-1.xml",
+    "sys-img/google_apis_playstore/sys-img2-1.xml",
+    "addon2-1.xml",
+    "glass/addon2-1.xml",
+    "extras/intel/addon2-1.xml",
+    "repository2-1.xml",
+]
+
+# Available hosts: linux, macosx and windows
+HOST_OS = "linux"
+
+# Mirroring options
+MIRROR_BUCKET = "rust-lang-ci-mirrors"
+MIRROR_BUCKET_REGION = "us-west-1"
+MIRROR_BASE_DIR = "rustc/android/"
+
+import argparse
+import hashlib
+import os
+import subprocess
+import sys
+import tempfile
+import urllib.request
+import xml.etree.ElementTree as ET
+
+class Package:
+    def __init__(self, path, url, sha1, deps=None):
+        if deps is None:
+            deps = []
+        self.path = path.strip()
+        self.url = url.strip()
+        self.sha1 = sha1.strip()
+        self.deps = deps
+
+    def download(self, base_url):
+        _, file = tempfile.mkstemp()
+        url = base_url + self.url
+        subprocess.run(["curl", "-o", file, url], check=True)
+        # Ensure there are no hash mismatches
+        with open(file, "rb") as f:
+            sha1 = hashlib.sha1(f.read()).hexdigest()
+            if sha1 != self.sha1:
+                raise RuntimeError(
+                    "hash mismatch for package " + self.path + ": " +
+                    sha1 + " vs " + self.sha1 + " (known good)"
+                )
+        return file
+
+    def __repr__(self):
+        return "<Package "+self.path+" at "+self.url+" (sha1="+self.sha1+")"
+
+def fetch_url(url):
+    page = urllib.request.urlopen(url)
+    return page.read()
+
+def fetch_repository(base, repo_url):
+    packages = {}
+    root = ET.fromstring(fetch_url(base + repo_url))
+    for package in root:
+        if package.tag != "remotePackage":
+            continue
+        path = package.attrib["path"]
+
+        for archive in package.find("archives"):
+            host_os = archive.find("host-os")
+            if host_os is not None and host_os.text != HOST_OS:
+                continue
+            complete = archive.find("complete")
+            url = os.path.join(os.path.dirname(repo_url), complete.find("url").text)
+            sha1 = complete.find("checksum").text
+
+            deps = []
+            dependencies = package.find("dependencies")
+            if dependencies is not None:
+                for dep in dependencies:
+                    deps.append(dep.attrib["path"])
+
+            packages[path] = Package(path, url, sha1, deps)
+            break
+
+    return packages
+
+def fetch_repositories():
+    packages = {}
+    for repo in REPOSITORIES:
+        packages.update(fetch_repository(BASE_REPOSITORY, repo))
+    return packages
+
+class Lockfile:
+    def __init__(self, path):
+        self.path = path
+        self.packages = {}
+        if os.path.exists(path):
+            with open(path) as f:
+                for line in f:
+                    path, url, sha1 = line.split(" ")
+                    self.packages[path] = Package(path, url, sha1)
+
+    def add(self, packages, name, *, update=True):
+        if name not in packages:
+            raise NameError("package not found: " + name)
+        if not update and name in self.packages:
+            return
+        self.packages[name] = packages[name]
+        for dep in packages[name].deps:
+            self.add(packages, dep, update=False)
+
+    def save(self):
+        packages = list(sorted(self.packages.values(), key=lambda p: p.path))
+        with open(self.path, "w") as f:
+            for package in packages:
+                f.write(package.path + " " + package.url + " " + package.sha1 + "\n")
+
+def cli_add_to_lockfile(args):
+    lockfile = Lockfile(args.lockfile)
+    packages = fetch_repositories()
+    for package in args.packages:
+        lockfile.add(packages, package)
+    lockfile.save()
+
+def cli_update_mirror(args):
+    lockfile = Lockfile(args.lockfile)
+    for package in lockfile.packages.values():
+        path = package.download(BASE_REPOSITORY)
+        subprocess.run([
+            "aws", "s3", "mv", path,
+            "s3://" + MIRROR_BUCKET + "/" + MIRROR_BASE_DIR + package.url,
+            "--profile=" + args.awscli_profile,
+        ], check=True)
+
+def cli_install(args):
+    lockfile = Lockfile(args.lockfile)
+    for package in lockfile.packages.values():
+        # Download the file from the mirror into a temp file
+        url = "https://" + MIRROR_BUCKET + ".s3-" + MIRROR_BUCKET_REGION + \
+              ".amazonaws.com/" + MIRROR_BASE_DIR
+        downloaded = package.download(url)
+        # Extract the file in a temporary directory
+        extract_dir = tempfile.mkdtemp()
+        subprocess.run([
+            "unzip", "-q", downloaded, "-d", extract_dir,
+        ], check=True)
+        # Figure out the prefix used in the zip
+        subdirs = [d for d in os.listdir(extract_dir) if not d.startswith(".")]
+        if len(subdirs) != 1:
+            raise RuntimeError("extracted directory contains more than one dir")
+        # Move the extracted files in the proper directory
+        dest = os.path.join(args.dest, package.path.replace(";", "/"))
+        os.makedirs("/".join(dest.split("/")[:-1]), exist_ok=True)
+        os.rename(os.path.join(extract_dir, subdirs[0]), dest)
+        os.unlink(downloaded)
+
+def cli():
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers()
+
+    add_to_lockfile = subparsers.add_parser("add-to-lockfile")
+    add_to_lockfile.add_argument("lockfile")
+    add_to_lockfile.add_argument("packages", nargs="+")
+    add_to_lockfile.set_defaults(func=cli_add_to_lockfile)
+
+    update_mirror = subparsers.add_parser("update-mirror")
+    update_mirror.add_argument("lockfile")
+    update_mirror.add_argument("--awscli-profile", default="default")
+    update_mirror.set_defaults(func=cli_update_mirror)
+
+    install = subparsers.add_parser("install")
+    install.add_argument("lockfile")
+    install.add_argument("dest")
+    install.set_defaults(func=cli_install)
+
+    args = parser.parse_args()
+    if not hasattr(args, "func"):
+        print("error: a subcommand is required (see --help)")
+        exit(1)
+    args.func(args)
+
+if __name__ == "__main__":
+    cli()