diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-17 12:02:58 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-17 12:02:58 +0000 |
commit | 698f8c2f01ea549d77d7dc3338a12e04c11057b9 (patch) | |
tree | 173a775858bd501c378080a10dca74132f05bc50 /src/ci/docker/scripts/android-sdk-manager.py | |
parent | Initial commit. (diff) | |
download | rustc-698f8c2f01ea549d77d7dc3338a12e04c11057b9.tar.xz rustc-698f8c2f01ea549d77d7dc3338a12e04c11057b9.zip |
Adding upstream version 1.64.0+dfsg1.upstream/1.64.0+dfsg1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/ci/docker/scripts/android-sdk-manager.py')
-rwxr-xr-x | src/ci/docker/scripts/android-sdk-manager.py | 192 |
1 files changed, 192 insertions, 0 deletions
diff --git a/src/ci/docker/scripts/android-sdk-manager.py b/src/ci/docker/scripts/android-sdk-manager.py new file mode 100755 index 000000000..c9e2961f6 --- /dev/null +++ b/src/ci/docker/scripts/android-sdk-manager.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +# Simpler reimplementation of Android's sdkmanager +# Extra features of this implementation are pinning and mirroring + +# These URLs are the Google repositories containing the list of available +# packages and their versions. The list has been generated by listing the URLs +# fetched while executing `tools/bin/sdkmanager --list` +BASE_REPOSITORY = "https://dl.google.com/android/repository/" +REPOSITORIES = [ + "sys-img/android/sys-img2-1.xml", + "sys-img/android-wear/sys-img2-1.xml", + "sys-img/android-wear-cn/sys-img2-1.xml", + "sys-img/android-tv/sys-img2-1.xml", + "sys-img/google_apis/sys-img2-1.xml", + "sys-img/google_apis_playstore/sys-img2-1.xml", + "addon2-1.xml", + "glass/addon2-1.xml", + "extras/intel/addon2-1.xml", + "repository2-1.xml", +] + +# Available hosts: linux, macosx and windows +HOST_OS = "linux" + +# Mirroring options +MIRROR_BUCKET = "rust-lang-ci-mirrors" +MIRROR_BUCKET_REGION = "us-west-1" +MIRROR_BASE_DIR = "rustc/android/" + +import argparse +import hashlib +import os +import subprocess +import sys +import tempfile +import urllib.request +import xml.etree.ElementTree as ET + +class Package: + def __init__(self, path, url, sha1, deps=None): + if deps is None: + deps = [] + self.path = path.strip() + self.url = url.strip() + self.sha1 = sha1.strip() + self.deps = deps + + def download(self, base_url): + _, file = tempfile.mkstemp() + url = base_url + self.url + subprocess.run(["curl", "-o", file, url], check=True) + # Ensure there are no hash mismatches + with open(file, "rb") as f: + sha1 = hashlib.sha1(f.read()).hexdigest() + if sha1 != self.sha1: + raise RuntimeError( + "hash mismatch for package " + self.path + ": " + + sha1 + " vs " + self.sha1 + " (known good)" + ) + return file + + def __repr__(self): + return "<Package "+self.path+" at "+self.url+" (sha1="+self.sha1+")" + +def fetch_url(url): + page = urllib.request.urlopen(url) + return page.read() + +def fetch_repository(base, repo_url): + packages = {} + root = ET.fromstring(fetch_url(base + repo_url)) + for package in root: + if package.tag != "remotePackage": + continue + path = package.attrib["path"] + + for archive in package.find("archives"): + host_os = archive.find("host-os") + if host_os is not None and host_os.text != HOST_OS: + continue + complete = archive.find("complete") + url = os.path.join(os.path.dirname(repo_url), complete.find("url").text) + sha1 = complete.find("checksum").text + + deps = [] + dependencies = package.find("dependencies") + if dependencies is not None: + for dep in dependencies: + deps.append(dep.attrib["path"]) + + packages[path] = Package(path, url, sha1, deps) + break + + return packages + +def fetch_repositories(): + packages = {} + for repo in REPOSITORIES: + packages.update(fetch_repository(BASE_REPOSITORY, repo)) + return packages + +class Lockfile: + def __init__(self, path): + self.path = path + self.packages = {} + if os.path.exists(path): + with open(path) as f: + for line in f: + path, url, sha1 = line.split(" ") + self.packages[path] = Package(path, url, sha1) + + def add(self, packages, name, *, update=True): + if name not in packages: + raise NameError("package not found: " + name) + if not update and name in self.packages: + return + self.packages[name] = packages[name] + for dep in packages[name].deps: + self.add(packages, dep, update=False) + + def save(self): + packages = list(sorted(self.packages.values(), key=lambda p: p.path)) + with open(self.path, "w") as f: + for package in packages: + f.write(package.path + " " + package.url + " " + package.sha1 + "\n") + +def cli_add_to_lockfile(args): + lockfile = Lockfile(args.lockfile) + packages = fetch_repositories() + for package in args.packages: + lockfile.add(packages, package) + lockfile.save() + +def cli_update_mirror(args): + lockfile = Lockfile(args.lockfile) + for package in lockfile.packages.values(): + path = package.download(BASE_REPOSITORY) + subprocess.run([ + "aws", "s3", "mv", path, + "s3://" + MIRROR_BUCKET + "/" + MIRROR_BASE_DIR + package.url, + "--profile=" + args.awscli_profile, + ], check=True) + +def cli_install(args): + lockfile = Lockfile(args.lockfile) + for package in lockfile.packages.values(): + # Download the file from the mirror into a temp file + url = "https://" + MIRROR_BUCKET + ".s3-" + MIRROR_BUCKET_REGION + \ + ".amazonaws.com/" + MIRROR_BASE_DIR + downloaded = package.download(url) + # Extract the file in a temporary directory + extract_dir = tempfile.mkdtemp() + subprocess.run([ + "unzip", "-q", downloaded, "-d", extract_dir, + ], check=True) + # Figure out the prefix used in the zip + subdirs = [d for d in os.listdir(extract_dir) if not d.startswith(".")] + if len(subdirs) != 1: + raise RuntimeError("extracted directory contains more than one dir") + # Move the extracted files in the proper directory + dest = os.path.join(args.dest, package.path.replace(";", "/")) + os.makedirs("/".join(dest.split("/")[:-1]), exist_ok=True) + os.rename(os.path.join(extract_dir, subdirs[0]), dest) + os.unlink(downloaded) + +def cli(): + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + + add_to_lockfile = subparsers.add_parser("add-to-lockfile") + add_to_lockfile.add_argument("lockfile") + add_to_lockfile.add_argument("packages", nargs="+") + add_to_lockfile.set_defaults(func=cli_add_to_lockfile) + + update_mirror = subparsers.add_parser("update-mirror") + update_mirror.add_argument("lockfile") + update_mirror.add_argument("--awscli-profile", default="default") + update_mirror.set_defaults(func=cli_update_mirror) + + install = subparsers.add_parser("install") + install.add_argument("lockfile") + install.add_argument("dest") + install.set_defaults(func=cli_install) + + args = parser.parse_args() + if not hasattr(args, "func"): + print("error: a subcommand is required (see --help)") + exit(1) + args.func(args) + +if __name__ == "__main__": + cli() |