diff options
Diffstat (limited to '')
-rw-r--r-- | taskcluster/scripts/misc/fetch-chromium.py | 235 |
1 files changed, 235 insertions, 0 deletions
diff --git a/taskcluster/scripts/misc/fetch-chromium.py b/taskcluster/scripts/misc/fetch-chromium.py new file mode 100644 index 0000000000..d9a03261d1 --- /dev/null +++ b/taskcluster/scripts/misc/fetch-chromium.py @@ -0,0 +1,235 @@ +#!/usr/bin/python3 -u +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +This script downloads the latest chromium build (or a manually +defined version) for a given platform. It then uploads the build, +with the revision of the build stored in a REVISION file. +""" + +import argparse +import errno +import os +import shutil +import subprocess +import tempfile + +import requests +from redo import retriable + +LAST_CHANGE_URL = ( + # formatted with platform + "https://www.googleapis.com/download/storage/v1/b/" + "chromium-browser-snapshots/o/{}%2FLAST_CHANGE?alt=media" +) + +CHROMIUM_BASE_URL = ( + # formatted with (platform/revision/archive) + "https://www.googleapis.com/download/storage/v1/b/" + "chromium-browser-snapshots/o/{}%2F{}%2F{}?alt=media" +) + + +CHROMIUM_INFO = { + "linux": { + "platform": "Linux_x64", + "chromium": "chrome-linux.zip", + "result": "chromium-linux.tar.bz2", + "chromedriver": "chromedriver_linux64.zip", + }, + "win32": { + "platform": "Win", + "chromium": "chrome-win.zip", + "result": "chromium-win32.tar.bz2", + "chromedriver": "chromedriver_win32.zip", + }, + "win64": { + "platform": "Win", + "chromium": "chrome-win.zip", + "result": "chromium-win64.tar.bz2", + "chromedriver": "chromedriver_win32.zip", + }, + "mac": { + "platform": "Mac", + "chromium": "chrome-mac.zip", + "result": "chromium-mac.tar.bz2", + "chromedriver": "chromedriver_mac64.zip", + }, +} + + +def log(msg): + print("build-chromium: %s" % msg) + + +@retriable(attempts=7, sleeptime=5, sleepscale=2) +def fetch_file(url, filepath): + """Download a file from the given url to a given file.""" + size = 4096 + r = requests.get(url, stream=True) + r.raise_for_status() + + with open(filepath, "wb") as fd: + for chunk in r.iter_content(size): + fd.write(chunk) + + +def unzip(zippath, target): + """Unzips an archive to the target location.""" + log("Unpacking archive at: %s to: %s" % (zippath, target)) + unzip_command = ["unzip", "-q", "-o", zippath, "-d", target] + subprocess.check_call(unzip_command) + + +@retriable(attempts=7, sleeptime=5, sleepscale=2) +def fetch_chromium_revision(platform): + """Get the revision of the latest chromium build.""" + chromium_platform = CHROMIUM_INFO[platform]["platform"] + revision_url = LAST_CHANGE_URL.format(chromium_platform) + + log("Getting revision number for latest %s chromium build..." % chromium_platform) + + # Expecting a file with a single number indicating the latest + # chromium build with a chromedriver that we can download + r = requests.get(revision_url, timeout=30) + r.raise_for_status() + + chromium_revision = r.content.decode("utf-8") + return chromium_revision.strip() + + +def fetch_chromium_build(platform, revision, zippath): + """Download a chromium build for a given revision, or the latest.""" + if not revision: + revision = fetch_chromium_revision(platform) + + download_platform = CHROMIUM_INFO[platform]["platform"] + download_url = CHROMIUM_BASE_URL.format( + download_platform, revision, CHROMIUM_INFO[platform]["chromium"] + ) + + log("Downloading %s chromium build revision %s..." % (download_platform, revision)) + log(download_url) + fetch_file(download_url, zippath) + return revision + + +def fetch_chromedriver(platform, revision, chromium_dir): + """Get the chromedriver for the given revision and repackage it.""" + download_url = CHROMIUM_BASE_URL.format( + CHROMIUM_INFO[platform]["platform"], + revision, + CHROMIUM_INFO[platform]["chromedriver"], + ) + + tmpzip = os.path.join(tempfile.mkdtemp(), "cd-tmp.zip") + log("Downloading chromedriver from %s" % download_url) + fetch_file(download_url, tmpzip) + + tmppath = tempfile.mkdtemp() + unzip(tmpzip, tmppath) + + # Find the chromedriver then copy it to the chromium directory + cd_path = None + for dirpath, _, filenames in os.walk(tmppath): + for filename in filenames: + if filename == "chromedriver" or filename == "chromedriver.exe": + cd_path = os.path.join(dirpath, filename) + break + if cd_path is not None: + break + if cd_path is None: + raise Exception("Could not find chromedriver binary in %s" % tmppath) + log("Copying chromedriver from: %s to: %s" % (cd_path, chromium_dir)) + shutil.copy(cd_path, chromium_dir) + + +def build_chromium_archive(platform, revision=None): + """ + Download and store a chromium build for a given platform. + + Retrieves either the latest version, or uses a pre-defined version if + the `--revision` option is given a revision. + """ + upload_dir = os.environ.get("UPLOAD_DIR") + if upload_dir: + # Create the upload directory if it doesn't exist. + try: + log("Creating upload directory in %s..." % os.path.abspath(upload_dir)) + os.makedirs(upload_dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + # Make a temporary location for the file + tmppath = tempfile.mkdtemp() + tmpzip = os.path.join(tmppath, "tmp-chromium.zip") + + revision = fetch_chromium_build(platform, revision, tmpzip) + + # Unpack archive in `tmpzip` to store the revision number and + # the chromedriver + unzip(tmpzip, tmppath) + + dirs = [ + d + for d in os.listdir(tmppath) + if os.path.isdir(os.path.join(tmppath, d)) and d.startswith("chrome-") + ] + + if len(dirs) > 1: + raise Exception( + "Too many directories starting with `chrome-` after extracting." + ) + elif len(dirs) == 0: + raise Exception( + "Could not find any directories after extraction of chromium zip." + ) + + chromium_dir = os.path.join(tmppath, dirs[0]) + revision_file = os.path.join(chromium_dir, ".REVISION") + with open(revision_file, "w+") as f: + f.write(str(revision)) + + # Get and store the chromedriver + fetch_chromedriver(platform, revision, chromium_dir) + + tar_file = CHROMIUM_INFO[platform]["result"] + tar_command = ["tar", "cjf", tar_file, "-C", tmppath, dirs[0]] + log("Added revision to %s file." % revision_file) + + log("Tarring with the command: %s" % str(tar_command)) + subprocess.check_call(tar_command) + + upload_dir = os.environ.get("UPLOAD_DIR") + if upload_dir: + # Move the tarball to the output directory for upload. + log("Moving %s to the upload directory..." % tar_file) + shutil.copy(tar_file, os.path.join(upload_dir, tar_file)) + + shutil.rmtree(tmppath) + + +def parse_args(): + """Read command line arguments and return options.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--platform", help="Platform version of chromium to build.", required=True + ) + parser.add_argument( + "--revision", + help="Revision of chromium to build to get. " + "(Defaults to the newest chromium build).", + default=None, + ) + + return parser.parse_args() + + +if __name__ == "__main__": + args = vars(parse_args()) + build_chromium_archive(**args) |