1 files changed, 235 insertions, 0 deletions
diff --git a/taskcluster/scripts/misc/fetch-chromium.py b/taskcluster/scripts/misc/fetch-chromium.py
new file mode 100644
index 0000000000..d9a03261d1
--- /dev/null
+++ b/taskcluster/scripts/misc/fetch-chromium.py
@@ -0,0 +1,235 @@
+#!/usr/bin/python3 -u
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+This script downloads the latest chromium build (or a manually
+defined version) for a given platform. It then uploads the build,
+with the revision of the build stored in a REVISION file.
+"""
+
+import argparse
+import errno
+import os
+import shutil
+import subprocess
+import tempfile
+
+import requests
+from redo import retriable
+
+LAST_CHANGE_URL = (
+    # formatted with platform
+    "https://www.googleapis.com/download/storage/v1/b/"
+    "chromium-browser-snapshots/o/{}%2FLAST_CHANGE?alt=media"
+)
+
+CHROMIUM_BASE_URL = (
+    # formatted with (platform/revision/archive)
+    "https://www.googleapis.com/download/storage/v1/b/"
+    "chromium-browser-snapshots/o/{}%2F{}%2F{}?alt=media"
+)
+
+
+CHROMIUM_INFO = {
+    "linux": {
+        "platform": "Linux_x64",
+        "chromium": "chrome-linux.zip",
+        "result": "chromium-linux.tar.bz2",
+        "chromedriver": "chromedriver_linux64.zip",
+    },
+    "win32": {
+        "platform": "Win",
+        "chromium": "chrome-win.zip",
+        "result": "chromium-win32.tar.bz2",
+        "chromedriver": "chromedriver_win32.zip",
+    },
+    "win64": {
+        "platform": "Win",
+        "chromium": "chrome-win.zip",
+        "result": "chromium-win64.tar.bz2",
+        "chromedriver": "chromedriver_win32.zip",
+    },
+    "mac": {
+        "platform": "Mac",
+        "chromium": "chrome-mac.zip",
+        "result": "chromium-mac.tar.bz2",
+        "chromedriver": "chromedriver_mac64.zip",
+    },
+}
+
+
+def log(msg):
+    print("build-chromium: %s" % msg)
+
+
+@retriable(attempts=7, sleeptime=5, sleepscale=2)
+def fetch_file(url, filepath):
+    """Download a file from the given url to a given file."""
+    size = 4096
+    r = requests.get(url, stream=True)
+    r.raise_for_status()
+
+    with open(filepath, "wb") as fd:
+        for chunk in r.iter_content(size):
+            fd.write(chunk)
+
+
+def unzip(zippath, target):
+    """Unzips an archive to the target location."""
+    log("Unpacking archive at: %s to: %s" % (zippath, target))
+    unzip_command = ["unzip", "-q", "-o", zippath, "-d", target]
+    subprocess.check_call(unzip_command)
+
+
+@retriable(attempts=7, sleeptime=5, sleepscale=2)
+def fetch_chromium_revision(platform):
+    """Get the revision of the latest chromium build."""
+    chromium_platform = CHROMIUM_INFO[platform]["platform"]
+    revision_url = LAST_CHANGE_URL.format(chromium_platform)
+
+    log("Getting revision number for latest %s chromium build..." % chromium_platform)
+
+    # Expecting a file with a single number indicating the latest
+    # chromium build with a chromedriver that we can download
+    r = requests.get(revision_url, timeout=30)
+    r.raise_for_status()
+
+    chromium_revision = r.content.decode("utf-8")
+    return chromium_revision.strip()
+
+
+def fetch_chromium_build(platform, revision, zippath):
+    """Download a chromium build for a given revision, or the latest."""
+    if not revision:
+        revision = fetch_chromium_revision(platform)
+
+    download_platform = CHROMIUM_INFO[platform]["platform"]
+    download_url = CHROMIUM_BASE_URL.format(
+        download_platform, revision, CHROMIUM_INFO[platform]["chromium"]
+    )
+
+    log("Downloading %s chromium build revision %s..." % (download_platform, revision))
+    log(download_url)
+    fetch_file(download_url, zippath)
+    return revision
+
+
+def fetch_chromedriver(platform, revision, chromium_dir):
+    """Get the chromedriver for the given revision and repackage it."""
+    download_url = CHROMIUM_BASE_URL.format(
+        CHROMIUM_INFO[platform]["platform"],
+        revision,
+        CHROMIUM_INFO[platform]["chromedriver"],
+    )
+
+    tmpzip = os.path.join(tempfile.mkdtemp(), "cd-tmp.zip")
+    log("Downloading chromedriver from %s" % download_url)
+    fetch_file(download_url, tmpzip)
+
+    tmppath = tempfile.mkdtemp()
+    unzip(tmpzip, tmppath)
+
+    # Find the chromedriver then copy it to the chromium directory
+    cd_path = None
+    for dirpath, _, filenames in os.walk(tmppath):
+        for filename in filenames:
+            if filename == "chromedriver" or filename == "chromedriver.exe":
+                cd_path = os.path.join(dirpath, filename)
+                break
+        if cd_path is not None:
+            break
+    if cd_path is None:
+        raise Exception("Could not find chromedriver binary in %s" % tmppath)
+    log("Copying chromedriver from: %s to: %s" % (cd_path, chromium_dir))
+    shutil.copy(cd_path, chromium_dir)
+
+
+def build_chromium_archive(platform, revision=None):
+    """
+    Download and store a chromium build for a given platform.
+
+    Retrieves either the latest version, or uses a pre-defined version if
+    the `--revision` option is given a revision.
+    """
+    upload_dir = os.environ.get("UPLOAD_DIR")
+    if upload_dir:
+        # Create the upload directory if it doesn't exist.
+        try:
+            log("Creating upload directory in %s..." % os.path.abspath(upload_dir))
+            os.makedirs(upload_dir)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+
+    # Make a temporary location for the file
+    tmppath = tempfile.mkdtemp()
+    tmpzip = os.path.join(tmppath, "tmp-chromium.zip")
+
+    revision = fetch_chromium_build(platform, revision, tmpzip)
+
+    # Unpack archive in `tmpzip` to store the revision number and
+    # the chromedriver
+    unzip(tmpzip, tmppath)
+
+    dirs = [
+        d
+        for d in os.listdir(tmppath)
+        if os.path.isdir(os.path.join(tmppath, d)) and d.startswith("chrome-")
+    ]
+
+    if len(dirs) > 1:
+        raise Exception(
+            "Too many directories starting with `chrome-` after extracting."
+        )
+    elif len(dirs) == 0:
+        raise Exception(
+            "Could not find any directories after extraction of chromium zip."
+        )
+
+    chromium_dir = os.path.join(tmppath, dirs[0])
+    revision_file = os.path.join(chromium_dir, ".REVISION")
+    with open(revision_file, "w+") as f:
+        f.write(str(revision))
+
+    # Get and store the chromedriver
+    fetch_chromedriver(platform, revision, chromium_dir)
+
+    tar_file = CHROMIUM_INFO[platform]["result"]
+    tar_command = ["tar", "cjf", tar_file, "-C", tmppath, dirs[0]]
+    log("Added revision to %s file." % revision_file)
+
+    log("Tarring with the command: %s" % str(tar_command))
+    subprocess.check_call(tar_command)
+
+    upload_dir = os.environ.get("UPLOAD_DIR")
+    if upload_dir:
+        # Move the tarball to the output directory for upload.
+        log("Moving %s to the upload directory..." % tar_file)
+        shutil.copy(tar_file, os.path.join(upload_dir, tar_file))
+
+    shutil.rmtree(tmppath)
+
+
+def parse_args():
+    """Read command line arguments and return options."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--platform", help="Platform version of chromium to build.", required=True
+    )
+    parser.add_argument(
+        "--revision",
+        help="Revision of chromium to build to get. "
+        "(Defaults to the newest chromium build).",
+        default=None,
+    )
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = vars(parse_args())
+    build_chromium_archive(**args)