summaryrefslogtreecommitdiffstats
path: root/taskcluster/scripts/misc/fetch-chromium.py
diff options
context:
space:
mode:
Diffstat (limited to 'taskcluster/scripts/misc/fetch-chromium.py')
-rw-r--r--taskcluster/scripts/misc/fetch-chromium.py235
1 files changed, 235 insertions, 0 deletions
diff --git a/taskcluster/scripts/misc/fetch-chromium.py b/taskcluster/scripts/misc/fetch-chromium.py
new file mode 100644
index 0000000000..d9a03261d1
--- /dev/null
+++ b/taskcluster/scripts/misc/fetch-chromium.py
@@ -0,0 +1,235 @@
+#!/usr/bin/python3 -u
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+This script downloads the latest chromium build (or a manually
+defined version) for a given platform. It then uploads the build,
+with the revision of the build stored in a REVISION file.
+"""
+
+import argparse
+import errno
+import os
+import shutil
+import subprocess
+import tempfile
+
+import requests
+from redo import retriable
+
+LAST_CHANGE_URL = (
+ # formatted with platform
+ "https://www.googleapis.com/download/storage/v1/b/"
+ "chromium-browser-snapshots/o/{}%2FLAST_CHANGE?alt=media"
+)
+
+CHROMIUM_BASE_URL = (
+ # formatted with (platform/revision/archive)
+ "https://www.googleapis.com/download/storage/v1/b/"
+ "chromium-browser-snapshots/o/{}%2F{}%2F{}?alt=media"
+)
+
+
+CHROMIUM_INFO = {
+ "linux": {
+ "platform": "Linux_x64",
+ "chromium": "chrome-linux.zip",
+ "result": "chromium-linux.tar.bz2",
+ "chromedriver": "chromedriver_linux64.zip",
+ },
+ "win32": {
+ "platform": "Win",
+ "chromium": "chrome-win.zip",
+ "result": "chromium-win32.tar.bz2",
+ "chromedriver": "chromedriver_win32.zip",
+ },
+ "win64": {
+ "platform": "Win",
+ "chromium": "chrome-win.zip",
+ "result": "chromium-win64.tar.bz2",
+ "chromedriver": "chromedriver_win32.zip",
+ },
+ "mac": {
+ "platform": "Mac",
+ "chromium": "chrome-mac.zip",
+ "result": "chromium-mac.tar.bz2",
+ "chromedriver": "chromedriver_mac64.zip",
+ },
+}
+
+
+def log(msg):
+ print("build-chromium: %s" % msg)
+
+
+@retriable(attempts=7, sleeptime=5, sleepscale=2)
+def fetch_file(url, filepath):
+ """Download a file from the given url to a given file."""
+ size = 4096
+ r = requests.get(url, stream=True)
+ r.raise_for_status()
+
+ with open(filepath, "wb") as fd:
+ for chunk in r.iter_content(size):
+ fd.write(chunk)
+
+
+def unzip(zippath, target):
+ """Unzips an archive to the target location."""
+ log("Unpacking archive at: %s to: %s" % (zippath, target))
+ unzip_command = ["unzip", "-q", "-o", zippath, "-d", target]
+ subprocess.check_call(unzip_command)
+
+
+@retriable(attempts=7, sleeptime=5, sleepscale=2)
+def fetch_chromium_revision(platform):
+ """Get the revision of the latest chromium build."""
+ chromium_platform = CHROMIUM_INFO[platform]["platform"]
+ revision_url = LAST_CHANGE_URL.format(chromium_platform)
+
+ log("Getting revision number for latest %s chromium build..." % chromium_platform)
+
+ # Expecting a file with a single number indicating the latest
+ # chromium build with a chromedriver that we can download
+ r = requests.get(revision_url, timeout=30)
+ r.raise_for_status()
+
+ chromium_revision = r.content.decode("utf-8")
+ return chromium_revision.strip()
+
+
+def fetch_chromium_build(platform, revision, zippath):
+ """Download a chromium build for a given revision, or the latest."""
+ if not revision:
+ revision = fetch_chromium_revision(platform)
+
+ download_platform = CHROMIUM_INFO[platform]["platform"]
+ download_url = CHROMIUM_BASE_URL.format(
+ download_platform, revision, CHROMIUM_INFO[platform]["chromium"]
+ )
+
+ log("Downloading %s chromium build revision %s..." % (download_platform, revision))
+ log(download_url)
+ fetch_file(download_url, zippath)
+ return revision
+
+
+def fetch_chromedriver(platform, revision, chromium_dir):
+ """Get the chromedriver for the given revision and repackage it."""
+ download_url = CHROMIUM_BASE_URL.format(
+ CHROMIUM_INFO[platform]["platform"],
+ revision,
+ CHROMIUM_INFO[platform]["chromedriver"],
+ )
+
+ tmpzip = os.path.join(tempfile.mkdtemp(), "cd-tmp.zip")
+ log("Downloading chromedriver from %s" % download_url)
+ fetch_file(download_url, tmpzip)
+
+ tmppath = tempfile.mkdtemp()
+ unzip(tmpzip, tmppath)
+
+ # Find the chromedriver then copy it to the chromium directory
+ cd_path = None
+ for dirpath, _, filenames in os.walk(tmppath):
+ for filename in filenames:
+ if filename == "chromedriver" or filename == "chromedriver.exe":
+ cd_path = os.path.join(dirpath, filename)
+ break
+ if cd_path is not None:
+ break
+ if cd_path is None:
+ raise Exception("Could not find chromedriver binary in %s" % tmppath)
+ log("Copying chromedriver from: %s to: %s" % (cd_path, chromium_dir))
+ shutil.copy(cd_path, chromium_dir)
+
+
+def build_chromium_archive(platform, revision=None):
+ """
+ Download and store a chromium build for a given platform.
+
+ Retrieves either the latest version, or uses a pre-defined version if
+ the `--revision` option is given a revision.
+ """
+ upload_dir = os.environ.get("UPLOAD_DIR")
+ if upload_dir:
+ # Create the upload directory if it doesn't exist.
+ try:
+ log("Creating upload directory in %s..." % os.path.abspath(upload_dir))
+ os.makedirs(upload_dir)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+
+ # Make a temporary location for the file
+ tmppath = tempfile.mkdtemp()
+ tmpzip = os.path.join(tmppath, "tmp-chromium.zip")
+
+ revision = fetch_chromium_build(platform, revision, tmpzip)
+
+ # Unpack archive in `tmpzip` to store the revision number and
+ # the chromedriver
+ unzip(tmpzip, tmppath)
+
+ dirs = [
+ d
+ for d in os.listdir(tmppath)
+ if os.path.isdir(os.path.join(tmppath, d)) and d.startswith("chrome-")
+ ]
+
+ if len(dirs) > 1:
+ raise Exception(
+ "Too many directories starting with `chrome-` after extracting."
+ )
+ elif len(dirs) == 0:
+ raise Exception(
+ "Could not find any directories after extraction of chromium zip."
+ )
+
+ chromium_dir = os.path.join(tmppath, dirs[0])
+ revision_file = os.path.join(chromium_dir, ".REVISION")
+ with open(revision_file, "w+") as f:
+ f.write(str(revision))
+
+ # Get and store the chromedriver
+ fetch_chromedriver(platform, revision, chromium_dir)
+
+ tar_file = CHROMIUM_INFO[platform]["result"]
+ tar_command = ["tar", "cjf", tar_file, "-C", tmppath, dirs[0]]
+ log("Added revision to %s file." % revision_file)
+
+ log("Tarring with the command: %s" % str(tar_command))
+ subprocess.check_call(tar_command)
+
+ upload_dir = os.environ.get("UPLOAD_DIR")
+ if upload_dir:
+ # Move the tarball to the output directory for upload.
+ log("Moving %s to the upload directory..." % tar_file)
+ shutil.copy(tar_file, os.path.join(upload_dir, tar_file))
+
+ shutil.rmtree(tmppath)
+
+
+def parse_args():
+ """Read command line arguments and return options."""
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--platform", help="Platform version of chromium to build.", required=True
+ )
+ parser.add_argument(
+ "--revision",
+ help="Revision of chromium to build to get. "
+ "(Defaults to the newest chromium build).",
+ default=None,
+ )
+
+ return parser.parse_args()
+
+
+if __name__ == "__main__":
+ args = vars(parse_args())
+ build_chromium_archive(**args)
generated by cgit v1.2.3 (git 2.39.1) at 2024-05-10 12:18:24 +0000