summaryrefslogtreecommitdiffstats
path: root/tools/browsertime/mach_commands.py
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--tools/browsertime/mach_commands.py689
1 files changed, 689 insertions, 0 deletions
diff --git a/tools/browsertime/mach_commands.py b/tools/browsertime/mach_commands.py
new file mode 100644
index 0000000000..4850c82b75
--- /dev/null
+++ b/tools/browsertime/mach_commands.py
@@ -0,0 +1,689 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+r"""Make it easy to install and run [browsertime](https://github.com/sitespeedio/browsertime).
+
+Browsertime is a harness for running performance tests, similar to
+Mozilla's Raptor testing framework. Browsertime is written in Node.js
+and uses Selenium WebDriver to drive multiple browsers including
+Chrome, Chrome for Android, Firefox, and (pending the resolution of
+[Bug 1525126](https://bugzilla.mozilla.org/show_bug.cgi?id=1525126)
+and similar tickets) Firefox for Android and GeckoView-based vehicles.
+
+Right now a custom version of browsertime and the underlying
+geckodriver binary are needed to support GeckoView-based vehicles;
+this module accommodates those in-progress custom versions.
+
+To get started, run
+```
+./mach browsertime --setup [--clobber]
+```
+This will populate `tools/browsertime/node_modules`.
+
+To invoke browsertime, run
+```
+./mach browsertime [ARGS]
+```
+All arguments are passed through to browsertime.
+"""
+
+import argparse
+import collections
+import contextlib
+import json
+import logging
+import os
+import platform
+import re
+import stat
+import subprocess
+import sys
+import time
+
+import mozpack.path as mozpath
+from mach.decorators import Command, CommandArgument
+from mozbuild.base import BinaryNotFoundException, MachCommandBase
+from mozbuild.util import mkdir
+from six import StringIO
+
+AUTOMATION = "MOZ_AUTOMATION" in os.environ
+BROWSERTIME_ROOT = os.path.dirname(__file__)
+
+PILLOW_VERSION = "8.4.0" # version 8.4.0 currently supports python 3.6 to 3.10
+PYSSIM_VERSION = "0.4"
+SCIPY_VERSION = "1.2.3"
+NUMPY_VERSION = "1.16.1"
+OPENCV_VERSION = "4.5.4.60"
+
+py3_minor = sys.version_info.minor
+if py3_minor > 7:
+ SCIPY_VERSION = "1.7.3"
+ NUMPY_VERSION = "1.22.0"
+ PILLOW_VERSION = "9.0.0"
+
+MIN_NODE_VERSION = "16.0.0"
+
+IS_APPLE_SILICON = sys.platform.startswith(
+ "darwin"
+) and platform.processor().startswith("arm")
+
+
+@contextlib.contextmanager
+def silence():
+ oldout, olderr = sys.stdout, sys.stderr
+ try:
+ sys.stdout, sys.stderr = StringIO(), StringIO()
+ yield
+ finally:
+ sys.stdout, sys.stderr = oldout, olderr
+
+
+def node_path(command_context):
+ import platform
+ from distutils.version import StrictVersion
+
+ from mozbuild.nodeutil import find_node_executable
+
+ state_dir = command_context._mach_context.state_dir
+ cache_path = os.path.join(state_dir, "browsertime", "node-16")
+
+ NODE_FAILURE_MSG = (
+ "Could not locate a node binary that is at least version {}. ".format(
+ MIN_NODE_VERSION
+ )
+ + "Please run `./mach raptor --browsertime -t amazon` to install it "
+ + "from the Taskcluster Toolchain artifacts."
+ )
+
+ # Check standard locations first
+ node_exe = find_node_executable(min_version=StrictVersion(MIN_NODE_VERSION))
+ if node_exe and (node_exe[0] is not None):
+ return os.path.abspath(node_exe[0])
+ if not os.path.exists(cache_path):
+ raise Exception(NODE_FAILURE_MSG)
+
+ # Check the browsertime-specific node location next
+ node_name = "node"
+ if platform.system() == "Windows":
+ node_name = "node.exe"
+ node_exe_path = os.path.join(
+ state_dir,
+ "browsertime",
+ "node-16",
+ "node",
+ )
+ else:
+ node_exe_path = os.path.join(
+ state_dir,
+ "browsertime",
+ "node-16",
+ "node",
+ "bin",
+ )
+
+ node_exe = os.path.join(node_exe_path, node_name)
+ if not os.path.exists(node_exe):
+ raise Exception(NODE_FAILURE_MSG)
+
+ return os.path.abspath(node_exe)
+
+
+def package_path():
+ """The path to the `browsertime` directory.
+
+ Override the default with the `BROWSERTIME` environment variable."""
+ override = os.environ.get("BROWSERTIME", None)
+ if override:
+ return override
+
+ return mozpath.join(BROWSERTIME_ROOT, "node_modules", "browsertime")
+
+
+def browsertime_path():
+ """The path to the `browsertime.js` script."""
+ # On Windows, invoking `node_modules/.bin/browsertime{.cmd}`
+ # doesn't work when invoked as an argument to our specific
+ # binary. Since we want our version of node, invoke the
+ # actual script directly.
+ return mozpath.join(package_path(), "bin", "browsertime.js")
+
+
+def visualmetrics_path():
+ """The path to the `visualmetrics.py` script."""
+ return mozpath.join(package_path(), "browsertime", "visualmetrics-portable.py")
+
+
+def host_platform():
+ is_64bits = sys.maxsize > 2 ** 32
+
+ if sys.platform.startswith("win"):
+ if is_64bits:
+ return "win64"
+ elif sys.platform.startswith("linux"):
+ if is_64bits:
+ return "linux64"
+ elif sys.platform.startswith("darwin"):
+ return "darwin"
+
+ raise ValueError("sys.platform is not yet supported: {}".format(sys.platform))
+
+
+# Map from `host_platform()` to a `fetch`-like syntax.
+host_fetches = {
+ "darwin": {
+ "ffmpeg": {
+ "type": "static-url",
+ "url": "https://github.com/mozilla/perf-automation/releases/download/FFMPEG-v4.4.1/ffmpeg-macos.zip", # noqa
+ # An extension to `fetch` syntax.
+ "path": "ffmpeg-macos",
+ },
+ },
+ "linux64": {
+ "ffmpeg": {
+ "type": "static-url",
+ "url": "https://github.com/mozilla/perf-automation/releases/download/FFMPEG-v4.4.1/ffmpeg-4.4.1-i686-static.tar.xz", # noqa
+ # An extension to `fetch` syntax.
+ "path": "ffmpeg-4.4.1-i686-static",
+ },
+ },
+ "win64": {
+ "ffmpeg": {
+ "type": "static-url",
+ "url": "https://github.com/mozilla/perf-automation/releases/download/FFMPEG-v4.4.1/ffmpeg-4.4.1-full_build.zip", # noqa
+ # An extension to `fetch` syntax.
+ "path": "ffmpeg-4.4.1-full_build",
+ },
+ },
+}
+
+
+def artifact_cache_path(command_context):
+ r"""Downloaded artifacts will be kept here."""
+ # The convention is $MOZBUILD_STATE_PATH/cache/$FEATURE.
+ return mozpath.join(command_context._mach_context.state_dir, "cache", "browsertime")
+
+
+def state_path(command_context):
+ r"""Unpacked artifacts will be kept here."""
+ # The convention is $MOZBUILD_STATE_PATH/$FEATURE.
+ return mozpath.join(command_context._mach_context.state_dir, "browsertime")
+
+
+def setup_prerequisites(command_context):
+ r"""Install browsertime and visualmetrics.py prerequisites."""
+
+ from mozbuild.action.tooltool import unpack_file
+ from mozbuild.artifact_cache import ArtifactCache
+
+ # Download the visualmetrics-portable.py requirements.
+ artifact_cache = ArtifactCache(
+ artifact_cache_path(command_context),
+ log=command_context.log,
+ skip_cache=False,
+ )
+
+ fetches = host_fetches[host_platform()]
+ for tool, fetch in sorted(fetches.items()):
+ archive = artifact_cache.fetch(fetch["url"])
+ # TODO: assert type, verify sha256 (and size?).
+
+ if fetch.get("unpack", True):
+ cwd = os.getcwd()
+ try:
+ mkdir(state_path(command_context))
+ os.chdir(state_path(command_context))
+ command_context.log(
+ logging.INFO,
+ "browsertime",
+ {"path": archive},
+ "Unpacking temporary location {path}",
+ )
+
+ unpack_file(archive)
+
+ # Make sure the expected path exists after extraction
+ path = os.path.join(state_path(command_context), fetch.get("path"))
+ if not os.path.exists(path):
+ raise Exception("Cannot find an extracted directory: %s" % path)
+
+ try:
+ # Some archives provide binaries that don't have the
+ # executable bit set so we need to set it here
+ for root, dirs, files in os.walk(path):
+ for edir in dirs:
+ loc_to_change = os.path.join(root, edir)
+ st = os.stat(loc_to_change)
+ os.chmod(loc_to_change, st.st_mode | stat.S_IEXEC)
+ for efile in files:
+ loc_to_change = os.path.join(root, efile)
+ st = os.stat(loc_to_change)
+ os.chmod(loc_to_change, st.st_mode | stat.S_IEXEC)
+ except Exception as e:
+ raise Exception(
+ "Could not set executable bit in %s, error: %s" % (path, str(e))
+ )
+ finally:
+ os.chdir(cwd)
+
+
+def setup_browsertime(
+ command_context,
+ should_clobber=False,
+ new_upstream_url="",
+ install_vismet_reqs=False,
+):
+ r"""Install browsertime and visualmetrics.py prerequisites and the Node.js package."""
+
+ sys.path.append(mozpath.join(command_context.topsrcdir, "tools", "lint", "eslint"))
+ import setup_helper
+
+ if not new_upstream_url:
+ setup_prerequisites(command_context)
+
+ if new_upstream_url:
+ package_json_path = os.path.join(BROWSERTIME_ROOT, "package.json")
+
+ command_context.log(
+ logging.INFO,
+ "browsertime",
+ {
+ "new_upstream_url": new_upstream_url,
+ "package_json_path": package_json_path,
+ },
+ "Updating browsertime node module version in {package_json_path} "
+ "to {new_upstream_url}",
+ )
+
+ if not re.search("/tarball/[a-f0-9]{40}$", new_upstream_url):
+ raise ValueError(
+ "New upstream URL does not end with /tarball/[a-f0-9]{40}: '%s'"
+ % new_upstream_url
+ )
+
+ with open(package_json_path) as f:
+ existing_body = json.loads(
+ f.read(), object_pairs_hook=collections.OrderedDict
+ )
+
+ existing_body["devDependencies"]["browsertime"] = new_upstream_url
+
+ updated_body = json.dumps(existing_body)
+
+ with open(package_json_path, "w") as f:
+ f.write(updated_body)
+
+ # Install the browsertime Node.js requirements.
+ if not setup_helper.check_node_executables_valid():
+ return 1
+
+ # To use a custom `geckodriver`, set
+ # os.environ[b"GECKODRIVER_BASE_URL"] = bytes(url)
+ # to an endpoint with binaries named like
+ # https://github.com/sitespeedio/geckodriver/blob/master/install.js#L31.
+ if AUTOMATION:
+ os.environ["CHROMEDRIVER_SKIP_DOWNLOAD"] = "true"
+ os.environ["GECKODRIVER_SKIP_DOWNLOAD"] = "true"
+
+ if install_vismet_reqs:
+ # Hide this behind a flag so we don't install them by default in Raptor
+ command_context.log(
+ logging.INFO, "browsertime", {}, "Installing python requirements"
+ )
+ activate_browsertime_virtualenv(command_context)
+
+ command_context.log(
+ logging.INFO,
+ "browsertime",
+ {"package_json": mozpath.join(BROWSERTIME_ROOT, "package.json")},
+ "Installing browsertime node module from {package_json}",
+ )
+
+ # Add the mozbuild Node binary path to the OS environment in Apple Silicon.
+ # During the browesertime installation, it seems installation of sitespeed.io
+ # sub dependencies look for a global Node rather than the mozbuild Node binary.
+ # Normally `--scripts-prepend-node-path` should prevent this but it seems to still
+ # look for a system Node in the environment. This logic ensures the same Node is used.
+ node_dir = os.path.dirname(node_path(command_context))
+ if IS_APPLE_SILICON and node_dir not in os.environ["PATH"]:
+ os.environ["PATH"] += os.pathsep + node_dir
+
+ status = setup_helper.package_setup(
+ BROWSERTIME_ROOT,
+ "browsertime",
+ should_update=new_upstream_url != "",
+ should_clobber=should_clobber,
+ no_optional=new_upstream_url or AUTOMATION,
+ )
+
+ if status:
+ return status
+ if new_upstream_url or AUTOMATION:
+ return 0
+ if install_vismet_reqs:
+ return check(command_context)
+
+ return 0
+
+
+def node(command_context, args):
+ r"""Invoke node (interactively) with the given arguments."""
+ return command_context.run_process(
+ [node_path(command_context)] + args,
+ append_env=append_env(command_context),
+ pass_thru=True, # Allow user to run Node interactively.
+ ensure_exit_code=False, # Don't throw on non-zero exit code.
+ cwd=mozpath.join(command_context.topsrcdir),
+ )
+
+
+def append_env(command_context, append_path=True):
+ fetches = host_fetches[host_platform()]
+
+ # Ensure that `ffmpeg` is found and added to the environment
+ path = os.environ.get("PATH", "").split(os.pathsep) if append_path else []
+ path_to_ffmpeg = mozpath.join(
+ state_path(command_context), fetches["ffmpeg"]["path"]
+ )
+
+ path.insert(
+ 0,
+ path_to_ffmpeg
+ if host_platform().startswith("linux")
+ else mozpath.join(path_to_ffmpeg, "bin"),
+ ) # noqa
+
+ # Ensure that bare `node` and `npm` in scripts, including post-install
+ # scripts, finds the binary we're invoking with. Without this, it's
+ # easy for compiled extensions to get mismatched versions of the Node.js
+ # extension API.
+ node_dir = os.path.dirname(node_path(command_context))
+ path = [node_dir] + path
+
+ append_env = {
+ "PATH": os.pathsep.join(path),
+ # Bug 1560193: The JS library browsertime uses to execute commands
+ # (execa) will muck up the PATH variable and put the directory that
+ # node is in first in path. If this is globally-installed node,
+ # that means `/usr/bin` will be inserted first which means that we
+ # will get `/usr/bin/python` for `python`.
+ #
+ # Our fork of browsertime supports a `PYTHON` environment variable
+ # that points to the exact python executable to use.
+ "PYTHON": command_context.virtualenv_manager.python_path,
+ }
+
+ return append_env
+
+
+def _need_install(command_context, package):
+ from pip._internal.req.constructors import install_req_from_line
+
+ req = install_req_from_line(package)
+ req.check_if_exists(use_user_site=False)
+ if req.satisfied_by is None:
+ return True
+ venv_site_lib = os.path.abspath(
+ os.path.join(command_context.virtualenv_manager.bin_path, "..", "lib")
+ )
+ site_packages = os.path.abspath(req.satisfied_by.location)
+ return not site_packages.startswith(venv_site_lib)
+
+
+def activate_browsertime_virtualenv(command_context, *args, **kwargs):
+ r"""Activates virtualenv.
+
+ This function will also install Pillow and pyssim if needed.
+ It will raise an error in case the install failed.
+ """
+ # TODO: Remove `./mach browsertime` completely, as a follow up to Bug 1758990
+ MachCommandBase.activate_virtualenv(command_context, *args, **kwargs)
+
+ # installing Python deps on the fly
+ for dep in (
+ "Pillow==%s" % PILLOW_VERSION,
+ "pyssim==%s" % PYSSIM_VERSION,
+ "scipy==%s" % SCIPY_VERSION,
+ "numpy==%s" % NUMPY_VERSION,
+ "opencv-python==%s" % OPENCV_VERSION,
+ ):
+ if _need_install(command_context, dep):
+ subprocess.check_call(
+ [
+ command_context.virtualenv_manager.python_path,
+ "-m",
+ "pip",
+ "install",
+ dep,
+ ]
+ )
+
+
+def check(command_context):
+ r"""Run `visualmetrics.py --check`."""
+ command_context.activate_virtualenv()
+
+ args = ["--check"]
+ status = command_context.run_process(
+ [command_context.virtualenv_manager.python_path, visualmetrics_path()] + args,
+ # For --check, don't allow user's path to interfere with path testing except on Linux
+ append_env=append_env(
+ command_context, append_path=host_platform().startswith("linux")
+ ),
+ pass_thru=True,
+ ensure_exit_code=False, # Don't throw on non-zero exit code.
+ cwd=mozpath.join(command_context.topsrcdir),
+ )
+
+ sys.stdout.flush()
+ sys.stderr.flush()
+
+ if status:
+ return status
+
+ # Avoid logging the command (and, on Windows, the environment).
+ command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL)
+ print("browsertime version:", end=" ")
+
+ sys.stdout.flush()
+ sys.stderr.flush()
+
+ return node(command_context, [browsertime_path()] + ["--version"])
+
+
+def extra_default_args(command_context, args=[]):
+ # Add Mozilla-specific default arguments. This is tricky because browsertime is quite
+ # loose about arguments; repeat arguments are generally accepted but then produce
+ # difficult to interpret type errors.
+
+ def extract_browser_name(args):
+ "Extracts the browser name if any"
+ # These are BT arguments, it's BT job to check them
+ # here we just want to extract the browser name
+ res = re.findall("(--browser|-b)[= ]([\w]+)", " ".join(args))
+ if res == []:
+ return None
+ return res[0][-1]
+
+ def matches(args, *flags):
+ "Return True if any argument matches any of the given flags (maybe with an argument)."
+ for flag in flags:
+ if flag in args or any(arg.startswith(flag + "=") for arg in args):
+ return True
+ return False
+
+ extra_args = []
+
+ # Default to Firefox. Override with `-b ...` or `--browser=...`.
+ specifies_browser = matches(args, "-b", "--browser")
+ if not specifies_browser:
+ extra_args.extend(("-b", "firefox"))
+
+ # Default to not collect HAR. Override with `--skipHar=false`.
+ specifies_har = matches(args, "--har", "--skipHar", "--gzipHar")
+ if not specifies_har:
+ extra_args.append("--skipHar")
+
+ if not matches(args, "--android"):
+ # If --firefox.binaryPath is not specified, default to the objdir binary
+ # Note: --firefox.release is not a real browsertime option, but it will
+ # silently ignore it instead and default to a release installation.
+ specifies_binaryPath = matches(
+ args,
+ "--firefox.binaryPath",
+ "--firefox.release",
+ "--firefox.nightly",
+ "--firefox.beta",
+ "--firefox.developer",
+ )
+
+ if not specifies_binaryPath:
+ specifies_binaryPath = extract_browser_name(args) == "chrome"
+
+ if not specifies_binaryPath:
+ try:
+ extra_args.extend(
+ ("--firefox.binaryPath", command_context.get_binary_path())
+ )
+ except BinaryNotFoundException as e:
+ command_context.log(
+ logging.ERROR,
+ "browsertime",
+ {"error": str(e)},
+ "ERROR: {error}",
+ )
+ command_context.log(
+ logging.INFO,
+ "browsertime",
+ {},
+ "Please run |./mach build| "
+ "or specify a Firefox binary with --firefox.binaryPath.",
+ )
+ return 1
+
+ if extra_args:
+ command_context.log(
+ logging.DEBUG,
+ "browsertime",
+ {"extra_args": extra_args},
+ "Running browsertime with extra default arguments: {extra_args}",
+ )
+
+ return extra_args
+
+
+def _verify_node_install(command_context):
+ # check if Node is installed
+ sys.path.append(mozpath.join(command_context.topsrcdir, "tools", "lint", "eslint"))
+ import setup_helper
+
+ with silence():
+ node_valid = setup_helper.check_node_executables_valid()
+ if not node_valid:
+ print("Can't find Node. did you run ./mach bootstrap ?")
+ return False
+
+ # check if the browsertime package has been deployed correctly
+ # for this we just check for the browsertime directory presence
+ if not os.path.exists(browsertime_path()):
+ print("Could not find browsertime.js, try ./mach browsertime --setup")
+ print("If that still fails, try ./mach browsertime --setup --clobber")
+ return False
+
+ return True
+
+
+@Command(
+ "browsertime",
+ category="testing",
+ description="Run [browsertime](https://github.com/sitespeedio/browsertime) "
+ "performance tests.",
+)
+@CommandArgument(
+ "--verbose",
+ action="store_true",
+ help="Verbose output for what commands the build is running.",
+)
+@CommandArgument("--update-upstream-url", default="")
+@CommandArgument("--setup", default=False, action="store_true")
+@CommandArgument("--clobber", default=False, action="store_true")
+@CommandArgument(
+ "--skip-cache",
+ default=False,
+ action="store_true",
+ help="Skip all local caches to force re-fetching remote artifacts.",
+)
+@CommandArgument("--check-browsertime", default=False, action="store_true")
+@CommandArgument(
+ "--install-vismet-reqs",
+ default=False,
+ action="store_true",
+ help="Add this flag to get the visual metrics requirements installed.",
+)
+@CommandArgument(
+ "--browsertime-help",
+ default=False,
+ action="store_true",
+ help="Show the browsertime help message.",
+)
+@CommandArgument("args", nargs=argparse.REMAINDER)
+def browsertime(
+ command_context,
+ args,
+ verbose=False,
+ update_upstream_url="",
+ setup=False,
+ clobber=False,
+ skip_cache=False,
+ check_browsertime=False,
+ browsertime_help=False,
+ install_vismet_reqs=False,
+):
+ command_context._set_log_level(verbose)
+
+ # Output a message before going further to make sure the
+ # user knows that this tool is unsupported by the perftest
+ # team and point them to our supported tools. Pause a bit to
+ # make sure the user sees this message.
+ command_context.log(
+ logging.INFO,
+ "browsertime",
+ {},
+ "[INFO] This command should be used for browsertime setup only.\n"
+ "If you are looking to run performance tests on your patch, use "
+ "`./mach raptor --browsertime` instead.\n\nYou can get visual-metrics "
+ "by using the --browsertime-video and --browsertime-visualmetrics. "
+ "Here is a sample command for raptor-browsertime: \n\t`./mach raptor "
+ "--browsertime -t amazon --browsertime-video --browsertime-visualmetrics`\n\n"
+ "See this wiki page for more information if needed: "
+ "https://wiki.mozilla.org/TestEngineering/Performance/Raptor/Browsertime\n\n",
+ )
+ time.sleep(5)
+
+ if update_upstream_url:
+ return setup_browsertime(
+ command_context,
+ new_upstream_url=update_upstream_url,
+ install_vismet_reqs=install_vismet_reqs,
+ )
+ elif setup:
+ return setup_browsertime(
+ command_context,
+ should_clobber=clobber,
+ install_vismet_reqs=install_vismet_reqs,
+ )
+ else:
+ if not _verify_node_install(command_context):
+ return 1
+
+ if check_browsertime:
+ return check(command_context)
+
+ if browsertime_help:
+ args.append("--help")
+
+ activate_browsertime_virtualenv(command_context)
+ default_args = extra_default_args(command_context, args)
+ if default_args == 1:
+ return 1
+ return node(command_context, [browsertime_path()] + default_args + args)