Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 689 insertions, 0 deletions

diff --git a/tools/browsertime/mach_commands.py b/tools/browsertime/mach_commands.py
new file mode 100644
index 0000000000..4850c82b75
--- /dev/null
+++ b/tools/browsertime/mach_commands.py
@@ -0,0 +1,689 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+r"""Make it easy to install and run [browsertime](https://github.com/sitespeedio/browsertime).
+
+Browsertime is a harness for running performance tests, similar to
+Mozilla's Raptor testing framework.  Browsertime is written in Node.js
+and uses Selenium WebDriver to drive multiple browsers including
+Chrome, Chrome for Android, Firefox, and (pending the resolution of
+[Bug 1525126](https://bugzilla.mozilla.org/show_bug.cgi?id=1525126)
+and similar tickets) Firefox for Android and GeckoView-based vehicles.
+
+Right now a custom version of browsertime and the underlying
+geckodriver binary are needed to support GeckoView-based vehicles;
+this module accommodates those in-progress custom versions.
+
+To get started, run
+```
+./mach browsertime --setup [--clobber]
+```
+This will populate `tools/browsertime/node_modules`.
+
+To invoke browsertime, run
+```
+./mach browsertime [ARGS]
+```
+All arguments are passed through to browsertime.
+"""
+
+import argparse
+import collections
+import contextlib
+import json
+import logging
+import os
+import platform
+import re
+import stat
+import subprocess
+import sys
+import time
+
+import mozpack.path as mozpath
+from mach.decorators import Command, CommandArgument
+from mozbuild.base import BinaryNotFoundException, MachCommandBase
+from mozbuild.util import mkdir
+from six import StringIO
+
+AUTOMATION = "MOZ_AUTOMATION" in os.environ
+BROWSERTIME_ROOT = os.path.dirname(__file__)
+
+PILLOW_VERSION = "8.4.0"  # version 8.4.0 currently supports python 3.6 to 3.10
+PYSSIM_VERSION = "0.4"
+SCIPY_VERSION = "1.2.3"
+NUMPY_VERSION = "1.16.1"
+OPENCV_VERSION = "4.5.4.60"
+
+py3_minor = sys.version_info.minor
+if py3_minor > 7:
+    SCIPY_VERSION = "1.7.3"
+    NUMPY_VERSION = "1.22.0"
+    PILLOW_VERSION = "9.0.0"
+
+MIN_NODE_VERSION = "16.0.0"
+
+IS_APPLE_SILICON = sys.platform.startswith(
+    "darwin"
+) and platform.processor().startswith("arm")
+
+
+@contextlib.contextmanager
+def silence():
+    oldout, olderr = sys.stdout, sys.stderr
+    try:
+        sys.stdout, sys.stderr = StringIO(), StringIO()
+        yield
+    finally:
+        sys.stdout, sys.stderr = oldout, olderr
+
+
+def node_path(command_context):
+    import platform
+    from distutils.version import StrictVersion
+
+    from mozbuild.nodeutil import find_node_executable
+
+    state_dir = command_context._mach_context.state_dir
+    cache_path = os.path.join(state_dir, "browsertime", "node-16")
+
+    NODE_FAILURE_MSG = (
+        "Could not locate a node binary that is at least version {}. ".format(
+            MIN_NODE_VERSION
+        )
+        + "Please run `./mach raptor --browsertime -t amazon` to install it "
+        + "from the Taskcluster Toolchain artifacts."
+    )
+
+    # Check standard locations first
+    node_exe = find_node_executable(min_version=StrictVersion(MIN_NODE_VERSION))
+    if node_exe and (node_exe[0] is not None):
+        return os.path.abspath(node_exe[0])
+    if not os.path.exists(cache_path):
+        raise Exception(NODE_FAILURE_MSG)
+
+    # Check the browsertime-specific node location next
+    node_name = "node"
+    if platform.system() == "Windows":
+        node_name = "node.exe"
+        node_exe_path = os.path.join(
+            state_dir,
+            "browsertime",
+            "node-16",
+            "node",
+        )
+    else:
+        node_exe_path = os.path.join(
+            state_dir,
+            "browsertime",
+            "node-16",
+            "node",
+            "bin",
+        )
+
+    node_exe = os.path.join(node_exe_path, node_name)
+    if not os.path.exists(node_exe):
+        raise Exception(NODE_FAILURE_MSG)
+
+    return os.path.abspath(node_exe)
+
+
+def package_path():
+    """The path to the `browsertime` directory.
+
+    Override the default with the `BROWSERTIME` environment variable."""
+    override = os.environ.get("BROWSERTIME", None)
+    if override:
+        return override
+
+    return mozpath.join(BROWSERTIME_ROOT, "node_modules", "browsertime")
+
+
+def browsertime_path():
+    """The path to the `browsertime.js` script."""
+    # On Windows, invoking `node_modules/.bin/browsertime{.cmd}`
+    # doesn't work when invoked as an argument to our specific
+    # binary.  Since we want our version of node, invoke the
+    # actual script directly.
+    return mozpath.join(package_path(), "bin", "browsertime.js")
+
+
+def visualmetrics_path():
+    """The path to the `visualmetrics.py` script."""
+    return mozpath.join(package_path(), "browsertime", "visualmetrics-portable.py")
+
+
+def host_platform():
+    is_64bits = sys.maxsize > 2 ** 32
+
+    if sys.platform.startswith("win"):
+        if is_64bits:
+            return "win64"
+    elif sys.platform.startswith("linux"):
+        if is_64bits:
+            return "linux64"
+    elif sys.platform.startswith("darwin"):
+        return "darwin"
+
+    raise ValueError("sys.platform is not yet supported: {}".format(sys.platform))
+
+
+# Map from `host_platform()` to a `fetch`-like syntax.
+host_fetches = {
+    "darwin": {
+        "ffmpeg": {
+            "type": "static-url",
+            "url": "https://github.com/mozilla/perf-automation/releases/download/FFMPEG-v4.4.1/ffmpeg-macos.zip",  # noqa
+            # An extension to `fetch` syntax.
+            "path": "ffmpeg-macos",
+        },
+    },
+    "linux64": {
+        "ffmpeg": {
+            "type": "static-url",
+            "url": "https://github.com/mozilla/perf-automation/releases/download/FFMPEG-v4.4.1/ffmpeg-4.4.1-i686-static.tar.xz",  # noqa
+            # An extension to `fetch` syntax.
+            "path": "ffmpeg-4.4.1-i686-static",
+        },
+    },
+    "win64": {
+        "ffmpeg": {
+            "type": "static-url",
+            "url": "https://github.com/mozilla/perf-automation/releases/download/FFMPEG-v4.4.1/ffmpeg-4.4.1-full_build.zip",  # noqa
+            # An extension to `fetch` syntax.
+            "path": "ffmpeg-4.4.1-full_build",
+        },
+    },
+}
+
+
+def artifact_cache_path(command_context):
+    r"""Downloaded artifacts will be kept here."""
+    # The convention is $MOZBUILD_STATE_PATH/cache/$FEATURE.
+    return mozpath.join(command_context._mach_context.state_dir, "cache", "browsertime")
+
+
+def state_path(command_context):
+    r"""Unpacked artifacts will be kept here."""
+    # The convention is $MOZBUILD_STATE_PATH/$FEATURE.
+    return mozpath.join(command_context._mach_context.state_dir, "browsertime")
+
+
+def setup_prerequisites(command_context):
+    r"""Install browsertime and visualmetrics.py prerequisites."""
+
+    from mozbuild.action.tooltool import unpack_file
+    from mozbuild.artifact_cache import ArtifactCache
+
+    # Download the visualmetrics-portable.py requirements.
+    artifact_cache = ArtifactCache(
+        artifact_cache_path(command_context),
+        log=command_context.log,
+        skip_cache=False,
+    )
+
+    fetches = host_fetches[host_platform()]
+    for tool, fetch in sorted(fetches.items()):
+        archive = artifact_cache.fetch(fetch["url"])
+        # TODO: assert type, verify sha256 (and size?).
+
+        if fetch.get("unpack", True):
+            cwd = os.getcwd()
+            try:
+                mkdir(state_path(command_context))
+                os.chdir(state_path(command_context))
+                command_context.log(
+                    logging.INFO,
+                    "browsertime",
+                    {"path": archive},
+                    "Unpacking temporary location {path}",
+                )
+
+                unpack_file(archive)
+
+                # Make sure the expected path exists after extraction
+                path = os.path.join(state_path(command_context), fetch.get("path"))
+                if not os.path.exists(path):
+                    raise Exception("Cannot find an extracted directory: %s" % path)
+
+                try:
+                    # Some archives provide binaries that don't have the
+                    # executable bit set so we need to set it here
+                    for root, dirs, files in os.walk(path):
+                        for edir in dirs:
+                            loc_to_change = os.path.join(root, edir)
+                            st = os.stat(loc_to_change)
+                            os.chmod(loc_to_change, st.st_mode | stat.S_IEXEC)
+                        for efile in files:
+                            loc_to_change = os.path.join(root, efile)
+                            st = os.stat(loc_to_change)
+                            os.chmod(loc_to_change, st.st_mode | stat.S_IEXEC)
+                except Exception as e:
+                    raise Exception(
+                        "Could not set executable bit in %s, error: %s" % (path, str(e))
+                    )
+            finally:
+                os.chdir(cwd)
+
+
+def setup_browsertime(
+    command_context,
+    should_clobber=False,
+    new_upstream_url="",
+    install_vismet_reqs=False,
+):
+    r"""Install browsertime and visualmetrics.py prerequisites and the Node.js package."""
+
+    sys.path.append(mozpath.join(command_context.topsrcdir, "tools", "lint", "eslint"))
+    import setup_helper
+
+    if not new_upstream_url:
+        setup_prerequisites(command_context)
+
+    if new_upstream_url:
+        package_json_path = os.path.join(BROWSERTIME_ROOT, "package.json")
+
+        command_context.log(
+            logging.INFO,
+            "browsertime",
+            {
+                "new_upstream_url": new_upstream_url,
+                "package_json_path": package_json_path,
+            },
+            "Updating browsertime node module version in {package_json_path} "
+            "to {new_upstream_url}",
+        )
+
+        if not re.search("/tarball/[a-f0-9]{40}$", new_upstream_url):
+            raise ValueError(
+                "New upstream URL does not end with /tarball/[a-f0-9]{40}: '%s'"
+                % new_upstream_url
+            )
+
+        with open(package_json_path) as f:
+            existing_body = json.loads(
+                f.read(), object_pairs_hook=collections.OrderedDict
+            )
+
+        existing_body["devDependencies"]["browsertime"] = new_upstream_url
+
+        updated_body = json.dumps(existing_body)
+
+        with open(package_json_path, "w") as f:
+            f.write(updated_body)
+
+    # Install the browsertime Node.js requirements.
+    if not setup_helper.check_node_executables_valid():
+        return 1
+
+    # To use a custom `geckodriver`, set
+    # os.environ[b"GECKODRIVER_BASE_URL"] = bytes(url)
+    # to an endpoint with binaries named like
+    # https://github.com/sitespeedio/geckodriver/blob/master/install.js#L31.
+    if AUTOMATION:
+        os.environ["CHROMEDRIVER_SKIP_DOWNLOAD"] = "true"
+        os.environ["GECKODRIVER_SKIP_DOWNLOAD"] = "true"
+
+    if install_vismet_reqs:
+        # Hide this behind a flag so we don't install them by default in Raptor
+        command_context.log(
+            logging.INFO, "browsertime", {}, "Installing python requirements"
+        )
+        activate_browsertime_virtualenv(command_context)
+
+    command_context.log(
+        logging.INFO,
+        "browsertime",
+        {"package_json": mozpath.join(BROWSERTIME_ROOT, "package.json")},
+        "Installing browsertime node module from {package_json}",
+    )
+
+    # Add the mozbuild Node binary path to the OS environment in Apple Silicon.
+    # During the browesertime installation, it seems installation of sitespeed.io
+    # sub dependencies look for a global Node rather than the mozbuild Node binary.
+    # Normally `--scripts-prepend-node-path` should prevent this but it seems to still
+    # look for a system Node in the environment. This logic ensures the same Node is used.
+    node_dir = os.path.dirname(node_path(command_context))
+    if IS_APPLE_SILICON and node_dir not in os.environ["PATH"]:
+        os.environ["PATH"] += os.pathsep + node_dir
+
+    status = setup_helper.package_setup(
+        BROWSERTIME_ROOT,
+        "browsertime",
+        should_update=new_upstream_url != "",
+        should_clobber=should_clobber,
+        no_optional=new_upstream_url or AUTOMATION,
+    )
+
+    if status:
+        return status
+    if new_upstream_url or AUTOMATION:
+        return 0
+    if install_vismet_reqs:
+        return check(command_context)
+
+    return 0
+
+
+def node(command_context, args):
+    r"""Invoke node (interactively) with the given arguments."""
+    return command_context.run_process(
+        [node_path(command_context)] + args,
+        append_env=append_env(command_context),
+        pass_thru=True,  # Allow user to run Node interactively.
+        ensure_exit_code=False,  # Don't throw on non-zero exit code.
+        cwd=mozpath.join(command_context.topsrcdir),
+    )
+
+
+def append_env(command_context, append_path=True):
+    fetches = host_fetches[host_platform()]
+
+    # Ensure that `ffmpeg` is found and added to the environment
+    path = os.environ.get("PATH", "").split(os.pathsep) if append_path else []
+    path_to_ffmpeg = mozpath.join(
+        state_path(command_context), fetches["ffmpeg"]["path"]
+    )
+
+    path.insert(
+        0,
+        path_to_ffmpeg
+        if host_platform().startswith("linux")
+        else mozpath.join(path_to_ffmpeg, "bin"),
+    )  # noqa
+
+    # Ensure that bare `node` and `npm` in scripts, including post-install
+    # scripts, finds the binary we're invoking with.  Without this, it's
+    # easy for compiled extensions to get mismatched versions of the Node.js
+    # extension API.
+    node_dir = os.path.dirname(node_path(command_context))
+    path = [node_dir] + path
+
+    append_env = {
+        "PATH": os.pathsep.join(path),
+        # Bug 1560193: The JS library browsertime uses to execute commands
+        # (execa) will muck up the PATH variable and put the directory that
+        # node is in first in path. If this is globally-installed node,
+        # that means `/usr/bin` will be inserted first which means that we
+        # will get `/usr/bin/python` for `python`.
+        #
+        # Our fork of browsertime supports a `PYTHON` environment variable
+        # that points to the exact python executable to use.
+        "PYTHON": command_context.virtualenv_manager.python_path,
+    }
+
+    return append_env
+
+
+def _need_install(command_context, package):
+    from pip._internal.req.constructors import install_req_from_line
+
+    req = install_req_from_line(package)
+    req.check_if_exists(use_user_site=False)
+    if req.satisfied_by is None:
+        return True
+    venv_site_lib = os.path.abspath(
+        os.path.join(command_context.virtualenv_manager.bin_path, "..", "lib")
+    )
+    site_packages = os.path.abspath(req.satisfied_by.location)
+    return not site_packages.startswith(venv_site_lib)
+
+
+def activate_browsertime_virtualenv(command_context, *args, **kwargs):
+    r"""Activates virtualenv.
+
+    This function will also install Pillow and pyssim if needed.
+    It will raise an error in case the install failed.
+    """
+    # TODO: Remove `./mach browsertime` completely, as a follow up to Bug 1758990
+    MachCommandBase.activate_virtualenv(command_context, *args, **kwargs)
+
+    # installing Python deps on the fly
+    for dep in (
+        "Pillow==%s" % PILLOW_VERSION,
+        "pyssim==%s" % PYSSIM_VERSION,
+        "scipy==%s" % SCIPY_VERSION,
+        "numpy==%s" % NUMPY_VERSION,
+        "opencv-python==%s" % OPENCV_VERSION,
+    ):
+        if _need_install(command_context, dep):
+            subprocess.check_call(
+                [
+                    command_context.virtualenv_manager.python_path,
+                    "-m",
+                    "pip",
+                    "install",
+                    dep,
+                ]
+            )
+
+
+def check(command_context):
+    r"""Run `visualmetrics.py --check`."""
+    command_context.activate_virtualenv()
+
+    args = ["--check"]
+    status = command_context.run_process(
+        [command_context.virtualenv_manager.python_path, visualmetrics_path()] + args,
+        # For --check, don't allow user's path to interfere with path testing except on Linux
+        append_env=append_env(
+            command_context, append_path=host_platform().startswith("linux")
+        ),
+        pass_thru=True,
+        ensure_exit_code=False,  # Don't throw on non-zero exit code.
+        cwd=mozpath.join(command_context.topsrcdir),
+    )
+
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+    if status:
+        return status
+
+    # Avoid logging the command (and, on Windows, the environment).
+    command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL)
+    print("browsertime version:", end=" ")
+
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+    return node(command_context, [browsertime_path()] + ["--version"])
+
+
+def extra_default_args(command_context, args=[]):
+    # Add Mozilla-specific default arguments.  This is tricky because browsertime is quite
+    # loose about arguments; repeat arguments are generally accepted but then produce
+    # difficult to interpret type errors.
+
+    def extract_browser_name(args):
+        "Extracts the browser name if any"
+        # These are BT arguments, it's BT job to check them
+        # here we just want to extract the browser name
+        res = re.findall("(--browser|-b)[= ]([\w]+)", " ".join(args))
+        if res == []:
+            return None
+        return res[0][-1]
+
+    def matches(args, *flags):
+        "Return True if any argument matches any of the given flags (maybe with an argument)."
+        for flag in flags:
+            if flag in args or any(arg.startswith(flag + "=") for arg in args):
+                return True
+        return False
+
+    extra_args = []
+
+    # Default to Firefox.  Override with `-b ...` or `--browser=...`.
+    specifies_browser = matches(args, "-b", "--browser")
+    if not specifies_browser:
+        extra_args.extend(("-b", "firefox"))
+
+    # Default to not collect HAR.  Override with `--skipHar=false`.
+    specifies_har = matches(args, "--har", "--skipHar", "--gzipHar")
+    if not specifies_har:
+        extra_args.append("--skipHar")
+
+    if not matches(args, "--android"):
+        # If --firefox.binaryPath is not specified, default to the objdir binary
+        # Note: --firefox.release is not a real browsertime option, but it will
+        #       silently ignore it instead and default to a release installation.
+        specifies_binaryPath = matches(
+            args,
+            "--firefox.binaryPath",
+            "--firefox.release",
+            "--firefox.nightly",
+            "--firefox.beta",
+            "--firefox.developer",
+        )
+
+        if not specifies_binaryPath:
+            specifies_binaryPath = extract_browser_name(args) == "chrome"
+
+        if not specifies_binaryPath:
+            try:
+                extra_args.extend(
+                    ("--firefox.binaryPath", command_context.get_binary_path())
+                )
+            except BinaryNotFoundException as e:
+                command_context.log(
+                    logging.ERROR,
+                    "browsertime",
+                    {"error": str(e)},
+                    "ERROR: {error}",
+                )
+                command_context.log(
+                    logging.INFO,
+                    "browsertime",
+                    {},
+                    "Please run |./mach build| "
+                    "or specify a Firefox binary with --firefox.binaryPath.",
+                )
+                return 1
+
+    if extra_args:
+        command_context.log(
+            logging.DEBUG,
+            "browsertime",
+            {"extra_args": extra_args},
+            "Running browsertime with extra default arguments: {extra_args}",
+        )
+
+    return extra_args
+
+
+def _verify_node_install(command_context):
+    # check if Node is installed
+    sys.path.append(mozpath.join(command_context.topsrcdir, "tools", "lint", "eslint"))
+    import setup_helper
+
+    with silence():
+        node_valid = setup_helper.check_node_executables_valid()
+    if not node_valid:
+        print("Can't find Node. did you run ./mach bootstrap ?")
+        return False
+
+    # check if the browsertime package has been deployed correctly
+    # for this we just check for the browsertime directory presence
+    if not os.path.exists(browsertime_path()):
+        print("Could not find browsertime.js, try ./mach browsertime --setup")
+        print("If that still fails, try ./mach browsertime --setup --clobber")
+        return False
+
+    return True
+
+
+@Command(
+    "browsertime",
+    category="testing",
+    description="Run [browsertime](https://github.com/sitespeedio/browsertime) "
+    "performance tests.",
+)
+@CommandArgument(
+    "--verbose",
+    action="store_true",
+    help="Verbose output for what commands the build is running.",
+)
+@CommandArgument("--update-upstream-url", default="")
+@CommandArgument("--setup", default=False, action="store_true")
+@CommandArgument("--clobber", default=False, action="store_true")
+@CommandArgument(
+    "--skip-cache",
+    default=False,
+    action="store_true",
+    help="Skip all local caches to force re-fetching remote artifacts.",
+)
+@CommandArgument("--check-browsertime", default=False, action="store_true")
+@CommandArgument(
+    "--install-vismet-reqs",
+    default=False,
+    action="store_true",
+    help="Add this flag to get the visual metrics requirements installed.",
+)
+@CommandArgument(
+    "--browsertime-help",
+    default=False,
+    action="store_true",
+    help="Show the browsertime help message.",
+)
+@CommandArgument("args", nargs=argparse.REMAINDER)
+def browsertime(
+    command_context,
+    args,
+    verbose=False,
+    update_upstream_url="",
+    setup=False,
+    clobber=False,
+    skip_cache=False,
+    check_browsertime=False,
+    browsertime_help=False,
+    install_vismet_reqs=False,
+):
+    command_context._set_log_level(verbose)
+
+    # Output a message before going further to make sure the
+    # user knows that this tool is unsupported by the perftest
+    # team and point them to our supported tools. Pause a bit to
+    # make sure the user sees this message.
+    command_context.log(
+        logging.INFO,
+        "browsertime",
+        {},
+        "[INFO] This command should be used for browsertime setup only.\n"
+        "If you are looking to run performance tests on your patch, use "
+        "`./mach raptor --browsertime` instead.\n\nYou can get visual-metrics "
+        "by using the --browsertime-video and --browsertime-visualmetrics. "
+        "Here is a sample command for raptor-browsertime: \n\t`./mach raptor "
+        "--browsertime -t amazon --browsertime-video --browsertime-visualmetrics`\n\n"
+        "See this wiki page for more information if needed: "
+        "https://wiki.mozilla.org/TestEngineering/Performance/Raptor/Browsertime\n\n",
+    )
+    time.sleep(5)
+
+    if update_upstream_url:
+        return setup_browsertime(
+            command_context,
+            new_upstream_url=update_upstream_url,
+            install_vismet_reqs=install_vismet_reqs,
+        )
+    elif setup:
+        return setup_browsertime(
+            command_context,
+            should_clobber=clobber,
+            install_vismet_reqs=install_vismet_reqs,
+        )
+    else:
+        if not _verify_node_install(command_context):
+            return 1
+
+    if check_browsertime:
+        return check(command_context)
+
+    if browsertime_help:
+        args.append("--help")
+
+    activate_browsertime_virtualenv(command_context)
+    default_args = extra_default_args(command_context, args)
+    if default_args == 1:
+        return 1
+    return node(command_context, [browsertime_path()] + default_args + args)