path: root/testing/mozharness/scripts/awsy_script.py

#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""
run awsy tests in a virtualenv
"""

import copy
import json
import os
import re
import sys

# load modules from parent dir
sys.path.insert(1, os.path.dirname(sys.path[0]))

import mozharness
import mozinfo
from mozharness.base.log import ERROR, INFO
from mozharness.base.script import PreScriptAction
from mozharness.base.vcs.vcsbase import MercurialScript
from mozharness.mozilla.structuredlog import StructuredOutputParser
from mozharness.mozilla.testing.codecoverage import (
    CodeCoverageMixin,
    code_coverage_config_options,
)
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
from mozharness.mozilla.tooltool import TooltoolMixin

PY2 = sys.version_info.major == 2
scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__)))
external_tools_path = os.path.join(scripts_path, "external_tools")


class AWSY(TestingMixin, MercurialScript, TooltoolMixin, CodeCoverageMixin):
    config_options = (
        [
            [
                ["--disable-e10s"],
                {
                    "action": "store_false",
                    "dest": "e10s",
                    "default": True,
                    "help": "Run tests without multiple processes (e10s). (Desktop builds only)",
                },
            ],
            [
                ["--setpref"],
                {
                    "action": "append",
                    "dest": "extra_prefs",
                    "default": [],
                    "help": "Extra user prefs.",
                },
            ],
            [
                ["--base"],
                {
                    "action": "store_true",
                    "dest": "test_about_blank",
                    "default": False,
                    "help": "Runs the about:blank base case memory test.",
                },
            ],
            [
                ["--dmd"],
                {
                    "action": "store_true",
                    "dest": "dmd",
                    "default": False,
                    "help": "Runs tests with DMD enabled.",
                },
            ],
            [
                ["--tp6"],
                {
                    "action": "store_true",
                    "dest": "tp6",
                    "default": False,
                    "help": "Runs tests with the tp6 pageset.",
                },
            ],
        ]
        + testing_config_options
        + copy.deepcopy(code_coverage_config_options)
    )

    error_list = [
        {"regex": re.compile(r"""(TEST-UNEXPECTED|PROCESS-CRASH)"""), "level": ERROR},
    ]

    def __init__(self, **kwargs):
        kwargs.setdefault("config_options", self.config_options)
        kwargs.setdefault(
            "all_actions",
            [
                "clobber",
                "download-and-extract",
                "populate-webroot",
                "create-virtualenv",
                "install",
                "run-tests",
            ],
        )
        kwargs.setdefault(
            "default_actions",
            [
                "clobber",
                "download-and-extract",
                "populate-webroot",
                "create-virtualenv",
                "install",
                "run-tests",
            ],
        )
        kwargs.setdefault("config", {})
        super(AWSY, self).__init__(**kwargs)
        self.installer_url = self.config.get("installer_url")
        self.tests = None

        self.testdir = self.query_abs_dirs()["abs_test_install_dir"]
        self.awsy_path = os.path.join(self.testdir, "awsy")
        self.awsy_libdir = os.path.join(self.awsy_path, "awsy")
        self.webroot_dir = os.path.join(self.testdir, "html")
        self.results_dir = os.path.join(self.testdir, "results")
        self.binary_path = self.config.get("binary_path")

    def query_abs_dirs(self):
        if self.abs_dirs:
            return self.abs_dirs
        abs_dirs = super(AWSY, self).query_abs_dirs()

        dirs = {}
        dirs["abs_blob_upload_dir"] = os.path.join(
            abs_dirs["abs_work_dir"], "blobber_upload_dir"
        )
        dirs["abs_test_install_dir"] = os.path.join(abs_dirs["abs_work_dir"], "tests")
        abs_dirs.update(dirs)
        self.abs_dirs = abs_dirs
        return self.abs_dirs

    def download_and_extract(self, extract_dirs=None, suite_categories=None):
        ret = super(AWSY, self).download_and_extract(
            suite_categories=["common", "awsy"]
        )
        return ret

    @PreScriptAction("create-virtualenv")
    def _pre_create_virtualenv(self, action):
        requirements_files = [
            os.path.join(self.testdir, "config", "marionette_requirements.txt")
        ]

        for requirements_file in requirements_files:
            self.register_virtualenv_module(
                requirements=[requirements_file], two_pass=True
            )

        self.register_virtualenv_module("awsy", self.awsy_path)

    def populate_webroot(self):
        """Populate the production test machines' webroots"""
        self.info("Downloading pageset with tooltool...")
        manifest_file = os.path.join(self.awsy_path, "tp5n-pageset.manifest")
        page_load_test_dir = os.path.join(self.webroot_dir, "page_load_test")
        if not os.path.isdir(page_load_test_dir):
            self.mkdir_p(page_load_test_dir)
        self.tooltool_fetch(
            manifest_file,
            output_dir=page_load_test_dir,
            cache=self.config.get("tooltool_cache"),
        )
        archive = os.path.join(page_load_test_dir, "tp5n.zip")
        unzip = self.query_exe("unzip")
        unzip_cmd = [unzip, "-q", "-o", archive, "-d", page_load_test_dir]
        self.run_command(unzip_cmd, halt_on_failure=False)
        self.run_command("ls %s" % page_load_test_dir)

    def run_tests(self, args=None, **kw):
        """
        AWSY test should be implemented here
        """
        dirs = self.abs_dirs
        env = {}
        error_summary_file = os.path.join(
            dirs["abs_blob_upload_dir"], "marionette_errorsummary.log"
        )

        runtime_testvars = {
            "webRootDir": self.webroot_dir,
            "resultsDir": self.results_dir,
            "bin": self.binary_path,
        }

        # Check if this is a DMD build and if so enable it.
        dmd_enabled = False
        dmd_py_lib_dir = os.path.dirname(self.binary_path)
        if mozinfo.os == "mac":
            # On mac binary is in MacOS and dmd.py is in Resources, ie:
            #   Name.app/Contents/MacOS/libdmd.dylib
            #   Name.app/Contents/Resources/dmd.py
            dmd_py_lib_dir = os.path.join(dmd_py_lib_dir, "../Resources/")

        dmd_path = os.path.join(dmd_py_lib_dir, "dmd.py")
        if self.config["dmd"] and os.path.isfile(dmd_path):
            dmd_enabled = True
            runtime_testvars["dmd"] = True

            # Allow the child process to import dmd.py
            python_path = os.environ.get("PYTHONPATH")

            if python_path:
                os.environ["PYTHONPATH"] = "%s%s%s" % (
                    python_path,
                    os.pathsep,
                    dmd_py_lib_dir,
                )
            else:
                os.environ["PYTHONPATH"] = dmd_py_lib_dir

            env["DMD"] = "--mode=dark-matter --stacks=full"

        runtime_testvars["tp6"] = self.config["tp6"]
        if self.config["tp6"]:
            # mitmproxy needs path to mozharness when installing the cert, and tooltool
            env["SCRIPTSPATH"] = scripts_path
            env["EXTERNALTOOLSPATH"] = external_tools_path

        runtime_testvars_path = os.path.join(self.awsy_path, "runtime-testvars.json")
        runtime_testvars_file = open(runtime_testvars_path, "wb" if PY2 else "w")
        runtime_testvars_file.write(json.dumps(runtime_testvars, indent=2))
        runtime_testvars_file.close()

        cmd = ["marionette"]

        test_vars_file = None
        if self.config["test_about_blank"]:
            test_vars_file = "base-testvars.json"
        else:
            if self.config["tp6"]:
                test_vars_file = "tp6-testvars.json"
            else:
                test_vars_file = "testvars.json"

        cmd.append(
            "--testvars=%s" % os.path.join(self.awsy_path, "conf", test_vars_file)
        )
        cmd.append("--testvars=%s" % runtime_testvars_path)
        cmd.append("--log-raw=-")
        cmd.append("--log-errorsummary=%s" % error_summary_file)
        cmd.append("--binary=%s" % self.binary_path)
        cmd.append("--profile=%s" % (os.path.join(dirs["abs_work_dir"], "profile")))
        if not self.config["e10s"]:
            cmd.append("--disable-e10s")
        cmd.extend(["--setpref={}".format(p) for p in self.config["extra_prefs"]])
        cmd.append(
            "--gecko-log=%s" % os.path.join(dirs["abs_blob_upload_dir"], "gecko.log")
        )
        # TestingMixin._download_and_extract_symbols() should set
        # self.symbols_path
        cmd.append("--symbols-path=%s" % self.symbols_path)

        if self.config["test_about_blank"]:
            test_file = os.path.join(self.awsy_libdir, "test_base_memory_usage.py")
            prefs_file = "base-prefs.json"
        else:
            test_file = os.path.join(self.awsy_libdir, "test_memory_usage.py")
            if self.config["tp6"]:
                prefs_file = "tp6-prefs.json"
            else:
                prefs_file = "prefs.json"

        cmd.append(
            "--preferences=%s" % os.path.join(self.awsy_path, "conf", prefs_file)
        )
        if dmd_enabled:
            cmd.append("--setpref=security.sandbox.content.level=0")
        cmd.append("--setpref=layout.css.stylo-threads=4")

        cmd.append(test_file)

        env["MOZ_UPLOAD_DIR"] = dirs["abs_blob_upload_dir"]
        if not os.path.isdir(env["MOZ_UPLOAD_DIR"]):
            self.mkdir_p(env["MOZ_UPLOAD_DIR"])
        if self.query_minidump_stackwalk():
            env["MINIDUMP_STACKWALK"] = self.minidump_stackwalk_path
        env["MINIDUMP_SAVE_PATH"] = dirs["abs_blob_upload_dir"]
        env["RUST_BACKTRACE"] = "1"
        env = self.query_env(partial_env=env)
        parser = StructuredOutputParser(
            config=self.config,
            log_obj=self.log_obj,
            error_list=self.error_list,
            strict=False,
        )
        return_code = self.run_command(
            command=cmd,
            cwd=self.awsy_path,
            output_timeout=self.config.get("cmd_timeout"),
            env=env,
            output_parser=parser,
        )

        level = INFO
        tbpl_status, log_level, summary = parser.evaluate_parser(
            return_code=return_code
        )

        self.log(
            "AWSY exited with return code %s: %s" % (return_code, tbpl_status),
            level=level,
        )
        self.record_status(tbpl_status)


if __name__ == "__main__":
    awsy_test = AWSY()
    awsy_test.run_and_exit()