diff options
Diffstat (limited to 'testing/mozharness/scripts/awsy_script.py')
| -rw-r--r-- | testing/mozharness/scripts/awsy_script.py | 322 |
1 files changed, 322 insertions, 0 deletions
diff --git a/testing/mozharness/scripts/awsy_script.py b/testing/mozharness/scripts/awsy_script.py new file mode 100644 index 0000000000..9071dab75d --- /dev/null +++ b/testing/mozharness/scripts/awsy_script.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +""" +run awsy tests in a virtualenv +""" + +import copy +import json +import os +import re +import sys + +# load modules from parent dir +sys.path.insert(1, os.path.dirname(sys.path[0])) + +import mozharness +import mozinfo +from mozharness.base.log import ERROR, INFO +from mozharness.base.script import PreScriptAction +from mozharness.base.vcs.vcsbase import MercurialScript +from mozharness.mozilla.structuredlog import StructuredOutputParser +from mozharness.mozilla.testing.codecoverage import ( + CodeCoverageMixin, + code_coverage_config_options, +) +from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options +from mozharness.mozilla.tooltool import TooltoolMixin + +PY2 = sys.version_info.major == 2 +scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))) +external_tools_path = os.path.join(scripts_path, "external_tools") + + +class AWSY(TestingMixin, MercurialScript, TooltoolMixin, CodeCoverageMixin): + config_options = ( + [ + [ + ["--disable-e10s"], + { + "action": "store_false", + "dest": "e10s", + "default": True, + "help": "Run tests without multiple processes (e10s). (Desktop builds only)", + }, + ], + [ + ["--setpref"], + { + "action": "append", + "dest": "extra_prefs", + "default": [], + "help": "Extra user prefs.", + }, + ], + [ + ["--base"], + { + "action": "store_true", + "dest": "test_about_blank", + "default": False, + "help": "Runs the about:blank base case memory test.", + }, + ], + [ + ["--dmd"], + { + "action": "store_true", + "dest": "dmd", + "default": False, + "help": "Runs tests with DMD enabled.", + }, + ], + [ + ["--tp6"], + { + "action": "store_true", + "dest": "tp6", + "default": False, + "help": "Runs tests with the tp6 pageset.", + }, + ], + ] + + testing_config_options + + copy.deepcopy(code_coverage_config_options) + ) + + error_list = [ + {"regex": re.compile(r"""(TEST-UNEXPECTED|PROCESS-CRASH)"""), "level": ERROR}, + ] + + def __init__(self, **kwargs): + + kwargs.setdefault("config_options", self.config_options) + kwargs.setdefault( + "all_actions", + [ + "clobber", + "download-and-extract", + "populate-webroot", + "create-virtualenv", + "install", + "run-tests", + ], + ) + kwargs.setdefault( + "default_actions", + [ + "clobber", + "download-and-extract", + "populate-webroot", + "create-virtualenv", + "install", + "run-tests", + ], + ) + kwargs.setdefault("config", {}) + super(AWSY, self).__init__(**kwargs) + self.installer_url = self.config.get("installer_url") + self.tests = None + + self.testdir = self.query_abs_dirs()["abs_test_install_dir"] + self.awsy_path = os.path.join(self.testdir, "awsy") + self.awsy_libdir = os.path.join(self.awsy_path, "awsy") + self.webroot_dir = os.path.join(self.testdir, "html") + self.results_dir = os.path.join(self.testdir, "results") + self.binary_path = self.config.get("binary_path") + + def query_abs_dirs(self): + if self.abs_dirs: + return self.abs_dirs + abs_dirs = super(AWSY, self).query_abs_dirs() + + dirs = {} + dirs["abs_blob_upload_dir"] = os.path.join( + abs_dirs["abs_work_dir"], "blobber_upload_dir" + ) + dirs["abs_test_install_dir"] = os.path.join(abs_dirs["abs_work_dir"], "tests") + abs_dirs.update(dirs) + self.abs_dirs = abs_dirs + return self.abs_dirs + + def download_and_extract(self, extract_dirs=None, suite_categories=None): + ret = super(AWSY, self).download_and_extract( + suite_categories=["common", "awsy"] + ) + return ret + + @PreScriptAction("create-virtualenv") + def _pre_create_virtualenv(self, action): + requirements_files = [ + os.path.join(self.testdir, "config", "marionette_requirements.txt") + ] + + for requirements_file in requirements_files: + self.register_virtualenv_module( + requirements=[requirements_file], two_pass=True + ) + + self.register_virtualenv_module("awsy", self.awsy_path) + + def populate_webroot(self): + """Populate the production test machines' webroots""" + self.info("Downloading pageset with tooltool...") + manifest_file = os.path.join(self.awsy_path, "tp5n-pageset.manifest") + page_load_test_dir = os.path.join(self.webroot_dir, "page_load_test") + if not os.path.isdir(page_load_test_dir): + self.mkdir_p(page_load_test_dir) + self.tooltool_fetch( + manifest_file, + output_dir=page_load_test_dir, + cache=self.config.get("tooltool_cache"), + ) + archive = os.path.join(page_load_test_dir, "tp5n.zip") + unzip = self.query_exe("unzip") + unzip_cmd = [unzip, "-q", "-o", archive, "-d", page_load_test_dir] + self.run_command(unzip_cmd, halt_on_failure=False) + self.run_command("ls %s" % page_load_test_dir) + + def run_tests(self, args=None, **kw): + """ + AWSY test should be implemented here + """ + dirs = self.abs_dirs + env = {} + error_summary_file = os.path.join( + dirs["abs_blob_upload_dir"], "marionette_errorsummary.log" + ) + + runtime_testvars = { + "webRootDir": self.webroot_dir, + "resultsDir": self.results_dir, + "bin": self.binary_path, + } + + # Check if this is a DMD build and if so enable it. + dmd_enabled = False + dmd_py_lib_dir = os.path.dirname(self.binary_path) + if mozinfo.os == "mac": + # On mac binary is in MacOS and dmd.py is in Resources, ie: + # Name.app/Contents/MacOS/libdmd.dylib + # Name.app/Contents/Resources/dmd.py + dmd_py_lib_dir = os.path.join(dmd_py_lib_dir, "../Resources/") + + dmd_path = os.path.join(dmd_py_lib_dir, "dmd.py") + if self.config["dmd"] and os.path.isfile(dmd_path): + dmd_enabled = True + runtime_testvars["dmd"] = True + + # Allow the child process to import dmd.py + python_path = os.environ.get("PYTHONPATH") + + if python_path: + os.environ["PYTHONPATH"] = "%s%s%s" % ( + python_path, + os.pathsep, + dmd_py_lib_dir, + ) + else: + os.environ["PYTHONPATH"] = dmd_py_lib_dir + + env["DMD"] = "--mode=dark-matter --stacks=full" + + runtime_testvars["tp6"] = self.config["tp6"] + if self.config["tp6"]: + # mitmproxy needs path to mozharness when installing the cert, and tooltool + env["SCRIPTSPATH"] = scripts_path + env["EXTERNALTOOLSPATH"] = external_tools_path + + runtime_testvars_path = os.path.join(self.awsy_path, "runtime-testvars.json") + runtime_testvars_file = open(runtime_testvars_path, "wb" if PY2 else "w") + runtime_testvars_file.write(json.dumps(runtime_testvars, indent=2)) + runtime_testvars_file.close() + + cmd = ["marionette"] + + test_vars_file = None + if self.config["test_about_blank"]: + test_vars_file = "base-testvars.json" + else: + if self.config["tp6"]: + test_vars_file = "tp6-testvars.json" + else: + test_vars_file = "testvars.json" + + cmd.append( + "--testvars=%s" % os.path.join(self.awsy_path, "conf", test_vars_file) + ) + cmd.append("--testvars=%s" % runtime_testvars_path) + cmd.append("--log-raw=-") + cmd.append("--log-errorsummary=%s" % error_summary_file) + cmd.append("--binary=%s" % self.binary_path) + cmd.append("--profile=%s" % (os.path.join(dirs["abs_work_dir"], "profile"))) + if not self.config["e10s"]: + cmd.append("--disable-e10s") + cmd.extend(["--setpref={}".format(p) for p in self.config["extra_prefs"]]) + cmd.append( + "--gecko-log=%s" % os.path.join(dirs["abs_blob_upload_dir"], "gecko.log") + ) + # TestingMixin._download_and_extract_symbols() should set + # self.symbols_path + cmd.append("--symbols-path=%s" % self.symbols_path) + + if self.config["test_about_blank"]: + test_file = os.path.join(self.awsy_libdir, "test_base_memory_usage.py") + prefs_file = "base-prefs.json" + else: + test_file = os.path.join(self.awsy_libdir, "test_memory_usage.py") + if self.config["tp6"]: + prefs_file = "tp6-prefs.json" + else: + prefs_file = "prefs.json" + + cmd.append( + "--preferences=%s" % os.path.join(self.awsy_path, "conf", prefs_file) + ) + if dmd_enabled: + cmd.append("--setpref=security.sandbox.content.level=0") + cmd.append("--setpref=layout.css.stylo-threads=4") + + cmd.append(test_file) + + env["MOZ_UPLOAD_DIR"] = dirs["abs_blob_upload_dir"] + if not os.path.isdir(env["MOZ_UPLOAD_DIR"]): + self.mkdir_p(env["MOZ_UPLOAD_DIR"]) + if self.query_minidump_stackwalk(): + env["MINIDUMP_STACKWALK"] = self.minidump_stackwalk_path + env["MINIDUMP_SAVE_PATH"] = dirs["abs_blob_upload_dir"] + env["RUST_BACKTRACE"] = "1" + env = self.query_env(partial_env=env) + parser = StructuredOutputParser( + config=self.config, + log_obj=self.log_obj, + error_list=self.error_list, + strict=False, + ) + return_code = self.run_command( + command=cmd, + cwd=self.awsy_path, + output_timeout=self.config.get("cmd_timeout"), + env=env, + output_parser=parser, + ) + + level = INFO + tbpl_status, log_level, summary = parser.evaluate_parser( + return_code=return_code + ) + + self.log( + "AWSY exited with return code %s: %s" % (return_code, tbpl_status), + level=level, + ) + self.record_status(tbpl_status) + + +if __name__ == "__main__": + awsy_test = AWSY() + awsy_test.run_and_exit() |