diff options
Diffstat (limited to 'testing/mozharness/mozharness')
47 files changed, 15377 insertions, 0 deletions
diff --git a/testing/mozharness/mozharness/__init__.py b/testing/mozharness/mozharness/__init__.py new file mode 100644 index 0000000000..ab191837df --- /dev/null +++ b/testing/mozharness/mozharness/__init__.py @@ -0,0 +1,6 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +version = (0, 7) +version_string = ".".join(["%d" % i for i in version]) diff --git a/testing/mozharness/mozharness/base/__init__.py b/testing/mozharness/mozharness/base/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/mozharness/mozharness/base/__init__.py diff --git a/testing/mozharness/mozharness/base/config.py b/testing/mozharness/mozharness/base/config.py new file mode 100644 index 0000000000..d12b3aecad --- /dev/null +++ b/testing/mozharness/mozharness/base/config.py @@ -0,0 +1,693 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +"""Generic config parsing and dumping, the way I remember it from scripts +gone by. + +The config should be built from script-level defaults, overlaid by +config-file defaults, overlaid by command line options. + + (For buildbot-analogues that would be factory-level defaults, + builder-level defaults, and build request/scheduler settings.) + +The config should then be locked (set to read-only, to prevent runtime +alterations). Afterwards we should dump the config to a file that is +uploaded with the build, and can be used to debug or replicate the build +at a later time. + +TODO: + +* check_required_settings or something -- run at init, assert that + these settings are set. +""" + +import os +import socket +import sys +import time +from copy import deepcopy +from optparse import Option, OptionGroup, OptionParser + +from mozharness.base.log import CRITICAL, DEBUG, ERROR, FATAL, INFO, WARNING + +try: + from urllib2 import URLError, urlopen +except ImportError: + from urllib.error import URLError + from urllib.request import urlopen + + +try: + import simplejson as json +except ImportError: + import json + + +# optparse {{{1 +class ExtendedOptionParser(OptionParser): + """OptionParser, but with ExtendOption as the option_class.""" + + def __init__(self, **kwargs): + kwargs["option_class"] = ExtendOption + OptionParser.__init__(self, **kwargs) + + +class ExtendOption(Option): + """from http://docs.python.org/library/optparse.html?highlight=optparse#adding-new-actions""" + + ACTIONS = Option.ACTIONS + ("extend",) + STORE_ACTIONS = Option.STORE_ACTIONS + ("extend",) + TYPED_ACTIONS = Option.TYPED_ACTIONS + ("extend",) + ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + ("extend",) + + def take_action(self, action, dest, opt, value, values, parser): + if action == "extend": + lvalue = value.split(",") + values.ensure_value(dest, []).extend(lvalue) + else: + Option.take_action(self, action, dest, opt, value, values, parser) + + +def make_immutable(item): + if isinstance(item, list) or isinstance(item, tuple): + result = LockedTuple(item) + elif isinstance(item, dict): + result = ReadOnlyDict(item) + result.lock() + else: + result = item + return result + + +class LockedTuple(tuple): + def __new__(cls, items): + return tuple.__new__(cls, (make_immutable(x) for x in items)) + + def __deepcopy__(self, memo): + return [deepcopy(elem, memo) for elem in self] + + +# ReadOnlyDict {{{1 +class ReadOnlyDict(dict): + def __init__(self, dictionary): + self._lock = False + self.update(dictionary.copy()) + + def _check_lock(self): + assert not self._lock, "ReadOnlyDict is locked!" + + def lock(self): + for (k, v) in list(self.items()): + self[k] = make_immutable(v) + self._lock = True + + def __setitem__(self, *args): + self._check_lock() + return dict.__setitem__(self, *args) + + def __delitem__(self, *args): + self._check_lock() + return dict.__delitem__(self, *args) + + def clear(self, *args): + self._check_lock() + return dict.clear(self, *args) + + def pop(self, *args): + self._check_lock() + return dict.pop(self, *args) + + def popitem(self, *args): + self._check_lock() + return dict.popitem(self, *args) + + def setdefault(self, *args): + self._check_lock() + return dict.setdefault(self, *args) + + def update(self, *args): + self._check_lock() + dict.update(self, *args) + + def __deepcopy__(self, memo): + cls = self.__class__ + result = cls.__new__(cls) + memo[id(self)] = result + for k, v in list(self.__dict__.items()): + setattr(result, k, deepcopy(v, memo)) + result._lock = False + for k, v in list(self.items()): + result[k] = deepcopy(v, memo) + return result + + +DEFAULT_CONFIG_PATH = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))), + "configs", +) + + +# parse_config_file {{{1 +def parse_config_file( + file_name, quiet=False, search_path=None, config_dict_name="config" +): + """Read a config file and return a dictionary.""" + file_path = None + if os.path.exists(file_name): + file_path = file_name + else: + if not search_path: + search_path = [".", DEFAULT_CONFIG_PATH] + for path in search_path: + if os.path.exists(os.path.join(path, file_name)): + file_path = os.path.join(path, file_name) + break + else: + raise IOError("Can't find %s in %s!" % (file_name, search_path)) + if file_name.endswith(".py"): + global_dict = {} + local_dict = {} + exec( + compile(open(file_path, "rb").read(), file_path, "exec"), + global_dict, + local_dict, + ) + config = local_dict[config_dict_name] + elif file_name.endswith(".json"): + fh = open(file_path) + config = {} + json_config = json.load(fh) + config = dict(json_config) + fh.close() + else: + raise RuntimeError( + "Unknown config file type %s! (config files must end in .json or .py)" + % file_name + ) + # TODO return file_path + return config + + +def download_config_file(url, file_name): + n = 0 + attempts = 5 + sleeptime = 60 + max_sleeptime = 5 * 60 + while True: + if n >= attempts: + print( + "Failed to download from url %s after %d attempts, quiting..." + % (url, attempts) + ) + raise SystemError(-1) + try: + contents = urlopen(url, timeout=30).read() + break + except URLError as e: + print("Error downloading from url %s: %s" % (url, str(e))) + except socket.timeout as e: + print("Time out accessing %s: %s" % (url, str(e))) + except socket.error as e: + print("Socket error when accessing %s: %s" % (url, str(e))) + print("Sleeping %d seconds before retrying" % sleeptime) + time.sleep(sleeptime) + sleeptime = sleeptime * 2 + if sleeptime > max_sleeptime: + sleeptime = max_sleeptime + n += 1 + + try: + f = open(file_name, "w") + f.write(contents) + f.close() + except IOError as e: + print("Error writing downloaded contents to file %s: %s" % (file_name, str(e))) + raise SystemError(-1) + + +# BaseConfig {{{1 +class BaseConfig(object): + """Basic config setting/getting.""" + + def __init__( + self, + config=None, + initial_config_file=None, + config_options=None, + all_actions=None, + default_actions=None, + volatile_config=None, + option_args=None, + require_config_file=False, + append_env_variables_from_configs=False, + usage="usage: %prog [options]", + ): + self._config = {} + self.all_cfg_files_and_dicts = [] + self.actions = [] + self.config_lock = False + self.require_config_file = require_config_file + # It allows to append env variables from multiple config files + self.append_env_variables_from_configs = append_env_variables_from_configs + + if all_actions: + self.all_actions = all_actions[:] + else: + self.all_actions = ["clobber", "build"] + if default_actions: + self.default_actions = default_actions[:] + else: + self.default_actions = self.all_actions[:] + if volatile_config is None: + self.volatile_config = { + "actions": None, + "add_actions": None, + "no_actions": None, + } + else: + self.volatile_config = deepcopy(volatile_config) + + if config: + self.set_config(config) + if initial_config_file: + initial_config = parse_config_file(initial_config_file) + self.all_cfg_files_and_dicts.append((initial_config_file, initial_config)) + self.set_config(initial_config) + # Since initial_config_file is only set when running unit tests, + # if no option_args have been specified, then the parser will + # parse sys.argv which in this case would be the command line + # options specified to run the tests, e.g. nosetests -v. Clearly, + # the options passed to nosetests (such as -v) should not be + # interpreted by mozharness as mozharness options, so we specify + # a dummy command line with no options, so that the parser does + # not add anything from the test invocation command line + # arguments to the mozharness options. + if option_args is None: + option_args = [ + "dummy_mozharness_script_with_no_command_line_options.py" + ] + if config_options is None: + config_options = [] + self._create_config_parser(config_options, usage) + # we allow manually passing of option args for things like nosetests + self.parse_args(args=option_args) + + def get_read_only_config(self): + return ReadOnlyDict(self._config) + + def _create_config_parser(self, config_options, usage): + self.config_parser = ExtendedOptionParser(usage=usage) + self.config_parser.add_option( + "--work-dir", + action="store", + dest="work_dir", + type="string", + default="build", + help="Specify the work_dir (subdir of base_work_dir)", + ) + self.config_parser.add_option( + "--base-work-dir", + action="store", + dest="base_work_dir", + type="string", + default=os.getcwd(), + help="Specify the absolute path of the parent of the working directory", + ) + self.config_parser.add_option( + "--extra-config-path", + action="extend", + dest="config_paths", + type="string", + help="Specify additional paths to search for config files.", + ) + self.config_parser.add_option( + "-c", + "--config-file", + "--cfg", + action="extend", + dest="config_files", + default=[], + type="string", + help="Specify a config file; can be repeated", + ) + self.config_parser.add_option( + "-C", + "--opt-config-file", + "--opt-cfg", + action="extend", + dest="opt_config_files", + type="string", + default=[], + help="Specify an optional config file, like --config-file but with no " + "error if the file is missing; can be repeated", + ) + self.config_parser.add_option( + "--dump-config", + action="store_true", + dest="dump_config", + help="List and dump the config generated from this run to " "a JSON file.", + ) + self.config_parser.add_option( + "--dump-config-hierarchy", + action="store_true", + dest="dump_config_hierarchy", + help="Like --dump-config but will list and dump which config " + "files were used making up the config and specify their own " + "keys/values that were not overwritten by another cfg -- " + "held the highest hierarchy.", + ) + self.config_parser.add_option( + "--append-env-variables-from-configs", + action="store_true", + dest="append_env_variables_from_configs", + help="Merge environment variables from config files.", + ) + + # Logging + log_option_group = OptionGroup(self.config_parser, "Logging") + log_option_group.add_option( + "--log-level", + action="store", + type="choice", + dest="log_level", + default=INFO, + choices=[DEBUG, INFO, WARNING, ERROR, CRITICAL, FATAL], + help="Set log level (debug|info|warning|error|critical|fatal)", + ) + log_option_group.add_option( + "-q", + "--quiet", + action="store_false", + dest="log_to_console", + default=True, + help="Don't log to the console", + ) + log_option_group.add_option( + "--append-to-log", + action="store_true", + dest="append_to_log", + default=False, + help="Append to the log", + ) + log_option_group.add_option( + "--multi-log", + action="store_const", + const="multi", + dest="log_type", + help="Log using MultiFileLogger", + ) + log_option_group.add_option( + "--simple-log", + action="store_const", + const="simple", + dest="log_type", + help="Log using SimpleFileLogger", + ) + self.config_parser.add_option_group(log_option_group) + + # Actions + action_option_group = OptionGroup( + self.config_parser, + "Actions", + "Use these options to list or enable/disable actions.", + ) + action_option_group.add_option( + "--list-actions", + action="store_true", + dest="list_actions", + help="List all available actions, then exit", + ) + action_option_group.add_option( + "--add-action", + action="extend", + dest="add_actions", + metavar="ACTIONS", + help="Add action %s to the list of actions" % self.all_actions, + ) + action_option_group.add_option( + "--no-action", + action="extend", + dest="no_actions", + metavar="ACTIONS", + help="Don't perform action", + ) + action_option_group.add_option( + "--requires-gpu", + action="store_true", + dest="requires_gpu", + default=False, + help="Indicates if the task requires gpu. ", + ) + for action in self.all_actions: + action_option_group.add_option( + "--%s" % action, + action="append_const", + dest="actions", + const=action, + help="Add %s to the limited list of actions" % action, + ) + action_option_group.add_option( + "--no-%s" % action, + action="append_const", + dest="no_actions", + const=action, + help="Remove %s from the list of actions to perform" % action, + ) + self.config_parser.add_option_group(action_option_group) + # Child-specified options + # TODO error checking for overlapping options + if config_options: + for option in config_options: + self.config_parser.add_option(*option[0], **option[1]) + + # Initial-config-specified options + config_options = self._config.get("config_options", None) + if config_options: + for option in config_options: + self.config_parser.add_option(*option[0], **option[1]) + + def set_config(self, config, overwrite=False): + """This is probably doable some other way.""" + if self._config and not overwrite: + self._config.update(config) + else: + self._config = config + return self._config + + def get_actions(self): + return self.actions + + def verify_actions(self, action_list, quiet=False): + for action in action_list: + if action not in self.all_actions: + if not quiet: + print("Invalid action %s not in %s!" % (action, self.all_actions)) + raise SystemExit(-1) + return action_list + + def verify_actions_order(self, action_list): + try: + indexes = [self.all_actions.index(elt) for elt in action_list] + sorted_indexes = sorted(indexes) + for i in range(len(indexes)): + if indexes[i] != sorted_indexes[i]: + print( + ("Action %s comes in different order in %s\n" + "than in %s") + % (action_list[i], action_list, self.all_actions) + ) + raise SystemExit(-1) + except ValueError as e: + print("Invalid action found: " + str(e)) + raise SystemExit(-1) + + def list_actions(self): + print("Actions available:") + for a in self.all_actions: + print(" " + ("*" if a in self.default_actions else " "), a) + raise SystemExit(0) + + def get_cfgs_from_files(self, all_config_files, options): + """Returns the configuration derived from the list of configuration + files. The result is represented as a list of `(filename, + config_dict)` tuples; they will be combined with keys in later + dictionaries taking precedence over earlier. + + `all_config_files` is all files specified with `--config-file` and + `--opt-config-file`; `options` is the argparse options object giving + access to any other command-line options. + + This function is also responsible for downloading any configuration + files specified by URL. It uses ``parse_config_file`` in this module + to parse individual files. + + This method can be overridden in a subclass to add extra logic to the + way that self.config is made up. See + `mozharness.mozilla.building.buildbase.BuildingConfig` for an example. + """ + config_paths = options.config_paths or ["."] + all_cfg_files_and_dicts = [] + for cf in all_config_files: + try: + if "://" in cf: # config file is an url + file_name = os.path.basename(cf) + file_path = os.path.join(os.getcwd(), file_name) + download_config_file(cf, file_path) + all_cfg_files_and_dicts.append( + ( + file_path, + parse_config_file( + file_path, + search_path=["."], + ), + ) + ) + else: + all_cfg_files_and_dicts.append( + ( + cf, + parse_config_file( + cf, + search_path=config_paths + [DEFAULT_CONFIG_PATH], + ), + ) + ) + except Exception: + if cf in options.opt_config_files: + print("WARNING: optional config file not found %s" % cf) + else: + raise + + if "EXTRA_MOZHARNESS_CONFIG" in os.environ: + env_config = json.loads(os.environ["EXTRA_MOZHARNESS_CONFIG"]) + all_cfg_files_and_dicts.append(("[EXTRA_MOZHARENSS_CONFIG]", env_config)) + + return all_cfg_files_and_dicts + + def parse_args(self, args=None): + """Parse command line arguments in a generic way. + Return the parser object after adding the basic options, so + child objects can manipulate it. + """ + self.command_line = " ".join(sys.argv) + if args is None: + args = sys.argv[1:] + (options, args) = self.config_parser.parse_args(args) + + defaults = self.config_parser.defaults.copy() + + if not options.config_files: + if self.require_config_file: + if options.list_actions: + self.list_actions() + print("Required config file not set! (use --config-file option)") + raise SystemExit(-1) + + os.environ["REQUIRE_GPU"] = "0" + if options.requires_gpu: + os.environ["REQUIRE_GPU"] = "1" + + # this is what get_cfgs_from_files returns. It will represent each + # config file name and its assoctiated dict + # eg ('builds/branch_specifics.py', {'foo': 'bar'}) + # let's store this to self for things like --interpret-config-files + self.all_cfg_files_and_dicts.extend( + self.get_cfgs_from_files( + # append opt_config to allow them to overwrite previous configs + options.config_files + options.opt_config_files, + options=options, + ) + ) + config = {} + if ( + self.append_env_variables_from_configs + or options.append_env_variables_from_configs + ): + # We only append values from various configs for the 'env' entry + # For everything else we follow the standard behaviour + for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts): + for v in list(c_dict.keys()): + if v == "env" and v in config: + config[v].update(c_dict[v]) + else: + config[v] = c_dict[v] + else: + for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts): + config.update(c_dict) + # assign or update self._config depending on if it exists or not + # NOTE self._config will be passed to ReadOnlyConfig's init -- a + # dict subclass with immutable locking capabilities -- and serve + # as the keys/values that make up that instance. Ultimately, + # this becomes self.config during BaseScript's init + self.set_config(config) + + for key in list(defaults.keys()): + value = getattr(options, key) + if value is None: + continue + # Don't override config_file defaults with config_parser defaults + if key in defaults and value == defaults[key] and key in self._config: + continue + self._config[key] = value + + # The idea behind the volatile_config is we don't want to save this + # info over multiple runs. This defaults to the action-specific + # config options, but can be anything. + for key in list(self.volatile_config.keys()): + if self._config.get(key) is not None: + self.volatile_config[key] = self._config[key] + del self._config[key] + + self.update_actions() + if options.list_actions: + self.list_actions() + + # Keep? This is for saving the volatile config in the dump_config + self._config["volatile_config"] = self.volatile_config + + self.options = options + self.args = args + return (self.options, self.args) + + def update_actions(self): + """Update actions after reading in config. + + Seems a little complex, but the logic goes: + + First, if default_actions is specified in the config, set our + default actions even if the script specifies other default actions. + + Without any other action-specific options, run with default actions. + + If we specify --ACTION or --only-ACTION once or multiple times, + we want to override the default_actions list with the one(s) we list. + + Otherwise, if we specify --add-action ACTION, we want to add an + action to the list. + + Finally, if we specify --no-ACTION, remove that from the list of + actions to perform. + """ + if self._config.get("default_actions"): + default_actions = self.verify_actions(self._config["default_actions"]) + self.default_actions = default_actions + self.verify_actions_order(self.default_actions) + self.actions = self.default_actions[:] + if self.volatile_config["actions"]: + actions = self.verify_actions(self.volatile_config["actions"]) + self.actions = actions + elif self.volatile_config["add_actions"]: + actions = self.verify_actions(self.volatile_config["add_actions"]) + self.actions.extend(actions) + if self.volatile_config["no_actions"]: + actions = self.verify_actions(self.volatile_config["no_actions"]) + for action in actions: + if action in self.actions: + self.actions.remove(action) + + +# __main__ {{{1 +if __name__ == "__main__": + pass diff --git a/testing/mozharness/mozharness/base/diskutils.py b/testing/mozharness/mozharness/base/diskutils.py new file mode 100644 index 0000000000..757a6ffb7a --- /dev/null +++ b/testing/mozharness/mozharness/base/diskutils.py @@ -0,0 +1,170 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Disk utility module, no mixins here! + + examples: + 1) get disk size + from mozharness.base.diskutils import DiskInfo, DiskutilsError + ... + try: + DiskSize().get_size(path='/', unit='Mb') + except DiskutilsError as e: + # manage the exception e.g: log.error(e) + pass + log.info("%s" % di) + + + 2) convert disk size: + from mozharness.base.diskutils import DiskutilsError, convert_to + ... + file_size = <function that gets file size in bytes> + # convert file_size to GB + try: + file_size = convert_to(file_size, from_unit='bytes', to_unit='GB') + except DiskutilsError as e: + # manage the exception e.g: log.error(e) + pass + +""" +import ctypes +import logging +import os +import sys + +from six import string_types + +from mozharness.base.log import INFO, numeric_log_level + +# use mozharness log +log = logging.getLogger(__name__) + + +class DiskutilsError(Exception): + """Exception thrown by Diskutils module""" + + pass + + +def convert_to(size, from_unit, to_unit): + """Helper method to convert filesystem sizes to kB/ MB/ GB/ TB/ + valid values for source_format and destination format are: + * bytes + * kB + * MB + * GB + * TB + returns: size converted from source_format to destination_format. + """ + sizes = { + "bytes": 1, + "kB": 1024, + "MB": 1024 * 1024, + "GB": 1024 * 1024 * 1024, + "TB": 1024 * 1024 * 1024 * 1024, + } + try: + df = sizes[to_unit] + sf = sizes[from_unit] + # pylint --py3k W1619 + return size * sf / df + except KeyError: + raise DiskutilsError("conversion error: Invalid source or destination format") + except TypeError: + raise DiskutilsError("conversion error: size (%s) is not a number" % size) + + +class DiskInfo(object): + """Stores basic information about the disk""" + + def __init__(self): + self.unit = "bytes" + self.free = 0 + self.used = 0 + self.total = 0 + + def __str__(self): + string = ["Disk space info (in %s)" % self.unit] + string += ["total: %s" % self.total] + string += ["used: %s" % self.used] + string += ["free: %s" % self.free] + return " ".join(string) + + def _to(self, unit): + from_unit = self.unit + to_unit = unit + self.free = convert_to(self.free, from_unit=from_unit, to_unit=to_unit) + self.used = convert_to(self.used, from_unit=from_unit, to_unit=to_unit) + self.total = convert_to(self.total, from_unit=from_unit, to_unit=to_unit) + self.unit = unit + + +class DiskSize(object): + """DiskSize object""" + + @staticmethod + def _posix_size(path): + """returns the disk size in bytes + disk size is relative to path + """ + # we are on a POSIX system + st = os.statvfs(path) + disk_info = DiskInfo() + disk_info.free = st.f_bavail * st.f_frsize + disk_info.used = (st.f_blocks - st.f_bfree) * st.f_frsize + disk_info.total = st.f_blocks * st.f_frsize + return disk_info + + @staticmethod + def _windows_size(path): + """returns size in bytes, works only on windows platforms""" + # we're on a non POSIX system (windows) + # DLL call + disk_info = DiskInfo() + dummy = ctypes.c_ulonglong() # needed by the dll call but not used + total = ctypes.c_ulonglong() # stores the total space value + free = ctypes.c_ulonglong() # stores the free space value + # depending on path format (unicode or not) and python version (2 or 3) + # we need to call GetDiskFreeSpaceExW or GetDiskFreeSpaceExA + called_function = ctypes.windll.kernel32.GetDiskFreeSpaceExA + if isinstance(path, string_types) or sys.version_info >= (3,): + called_function = ctypes.windll.kernel32.GetDiskFreeSpaceExW + # we're ready for the dll call. On error it returns 0 + if ( + called_function( + path, ctypes.byref(dummy), ctypes.byref(total), ctypes.byref(free) + ) + != 0 + ): + # success, we can use the values returned by the dll call + disk_info.free = free.value + disk_info.total = total.value + disk_info.used = total.value - free.value + return disk_info + + @staticmethod + def get_size(path, unit, log_level=INFO): + """Disk info stats: + total => size of the disk + used => space used + free => free space + In case of error raises a DiskutilError Exception + """ + try: + # let's try to get the disk size using os module + disk_info = DiskSize()._posix_size(path) + except AttributeError: + try: + # os module failed. let's try to get the size using + # ctypes.windll... + disk_info = DiskSize()._windows_size(path) + except AttributeError: + # No luck! This is not a posix nor window platform + # raise an exception + raise DiskutilsError("Unsupported platform") + + disk_info._to(unit) + lvl = numeric_log_level(log_level) + log.log(lvl, msg="%s" % disk_info) + return disk_info diff --git a/testing/mozharness/mozharness/base/errors.py b/testing/mozharness/mozharness/base/errors.py new file mode 100755 index 0000000000..814dd2e045 --- /dev/null +++ b/testing/mozharness/mozharness/base/errors.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +"""Generic error lists. + +Error lists are used to parse output in mozharness.base.log.OutputParser. + +Each line of output is matched against each substring or regular expression +in the error list. On a match, we determine the 'level' of that line, +whether IGNORE, DEBUG, INFO, WARNING, ERROR, CRITICAL, or FATAL. + +TODO: Context lines (requires work on the OutputParser side) + +TODO: We could also create classes that generate these, but with the +appropriate level (please don't die on any errors; please die on any +warning; etc.) or platform or language or whatever. +""" + +import re + +from mozharness.base.log import CRITICAL, DEBUG, ERROR, WARNING + + +# Exceptions +class VCSException(Exception): + pass + + +# ErrorLists {{{1 +BaseErrorList = [{"substr": r"""command not found""", "level": ERROR}] + +HgErrorList = BaseErrorList + [ + { + "regex": re.compile(r"""^abort:"""), + "level": ERROR, + "explanation": "Automation Error: hg not responding", + }, + { + "substr": r"""unknown exception encountered""", + "level": ERROR, + "explanation": "Automation Error: python exception in hg", + }, + { + "substr": r"""failed to import extension""", + "level": WARNING, + "explanation": "Automation Error: hg extension missing", + }, +] + +GitErrorList = BaseErrorList + [ + {"substr": r"""Permission denied (publickey).""", "level": ERROR}, + {"substr": r"""fatal: The remote end hung up unexpectedly""", "level": ERROR}, + {"substr": r"""does not appear to be a git repository""", "level": ERROR}, + {"substr": r"""error: src refspec""", "level": ERROR}, + {"substr": r"""invalid author/committer line -""", "level": ERROR}, + {"substr": r"""remote: fatal: Error in object""", "level": ERROR}, + { + "substr": r"""fatal: sha1 file '<stdout>' write error: Broken pipe""", + "level": ERROR, + }, + {"substr": r"""error: failed to push some refs to """, "level": ERROR}, + {"substr": r"""remote: error: denying non-fast-forward """, "level": ERROR}, + {"substr": r"""! [remote rejected] """, "level": ERROR}, + {"regex": re.compile(r"""remote:.*No such file or directory"""), "level": ERROR}, +] + +PythonErrorList = BaseErrorList + [ + {"regex": re.compile(r"""Warning:.*Error: """), "level": WARNING}, + {"regex": re.compile(r"""package.*> Error:"""), "level": ERROR}, + {"substr": r"""Traceback (most recent call last)""", "level": ERROR}, + {"substr": r"""SyntaxError: """, "level": ERROR}, + {"substr": r"""TypeError: """, "level": ERROR}, + {"substr": r"""NameError: """, "level": ERROR}, + {"substr": r"""ZeroDivisionError: """, "level": ERROR}, + {"regex": re.compile(r"""raise \w*Exception: """), "level": CRITICAL}, + {"regex": re.compile(r"""raise \w*Error: """), "level": CRITICAL}, +] + +VirtualenvErrorList = [ + {"substr": r"""not found or a compiler error:""", "level": WARNING}, + {"regex": re.compile("""\d+: error: """), "level": ERROR}, + {"regex": re.compile("""\d+: warning: """), "level": WARNING}, + { + "regex": re.compile(r"""Downloading .* \(.*\): *([0-9]+%)? *[0-9\.]+[kmKM]b"""), + "level": DEBUG, + }, +] + PythonErrorList + +RustErrorList = [ + {"regex": re.compile(r"""error\[E\d+\]:"""), "level": ERROR}, + {"substr": r"""error: Could not compile""", "level": ERROR}, + {"substr": r"""error: aborting due to previous error""", "level": ERROR}, + {"substr": r"""thread 'main' panicked at""", "level": ERROR}, +] + +# We may need to have various MakefileErrorLists for differing amounts of +# warning-ignoring-ness. +MakefileErrorList = ( + BaseErrorList + + PythonErrorList + + RustErrorList + + [ + {"substr": r""": error: """, "level": ERROR}, + {"substr": r"""No rule to make target """, "level": ERROR}, + {"regex": re.compile(r"""akefile.*was not found\."""), "level": ERROR}, + {"regex": re.compile(r"""Stop\.$"""), "level": ERROR}, + { + "regex": re.compile(r"""make\[\d+\]: \*\*\* \[.*\] Error \d+"""), + "level": ERROR, + }, + {"regex": re.compile(r""":\d+: warning:"""), "level": WARNING}, + {"regex": re.compile(r"""make(?:\[\d+\])?: \*\*\*/"""), "level": ERROR}, + {"substr": r"""Warning: """, "level": WARNING}, + ] +) + +TarErrorList = BaseErrorList + [ + {"substr": r"""(stdin) is not a bzip2 file.""", "level": ERROR}, + {"regex": re.compile(r"""Child returned status [1-9]"""), "level": ERROR}, + {"substr": r"""Error exit delayed from previous errors""", "level": ERROR}, + {"substr": r"""stdin: unexpected end of file""", "level": ERROR}, + {"substr": r"""stdin: not in gzip format""", "level": ERROR}, + {"substr": r"""Cannot exec: No such file or directory""", "level": ERROR}, + {"substr": r""": Error is not recoverable: exiting now""", "level": ERROR}, +] + +ZipErrorList = BaseErrorList + [ + { + "substr": r"""zip warning:""", + "level": WARNING, + }, + { + "substr": r"""zip error:""", + "level": ERROR, + }, + { + "substr": r"""Cannot open file: it does not appear to be a valid archive""", + "level": ERROR, + }, +] + +ZipalignErrorList = BaseErrorList + [ + { + "regex": re.compile(r"""Unable to open .* as a zip archive"""), + "level": ERROR, + }, + { + "regex": re.compile(r"""Output file .* exists"""), + "level": ERROR, + }, + { + "substr": r"""Input and output can't be the same file""", + "level": ERROR, + }, +] + + +# __main__ {{{1 +if __name__ == "__main__": + """TODO: unit tests.""" + pass diff --git a/testing/mozharness/mozharness/base/log.py b/testing/mozharness/mozharness/base/log.py new file mode 100755 index 0000000000..3276696751 --- /dev/null +++ b/testing/mozharness/mozharness/base/log.py @@ -0,0 +1,783 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +"""Generic logging classes and functionalities for single and multi file logging. +Capturing console output and providing general logging functionalities. + +Attributes: + FATAL_LEVEL (int): constant logging level value set based on the logging.CRITICAL + value + DEBUG (str): mozharness `debug` log name + INFO (str): mozharness `info` log name + WARNING (str): mozharness `warning` log name + CRITICAL (str): mozharness `critical` log name + FATAL (str): mozharness `fatal` log name + IGNORE (str): mozharness `ignore` log name + LOG_LEVELS (dict): mapping of the mozharness log level names to logging values + ROOT_LOGGER (logging.Logger): instance of a logging.Logger class + +TODO: +- network logging support. +- log rotation config +""" + +import logging +import os +import sys +import time +import traceback +from datetime import datetime + +from six import binary_type + +# Define our own FATAL_LEVEL +FATAL_LEVEL = logging.CRITICAL + 10 +logging.addLevelName(FATAL_LEVEL, "FATAL") + +# mozharness log levels. +DEBUG, INFO, WARNING, ERROR, CRITICAL, FATAL, IGNORE = ( + "debug", + "info", + "warning", + "error", + "critical", + "fatal", + "ignore", +) + + +LOG_LEVELS = { + DEBUG: logging.DEBUG, + INFO: logging.INFO, + WARNING: logging.WARNING, + ERROR: logging.ERROR, + CRITICAL: logging.CRITICAL, + FATAL: FATAL_LEVEL, +} + +# mozharness root logger +ROOT_LOGGER = logging.getLogger() + +# Force logging to use UTC timestamps +logging.Formatter.converter = time.gmtime + + +# LogMixin {{{1 +class LogMixin(object): + """This is a mixin for any object to access similar logging functionality + + The logging functionality described here is specially useful for those + objects with self.config and self.log_obj member variables + """ + + def _log_level_at_least(self, level): + """Check if the current logging level is greater or equal than level + + Args: + level (str): log level name to compare against mozharness log levels + names + + Returns: + bool: True if the current logging level is great or equal than level, + False otherwise + """ + log_level = INFO + levels = [DEBUG, INFO, WARNING, ERROR, CRITICAL, FATAL] + if hasattr(self, "config"): + log_level = self.config.get("log_level", INFO) + return levels.index(level) >= levels.index(log_level) + + def _print(self, message, stderr=False): + """prints a message to the sys.stdout or sys.stderr according to the + value of the stderr argument. + + Args: + message (str): The message to be printed + stderr (bool, optional): if True, message will be printed to + sys.stderr. Defaults to False. + + Returns: + None + """ + if not hasattr(self, "config") or self.config.get("log_to_console", True): + if stderr: + print(message, file=sys.stderr) + else: + print(message) + + def log(self, message, level=INFO, exit_code=-1): + """log the message passed to it according to level, exit if level == FATAL + + Args: + message (str): message to be logged + level (str, optional): logging level of the message. Defaults to INFO + exit_code (int, optional): exit code to log before the scripts calls + SystemExit. + + Returns: + None + """ + if self.log_obj: + return self.log_obj.log_message( + message, + level=level, + exit_code=exit_code, + post_fatal_callback=self._post_fatal, + ) + if level == INFO: + if self._log_level_at_least(level): + self._print(message) + elif level == DEBUG: + if self._log_level_at_least(level): + self._print("DEBUG: %s" % message) + elif level in (WARNING, ERROR, CRITICAL): + if self._log_level_at_least(level): + self._print("%s: %s" % (level.upper(), message), stderr=True) + elif level == FATAL: + if self._log_level_at_least(level): + self._print("FATAL: %s" % message, stderr=True) + raise SystemExit(exit_code) + + def worst_level(self, target_level, existing_level, levels=None): + """Compare target_level with existing_level according to levels values + and return the worst among them. + + Args: + target_level (str): minimum logging level to which the current object + should be set + existing_level (str): current logging level + levels (list(str), optional): list of logging levels names to compare + target_level and existing_level against. + Defaults to mozharness log level + list sorted from most to less critical. + + Returns: + str: the logging lavel that is closest to the first levels value, + i.e. levels[0] + """ + if not levels: + levels = [FATAL, CRITICAL, ERROR, WARNING, INFO, DEBUG, IGNORE] + if target_level not in levels: + self.fatal("'%s' not in %s'." % (target_level, levels)) + for l in levels: + if l in (target_level, existing_level): + return l + + # Copying Bear's dumpException(): + # https://hg.mozilla.org/build/tools/annotate/1485f23c38e0/sut_tools/sut_lib.py#l23 + def exception(self, message=None, level=ERROR): + """log an exception message base on the log level passed to it. + + This function fetches the information of the current exception being handled and + adds it to the message argument. + + Args: + message (str, optional): message to be printed at the beginning of the log. + Default to an empty string. + level (str, optional): log level to use for the logging. Defaults to ERROR + + Returns: + None + """ + tb_type, tb_value, tb_traceback = sys.exc_info() + if message is None: + message = "" + else: + message = "%s\n" % message + for s in traceback.format_exception(tb_type, tb_value, tb_traceback): + message += "%s\n" % s + # Log at the end, as a fatal will attempt to exit after the 1st line. + self.log(message, level=level) + + def debug(self, message): + """calls the log method with DEBUG as logging level + + Args: + message (str): message to log + """ + self.log(message, level=DEBUG) + + def info(self, message): + """calls the log method with INFO as logging level + + Args: + message (str): message to log + """ + self.log(message, level=INFO) + + def warning(self, message): + """calls the log method with WARNING as logging level + + Args: + message (str): message to log + """ + self.log(message, level=WARNING) + + def error(self, message): + """calls the log method with ERROR as logging level + + Args: + message (str): message to log + """ + self.log(message, level=ERROR) + + def critical(self, message): + """calls the log method with CRITICAL as logging level + + Args: + message (str): message to log + """ + self.log(message, level=CRITICAL) + + def fatal(self, message, exit_code=-1): + """calls the log method with FATAL as logging level + + Args: + message (str): message to log + exit_code (int, optional): exit code to use for the SystemExit + exception to be raised. Default to -1. + """ + self.log(message, level=FATAL, exit_code=exit_code) + + def _post_fatal(self, message=None, exit_code=None): + """Sometimes you want to create a report or cleanup + or notify on fatal(); override this method to do so. + + Please don't use this for anything significantly long-running. + + Args: + message (str, optional): message to report. Default to None + exit_code (int, optional): exit code to use for the SystemExit + exception to be raised. Default to None + """ + pass + + +# OutputParser {{{1 +class OutputParser(LogMixin): + """Helper object to parse command output. + + This will buffer output if needed, so we can go back and mark + [(linenum - 10) : linenum+10] as errors if need be, without having to + get all the output first. + + linenum+10 will be easy; we can set self.num_post_context_lines to 10, + and self.num_post_context_lines-- as we mark each line to at least error + level X. + + linenum-10 will be trickier. We'll not only need to save the line + itself, but also the level that we've set for that line previously, + whether by matching on that line, or by a previous line's context. + We should only log that line if all output has ended (self.finish() ?); + otherwise store a list of dictionaries in self.context_buffer that is + buffered up to self.num_pre_context_lines (set to the largest + pre-context-line setting in error_list.) + """ + + def __init__( + self, config=None, log_obj=None, error_list=None, log_output=True, **kwargs + ): + """Initialization method for the OutputParser class + + Args: + config (dict, optional): dictionary containing values such as `log_level` + or `log_to_console`. Defaults to `None`. + log_obj (BaseLogger, optional): instance of the BaseLogger class. Defaults + to `None`. + error_list (list, optional): list of the error to look for. Defaults to + `None`. + log_output (boolean, optional): flag for deciding if the commands + output should be logged or not. + Defaults to `True`. + """ + self.config = config + self.log_obj = log_obj + self.error_list = error_list or [] + self.log_output = log_output + self.num_errors = 0 + self.num_warnings = 0 + # TODO context_lines. + # Not in use yet, but will be based off error_list. + self.context_buffer = [] + self.num_pre_context_lines = 0 + self.num_post_context_lines = 0 + self.worst_log_level = INFO + + def parse_single_line(self, line): + """parse a console output line and check if it matches one in `error_list`, + if so then log it according to `log_output`. + + Args: + line (str): command line output to parse. + + Returns: + If the line hits a match in the error_list, the new log level the line was + (or should be) logged at is returned. Otherwise, returns None. + """ + for error_check in self.error_list: + # TODO buffer for context_lines. + match = False + if "substr" in error_check: + if error_check["substr"] in line: + match = True + elif "regex" in error_check: + if error_check["regex"].search(line): + match = True + else: + self.warning("error_list: 'substr' and 'regex' not in %s" % error_check) + if match: + log_level = error_check.get("level", INFO) + if self.log_output: + message = " %s" % line + if error_check.get("explanation"): + message += "\n %s" % error_check["explanation"] + if error_check.get("summary"): + self.add_summary(message, level=log_level) + else: + self.log(message, level=log_level) + if log_level in (ERROR, CRITICAL, FATAL): + self.num_errors += 1 + if log_level == WARNING: + self.num_warnings += 1 + self.worst_log_level = self.worst_level(log_level, self.worst_log_level) + return log_level + + if self.log_output: + self.info(" %s" % line) + + def add_lines(self, output): + """process a string or list of strings, decode them to utf-8,strip + them of any trailing whitespaces and parse them using `parse_single_line` + + strings consisting only of whitespaces are ignored. + + Args: + output (str | list): string or list of string to parse + """ + if not isinstance(output, list): + output = [output] + + for line in output: + if not line or line.isspace(): + continue + + if isinstance(line, binary_type): + line = line.decode("utf-8", "replace") + + line = line.rstrip() + self.parse_single_line(line) + + +# BaseLogger {{{1 +class BaseLogger(object): + """Base class in charge of logging handling logic such as creating logging + files, dirs, attaching to the console output and managing its output. + + Attributes: + LEVELS (dict): flat copy of the `LOG_LEVELS` attribute of the `log` module. + + TODO: status? There may be a status object or status capability in + either logging or config that allows you to count the number of + error,critical,fatal messages for us to count up at the end (aiming + for 0). + """ + + LEVELS = LOG_LEVELS + + def __init__( + self, + log_level=INFO, + log_format="%(message)s", + log_date_format="%H:%M:%S", + log_name="test", + log_to_console=True, + log_dir=".", + log_to_raw=False, + logger_name="", + append_to_log=False, + ): + """BaseLogger constructor + + Args: + log_level (str, optional): mozharness log level name. Defaults to INFO. + log_format (str, optional): message format string to instantiate a + `logging.Formatter`. Defaults to '%(message)s' + log_date_format (str, optional): date format string to instantiate a + `logging.Formatter`. Defaults to '%H:%M:%S' + log_name (str, optional): name to use for the log files to be created. + Defaults to 'test' + log_to_console (bool, optional): set to True in order to create a Handler + instance base on the `Logger` + current instance. Defaults to True. + log_dir (str, optional): directory location to store the log files. + Defaults to '.', i.e. current working directory. + log_to_raw (bool, optional): set to True in order to create a *raw.log + file. Defaults to False. + logger_name (str, optional): currently useless parameter. According + to the code comments, it could be useful + if we were to have multiple logging + objects that don't trample each other. + append_to_log (bool, optional): set to True if the logging content should + be appended to old logging files. Defaults to False + """ + + self.log_format = log_format + self.log_date_format = log_date_format + self.log_to_console = log_to_console + self.log_to_raw = log_to_raw + self.log_level = log_level + self.log_name = log_name + self.log_dir = log_dir + self.append_to_log = append_to_log + + # Not sure what I'm going to use this for; useless unless we + # can have multiple logging objects that don't trample each other + self.logger_name = logger_name + + self.all_handlers = [] + self.log_files = {} + + self.create_log_dir() + + def create_log_dir(self): + """create a logging directory if it doesn't exits. If there is a file with + same name as the future logging directory it will be deleted. + """ + + if os.path.exists(self.log_dir): + if not os.path.isdir(self.log_dir): + os.remove(self.log_dir) + if not os.path.exists(self.log_dir): + os.makedirs(self.log_dir) + self.abs_log_dir = os.path.abspath(self.log_dir) + + def init_message(self, name=None): + """log an init message stating the name passed to it, the current date + and time and, the current working directory. + + Args: + name (str, optional): name to use for the init log message. Defaults to + the current instance class name. + """ + + if not name: + name = self.__class__.__name__ + self.log_message( + "%s online at %s in %s" + % (name, datetime.utcnow().strftime("%Y%m%d %H:%M:%SZ"), os.getcwd()) + ) + + def get_logger_level(self, level=None): + """translate the level name passed to it and return its numeric value + according to `LEVELS` values. + + Args: + level (str, optional): level name to be translated. Defaults to the current + instance `log_level`. + + Returns: + int: numeric value of the log level name passed to it or 0 (NOTSET) if the + name doesn't exists + """ + + if not level: + level = self.log_level + return self.LEVELS.get(level, logging.NOTSET) + + def get_log_formatter(self, log_format=None, date_format=None): + """create a `logging.Formatter` base on the log and date format. + + Args: + log_format (str, optional): log format to use for the Formatter constructor. + Defaults to the current instance log format. + date_format (str, optional): date format to use for the Formatter constructor. + Defaults to the current instance date format. + + Returns: + logging.Formatter: instance created base on the passed arguments + """ + + if not log_format: + log_format = self.log_format + if not date_format: + date_format = self.log_date_format + return logging.Formatter(log_format, date_format) + + def new_logger(self): + """Create a new logger based on the ROOT_LOGGER instance. By default there are no handlers. + The new logger becomes a member variable of the current instance as `self.logger`. + """ + + self.logger = ROOT_LOGGER + self.logger.setLevel(self.get_logger_level()) + self._clear_handlers() + if self.log_to_console: + self.add_console_handler() + if self.log_to_raw: + self.log_files["raw"] = "%s_raw.log" % self.log_name + self.add_file_handler( + os.path.join(self.abs_log_dir, self.log_files["raw"]), + log_format="%(message)s", + ) + + def _clear_handlers(self): + """remove all handlers stored in `self.all_handlers`. + + To prevent dups -- logging will preserve Handlers across + objects :( + """ + attrs = dir(self) + if "all_handlers" in attrs and "logger" in attrs: + for handler in self.all_handlers: + self.logger.removeHandler(handler) + self.all_handlers = [] + + def __del__(self): + """BaseLogger class destructor; shutdown, flush and remove all handlers""" + logging.shutdown() + self._clear_handlers() + + def add_console_handler(self, log_level=None, log_format=None, date_format=None): + """create a `logging.StreamHandler` using `sys.stderr` for logging the console + output and add it to the `all_handlers` member variable + + Args: + log_level (str, optional): useless argument. Not used here. + Defaults to None. + log_format (str, optional): format used for the Formatter attached to the + StreamHandler. Defaults to None. + date_format (str, optional): format used for the Formatter attached to the + StreamHandler. Defaults to None. + """ + + console_handler = logging.StreamHandler() + console_handler.setFormatter( + self.get_log_formatter(log_format=log_format, date_format=date_format) + ) + self.logger.addHandler(console_handler) + self.all_handlers.append(console_handler) + + def add_file_handler( + self, log_path, log_level=None, log_format=None, date_format=None + ): + """create a `logging.FileHandler` base on the path, log and date format + and add it to the `all_handlers` member variable. + + Args: + log_path (str): filepath to use for the `FileHandler`. + log_level (str, optional): useless argument. Not used here. + Defaults to None. + log_format (str, optional): log format to use for the Formatter constructor. + Defaults to the current instance log format. + date_format (str, optional): date format to use for the Formatter constructor. + Defaults to the current instance date format. + """ + + if not self.append_to_log and os.path.exists(log_path): + os.remove(log_path) + file_handler = logging.FileHandler(log_path) + file_handler.setLevel(self.get_logger_level(log_level)) + file_handler.setFormatter( + self.get_log_formatter(log_format=log_format, date_format=date_format) + ) + self.logger.addHandler(file_handler) + self.all_handlers.append(file_handler) + + def log_message(self, message, level=INFO, exit_code=-1, post_fatal_callback=None): + """Generic log method. + There should be more options here -- do or don't split by line, + use os.linesep instead of assuming \n, be able to pass in log level + by name or number. + + Adding the IGNORE special level for runCommand. + + Args: + message (str): message to log using the current `logger` + level (str, optional): log level of the message. Defaults to INFO. + exit_code (int, optional): exit code to use in case of a FATAL level is used. + Defaults to -1. + post_fatal_callback (function, optional): function to callback in case of + of a fatal log level. Defaults None. + """ + + if level == IGNORE: + return + for line in message.splitlines(): + self.logger.log(self.get_logger_level(level), line) + if level == FATAL: + if callable(post_fatal_callback): + self.logger.log(FATAL_LEVEL, "Running post_fatal callback...") + post_fatal_callback(message=message, exit_code=exit_code) + self.logger.log(FATAL_LEVEL, "Exiting %d" % exit_code) + raise SystemExit(exit_code) + + +# SimpleFileLogger {{{1 +class SimpleFileLogger(BaseLogger): + """Subclass of the BaseLogger. + + Create one logFile. Possibly also output to the terminal and a raw log + (no prepending of level or date) + """ + + def __init__( + self, + log_format="%(asctime)s %(levelname)8s - %(message)s", + logger_name="Simple", + log_dir="logs", + **kwargs + ): + """SimpleFileLogger constructor. Calls its superclass constructor, + creates a new logger instance and log an init message. + + Args: + log_format (str, optional): message format string to instantiate a + `logging.Formatter`. Defaults to + '%(asctime)s %(levelname)8s - %(message)s' + log_name (str, optional): name to use for the log files to be created. + Defaults to 'Simple' + log_dir (str, optional): directory location to store the log files. + Defaults to 'logs' + **kwargs: Arbitrary keyword arguments passed to the BaseLogger constructor + """ + + BaseLogger.__init__( + self, + logger_name=logger_name, + log_format=log_format, + log_dir=log_dir, + **kwargs + ) + self.new_logger() + self.init_message() + + def new_logger(self): + """calls the BaseLogger.new_logger method and adds a file handler to it.""" + + BaseLogger.new_logger(self) + self.log_path = os.path.join(self.abs_log_dir, "%s.log" % self.log_name) + self.log_files["default"] = self.log_path + self.add_file_handler(self.log_path) + + +# MultiFileLogger {{{1 +class MultiFileLogger(BaseLogger): + """Subclass of the BaseLogger class. Create a log per log level in log_dir. + Possibly also output to the terminal and a raw log (no prepending of level or date) + """ + + def __init__( + self, + logger_name="Multi", + log_format="%(asctime)s %(levelname)8s - %(message)s", + log_dir="logs", + log_to_raw=True, + **kwargs + ): + """MultiFileLogger constructor. Calls its superclass constructor, + creates a new logger instance and log an init message. + + Args: + log_format (str, optional): message format string to instantiate a + `logging.Formatter`. Defaults to + '%(asctime)s %(levelname)8s - %(message)s' + log_name (str, optional): name to use for the log files to be created. + Defaults to 'Multi' + log_dir (str, optional): directory location to store the log files. + Defaults to 'logs' + log_to_raw (bool, optional): set to True in order to create a *raw.log + file. Defaults to False. + **kwargs: Arbitrary keyword arguments passed to the BaseLogger constructor + """ + + BaseLogger.__init__( + self, + logger_name=logger_name, + log_format=log_format, + log_to_raw=log_to_raw, + log_dir=log_dir, + **kwargs + ) + + self.new_logger() + self.init_message() + + def new_logger(self): + """calls the BaseLogger.new_logger method and adds a file handler per + logging level in the `LEVELS` class attribute. + """ + + BaseLogger.new_logger(self) + min_logger_level = self.get_logger_level(self.log_level) + for level in list(self.LEVELS.keys()): + if self.get_logger_level(level) >= min_logger_level: + self.log_files[level] = "%s_%s.log" % (self.log_name, level) + self.add_file_handler( + os.path.join(self.abs_log_dir, self.log_files[level]), + log_level=level, + ) + + +# ConsoleLogger {{{1 +class ConsoleLogger(BaseLogger): + """Subclass of the BaseLogger. + + Output logs to stderr. + """ + + def __init__( + self, + log_format="%(levelname)8s - %(message)s", + log_date_format="%H:%M:%S", + logger_name="Console", + **kwargs + ): + """ConsoleLogger constructor. Calls its superclass constructor, + creates a new logger instance and log an init message. + + Args: + log_format (str, optional): message format string to instantiate a + `logging.Formatter`. Defaults to + '%(levelname)8s - %(message)s' + **kwargs: Arbitrary keyword arguments passed to the BaseLogger + constructor + """ + + BaseLogger.__init__( + self, logger_name=logger_name, log_format=log_format, **kwargs + ) + self.new_logger() + self.init_message() + + def new_logger(self): + """Create a new logger based on the ROOT_LOGGER instance. By default + there are no handlers. The new logger becomes a member variable of the + current instance as `self.logger`. + """ + self.logger = ROOT_LOGGER + self.logger.setLevel(self.get_logger_level()) + self._clear_handlers() + self.add_console_handler() + + +def numeric_log_level(level): + """Converts a mozharness log level (string) to the corresponding logger + level (number). This function makes possible to set the log level + in functions that do not inherit from LogMixin + + Args: + level (str): log level name to convert. + + Returns: + int: numeric value of the log level name. + """ + return LOG_LEVELS[level] + + +# __main__ {{{1 +if __name__ == "__main__": + """Useless comparison, due to the `pass` keyword on its body""" + pass diff --git a/testing/mozharness/mozharness/base/parallel.py b/testing/mozharness/mozharness/base/parallel.py new file mode 100755 index 0000000000..678dadeede --- /dev/null +++ b/testing/mozharness/mozharness/base/parallel.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +"""Generic ways to parallelize jobs. +""" + + +# ChunkingMixin {{{1 +class ChunkingMixin(object): + """Generic Chunking helper methods.""" + + def query_chunked_list(self, possible_list, this_chunk, total_chunks, sort=False): + """Split a list of items into a certain number of chunks and + return the subset of that will occur in this chunk. + + Ported from build.l10n.getLocalesForChunk in build/tools. + """ + if sort: + possible_list = sorted(possible_list) + else: + # Copy to prevent altering + possible_list = possible_list[:] + length = len(possible_list) + for c in range(1, total_chunks + 1): + n = length // total_chunks + # If the total number of items isn't evenly divisible by the + # number of chunks, we need to append one more onto some chunks + if c <= (length % total_chunks): + n += 1 + if c == this_chunk: + return possible_list[0:n] + del possible_list[0:n] diff --git a/testing/mozharness/mozharness/base/python.py b/testing/mozharness/mozharness/base/python.py new file mode 100644 index 0000000000..73e50bfe7c --- /dev/null +++ b/testing/mozharness/mozharness/base/python.py @@ -0,0 +1,1182 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +"""Python usage, esp. virtualenv. +""" + +import errno +import json +import os +import shutil +import site +import socket +import subprocess +import sys +import traceback +from pathlib import Path + +try: + import urlparse +except ImportError: + import urllib.parse as urlparse + +from six import string_types + +import mozharness +from mozharness.base.errors import VirtualenvErrorList +from mozharness.base.log import FATAL, WARNING +from mozharness.base.script import ( + PostScriptAction, + PostScriptRun, + PreScriptAction, + ScriptMixin, +) + +external_tools_path = os.path.join( + os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))), + "external_tools", +) + + +def get_tlsv1_post(): + # Monkeypatch to work around SSL errors in non-bleeding-edge Python. + # Taken from https://lukasa.co.uk/2013/01/Choosing_SSL_Version_In_Requests/ + import ssl + + import requests + from requests.packages.urllib3.poolmanager import PoolManager + + class TLSV1Adapter(requests.adapters.HTTPAdapter): + def init_poolmanager(self, connections, maxsize, block=False): + self.poolmanager = PoolManager( + num_pools=connections, + maxsize=maxsize, + block=block, + ssl_version=ssl.PROTOCOL_TLSv1, + ) + + s = requests.Session() + s.mount("https://", TLSV1Adapter()) + return s.post + + +# Virtualenv {{{1 +virtualenv_config_options = [ + [ + ["--virtualenv-path"], + { + "action": "store", + "dest": "virtualenv_path", + "default": "venv", + "help": "Specify the path to the virtualenv top level directory", + }, + ], + [ + ["--find-links"], + { + "action": "extend", + "dest": "find_links", + "default": ["https://pypi.pub.build.mozilla.org/pub/"], + "help": "URL to look for packages at", + }, + ], + [ + ["--pip-index"], + { + "action": "store_true", + "default": False, + "dest": "pip_index", + "help": "Use pip indexes", + }, + ], + [ + ["--no-pip-index"], + { + "action": "store_false", + "dest": "pip_index", + "help": "Don't use pip indexes (default)", + }, + ], +] + + +class VirtualenvMixin(object): + """BaseScript mixin, designed to create and use virtualenvs. + + Config items: + * virtualenv_path points to the virtualenv location on disk. + * virtualenv_modules lists the module names. + * MODULE_url list points to the module URLs (optional) + Requires virtualenv to be in PATH. + Depends on ScriptMixin + """ + + python_paths = {} + site_packages_path = None + + def __init__(self, *args, **kwargs): + self._virtualenv_modules = [] + super(VirtualenvMixin, self).__init__(*args, **kwargs) + + def register_virtualenv_module( + self, + name=None, + url=None, + method=None, + requirements=None, + optional=False, + two_pass=False, + editable=False, + ): + """Register a module to be installed with the virtualenv. + + This method can be called up until create_virtualenv() to register + modules that should be installed in the virtualenv. + + See the documentation for install_module for how the arguments are + applied. + """ + self._virtualenv_modules.append( + (name, url, method, requirements, optional, two_pass, editable) + ) + + def query_virtualenv_path(self): + """Determine the absolute path to the virtualenv.""" + dirs = self.query_abs_dirs() + + if "abs_virtualenv_dir" in dirs: + return dirs["abs_virtualenv_dir"] + + p = self.config["virtualenv_path"] + if not p: + self.fatal( + "virtualenv_path config option not set; " "this should never happen" + ) + + if os.path.isabs(p): + return p + else: + return os.path.join(dirs["abs_work_dir"], p) + + def query_python_path(self, binary="python"): + """Return the path of a binary inside the virtualenv, if + c['virtualenv_path'] is set; otherwise return the binary name. + Otherwise return None + """ + if binary not in self.python_paths: + bin_dir = "bin" + if self._is_windows(): + bin_dir = "Scripts" + virtualenv_path = self.query_virtualenv_path() + self.python_paths[binary] = os.path.abspath( + os.path.join(virtualenv_path, bin_dir, binary) + ) + + return self.python_paths[binary] + + def query_python_site_packages_path(self): + if self.site_packages_path: + return self.site_packages_path + python = self.query_python_path() + self.site_packages_path = self.get_output_from_command( + [ + python, + "-c", + "from distutils.sysconfig import get_python_lib; " + + "print(get_python_lib())", + ] + ) + return self.site_packages_path + + def package_versions( + self, pip_freeze_output=None, error_level=WARNING, log_output=False + ): + """ + reads packages from `pip freeze` output and returns a dict of + {package_name: 'version'} + """ + packages = {} + + if pip_freeze_output is None: + # get the output from `pip freeze` + pip = self.query_python_path("pip") + if not pip: + self.log("package_versions: Program pip not in path", level=error_level) + return {} + pip_freeze_output = self.get_output_from_command( + [pip, "list", "--format", "freeze", "--no-index"], + silent=True, + ignore_errors=True, + ) + if not isinstance(pip_freeze_output, string_types): + self.fatal( + "package_versions: Error encountered running `pip freeze`: " + + pip_freeze_output + ) + + for l in pip_freeze_output.splitlines(): + # parse the output into package, version + line = l.strip() + if not line: + # whitespace + continue + if line.startswith("-"): + # not a package, probably like '-e http://example.com/path#egg=package-dev' + continue + if "==" not in line: + self.fatal("pip_freeze_packages: Unrecognized output line: %s" % line) + package, version = line.split("==", 1) + packages[package] = version + + if log_output: + self.info("Current package versions:") + for package in sorted(packages): + self.info(" %s == %s" % (package, packages[package])) + + return packages + + def is_python_package_installed(self, package_name, error_level=WARNING): + """ + Return whether the package is installed + """ + # pylint --py3k W1655 + package_versions = self.package_versions(error_level=error_level) + return package_name.lower() in [package.lower() for package in package_versions] + + def install_module( + self, + module=None, + module_url=None, + install_method=None, + requirements=(), + optional=False, + global_options=[], + no_deps=False, + editable=False, + ): + """ + Install module via pip. + + module_url can be a url to a python package tarball, a path to + a directory containing a setup.py (absolute or relative to work_dir) + or None, in which case it will default to the module name. + + requirements is a list of pip requirements files. If specified, these + will be combined with the module_url (if any), like so: + + pip install -r requirements1.txt -r requirements2.txt module_url + """ + import http.client + import time + import urllib.error + import urllib.request + + c = self.config + dirs = self.query_abs_dirs() + env = self.query_env() + venv_path = self.query_virtualenv_path() + self.info("Installing %s into virtualenv %s" % (module, venv_path)) + if not module_url: + module_url = module + if install_method in (None, "pip"): + if not module_url and not requirements: + self.fatal("Must specify module and/or requirements") + pip = self.query_python_path("pip") + if c.get("verbose_pip"): + command = [pip, "-v", "install"] + else: + command = [pip, "install"] + if no_deps: + command += ["--no-deps"] + # To avoid timeouts with our pypi server, increase default timeout: + # https://bugzilla.mozilla.org/show_bug.cgi?id=1007230#c802 + command += ["--timeout", str(c.get("pip_timeout", 120))] + for requirement in requirements: + command += ["-r", requirement] + if c.get("find_links") and not c["pip_index"]: + command += ["--no-index"] + for opt in global_options: + command += ["--global-option", opt] + else: + self.fatal( + "install_module() doesn't understand an install_method of %s!" + % install_method + ) + + # find_links connection check while loop + find_links_added = 0 + fl_retry_sleep_seconds = 10 + fl_max_retry_minutes = 5 + fl_retry_loops = (fl_max_retry_minutes * 60) / fl_retry_sleep_seconds + for link in c.get("find_links", []): + parsed = urlparse.urlparse(link) + dns_result = None + get_result = None + retry_counter = 0 + while retry_counter < fl_retry_loops and ( + dns_result is None or get_result is None + ): + try: + dns_result = socket.gethostbyname(parsed.hostname) + get_result = urllib.request.urlopen(link, timeout=10).read() + break + except socket.gaierror: + retry_counter += 1 + self.warning( + "find_links: dns check failed for %s, sleeping %ss and retrying..." + % (parsed.hostname, fl_retry_sleep_seconds) + ) + time.sleep(fl_retry_sleep_seconds) + except ( + urllib.error.HTTPError, + urllib.error.URLError, + socket.timeout, + http.client.RemoteDisconnected, + ) as e: + retry_counter += 1 + self.warning( + "find_links: connection check failed for %s, sleeping %ss and retrying..." + % (link, fl_retry_sleep_seconds) + ) + self.warning("find_links: exception: %s" % e) + time.sleep(fl_retry_sleep_seconds) + # now that the connectivity check is good, add the link + if dns_result and get_result: + self.info("find_links: connection checks passed for %s, adding." % link) + find_links_added += 1 + command.extend(["--find-links", link]) + else: + self.warning( + "find_links: connection checks failed for %s" + ", but max retries reached. continuing..." % link + ) + + # TODO: make this fatal if we always see failures after this + if find_links_added == 0: + self.warning( + "find_links: no find_links added. pip installation will probably fail!" + ) + + # module_url can be None if only specifying requirements files + if module_url: + if editable: + if install_method in (None, "pip"): + command += ["-e"] + else: + self.fatal( + "editable installs not supported for install_method %s" + % install_method + ) + command += [module_url] + + # If we're only installing a single requirements file, use + # the file's directory as cwd, so relative paths work correctly. + cwd = dirs["abs_work_dir"] + if not module and len(requirements) == 1: + cwd = os.path.dirname(requirements[0]) + + # Allow for errors while building modules, but require a + # return status of 0. + self.retry( + self.run_command, + # None will cause default value to be used + attempts=1 if optional else None, + good_statuses=(0,), + error_level=WARNING if optional else FATAL, + error_message=("Could not install python package: failed all attempts."), + args=[ + command, + ], + kwargs={ + "error_list": VirtualenvErrorList, + "cwd": cwd, + "env": env, + # WARNING only since retry will raise final FATAL if all + # retry attempts are unsuccessful - and we only want + # an ERROR of FATAL if *no* retry attempt works + "error_level": WARNING, + }, + ) + + def create_virtualenv(self, modules=(), requirements=()): + """ + Create a python virtualenv. + + This uses the copy of virtualenv that is vendored in mozharness. + + virtualenv_modules can be a list of module names to install, e.g. + + virtualenv_modules = ['module1', 'module2'] + + or it can be a heterogeneous list of modules names and dicts that + define a module by its name, url-or-path, and a list of its global + options. + + virtualenv_modules = [ + { + 'name': 'module1', + 'url': None, + 'global_options': ['--opt', '--without-gcc'] + }, + { + 'name': 'module2', + 'url': 'http://url/to/package', + 'global_options': ['--use-clang'] + }, + { + 'name': 'module3', + 'url': os.path.join('path', 'to', 'setup_py', 'dir') + 'global_options': [] + }, + 'module4' + ] + + virtualenv_requirements is an optional list of pip requirements files to + use when invoking pip, e.g., + + virtualenv_requirements = [ + '/path/to/requirements1.txt', + '/path/to/requirements2.txt' + ] + """ + c = self.config + dirs = self.query_abs_dirs() + venv_path = self.query_virtualenv_path() + self.info("Creating virtualenv %s" % venv_path) + + # Always use the virtualenv that is vendored since that is deterministic. + # base_work_dir is for when we're running with mozharness.zip, e.g. on + # test jobs + # abs_src_dir is for when we're running out of a checked out copy of + # the source code + vendor_search_dirs = [ + os.path.join("{base_work_dir}", "mozharness"), + "{abs_src_dir}", + ] + if "abs_src_dir" not in dirs and "repo_path" in self.config: + dirs["abs_src_dir"] = os.path.normpath(self.config["repo_path"]) + for d in vendor_search_dirs: + try: + src_dir = Path(d.format(**dirs)) + except KeyError: + continue + + pip_wheel_path = ( + src_dir + / "third_party" + / "python" + / "_venv" + / "wheels" + / "pip-23.0.1-py3-none-any.whl" + ) + setuptools_wheel_path = ( + src_dir + / "third_party" + / "python" + / "_venv" + / "wheels" + / "setuptools-51.2.0-py3-none-any.whl" + ) + + if all(path.exists() for path in (pip_wheel_path, setuptools_wheel_path)): + break + else: + self.fatal("Can't find 'pip' and 'setuptools' wheels") + + venv_python_bin = Path(self.query_python_path()) + + if venv_python_bin.exists(): + self.info( + "Virtualenv %s appears to already exist; " + "skipping virtualenv creation." % self.query_python_path() + ) + else: + self.run_command( + [sys.executable, "--version"], + ) + + # Temporary hack to get around a bug with venv in Python 3.7.3 in CI + # https://bugs.python.org/issue36441 + if self._is_windows(): + if sys.version_info[:3] == (3, 7, 3): + python_exe = Path(sys.executable) + debug_exe_dir = ( + python_exe.parent / "lib" / "venv" / "scripts" / "nt" + ) + + if debug_exe_dir.exists(): + + for executable in { + "python.exe", + "python_d.exe", + "pythonw.exe", + "pythonw_d.exe", + }: + expected_python_debug_exe = debug_exe_dir / executable + if not expected_python_debug_exe.exists(): + shutil.copy( + sys.executable, str(expected_python_debug_exe) + ) + + venv_creation_flags = ["-m", "venv", venv_path] + + if self._is_windows(): + # To workaround an issue on Windows10 jobs in CI we have to + # explicitly install the default pip separately. Ideally we + # could just remove the "--without-pip" above and get the same + # result, but that's apparently not always the case. + venv_creation_flags = venv_creation_flags + ["--without-pip"] + + self.mkdir_p(dirs["abs_work_dir"]) + self.run_command( + [sys.executable] + venv_creation_flags, + cwd=dirs["abs_work_dir"], + error_list=VirtualenvErrorList, + halt_on_failure=True, + ) + + if self._is_windows(): + self.run_command( + [str(venv_python_bin), "-m", "ensurepip", "--default-pip"], + cwd=dirs["abs_work_dir"], + halt_on_failure=True, + ) + + self._ensure_python_exe(venv_python_bin.parent) + + # We can work around a bug on some versions of Python 3.6 on + # Windows by copying the 'pyvenv.cfg' of the current venv + # to the new venv. This will make the new venv reference + # the original Python install instead of the current venv, + # which resolves the issue. There shouldn't be any harm in + # always doing this, but we'll play it safe and restrict it + # to Windows Python 3.6 anyway. + if self._is_windows() and sys.version_info[:2] == (3, 6): + this_venv = Path(sys.executable).parent.parent + this_venv_config = this_venv / "pyvenv.cfg" + if this_venv_config.exists(): + new_venv_config = Path(venv_path) / "pyvenv.cfg" + shutil.copyfile(str(this_venv_config), str(new_venv_config)) + + self.run_command( + [ + str(venv_python_bin), + "-m", + "pip", + "install", + "--only-binary", + ":all:", + "--disable-pip-version-check", + str(pip_wheel_path), + str(setuptools_wheel_path), + ], + cwd=dirs["abs_work_dir"], + error_list=VirtualenvErrorList, + halt_on_failure=True, + ) + + self.info(self.platform_name()) + if self.platform_name().startswith("macos"): + tmp_path = "{}/bin/bak".format(venv_path) + self.info( + "Copying venv python binaries to {} to clear for re-sign".format( + tmp_path + ) + ) + subprocess.call("mkdir -p {}".format(tmp_path), shell=True) + subprocess.call( + "cp {}/bin/python* {}/".format(venv_path, tmp_path), shell=True + ) + self.info("Replacing venv python binaries with reset copies") + subprocess.call( + "mv -f {}/* {}/bin/".format(tmp_path, venv_path), shell=True + ) + self.info( + "codesign -s - --preserve-metadata=identifier,entitlements,flags,runtime " + "-f {}/bin/*".format(venv_path) + ) + subprocess.call( + "codesign -s - --preserve-metadata=identifier,entitlements,flags,runtime -f " + "{}/bin/python*".format(venv_path), + shell=True, + ) + + if not modules: + modules = c.get("virtualenv_modules", []) + if not requirements: + requirements = c.get("virtualenv_requirements", []) + if not modules and requirements: + self.install_module(requirements=requirements, install_method="pip") + for module in modules: + module_url = module + global_options = [] + if isinstance(module, dict): + if module.get("name", None): + module_name = module["name"] + else: + self.fatal( + "Can't install module without module name: %s" % str(module) + ) + module_url = module.get("url", None) + global_options = module.get("global_options", []) + else: + module_url = self.config.get("%s_url" % module, module_url) + module_name = module + install_method = "pip" + self.install_module( + module=module_name, + module_url=module_url, + install_method=install_method, + requirements=requirements, + global_options=global_options, + ) + + for ( + module, + url, + method, + requirements, + optional, + two_pass, + editable, + ) in self._virtualenv_modules: + if two_pass: + self.install_module( + module=module, + module_url=url, + install_method=method, + requirements=requirements or (), + optional=optional, + no_deps=True, + editable=editable, + ) + self.install_module( + module=module, + module_url=url, + install_method=method, + requirements=requirements or (), + optional=optional, + editable=editable, + ) + + self.info("Done creating virtualenv %s." % venv_path) + + self.package_versions(log_output=True) + + def activate_virtualenv(self): + """Import the virtualenv's packages into this Python interpreter.""" + venv_root_dir = Path(self.query_virtualenv_path()) + venv_name = venv_root_dir.name + bin_path = Path(self.query_python_path()) + bin_dir = bin_path.parent + + if self._is_windows(): + site_packages_dir = venv_root_dir / "Lib" / "site-packages" + else: + site_packages_dir = ( + venv_root_dir + / "lib" + / "python{}.{}".format(*sys.version_info) + / "site-packages" + ) + + os.environ["PATH"] = os.pathsep.join( + [str(bin_dir)] + os.environ.get("PATH", "").split(os.pathsep) + ) + os.environ["VIRTUAL_ENV"] = venv_name + + prev_path = set(sys.path) + + site.addsitedir(str(site_packages_dir.resolve())) + + new_path = list(sys.path) + + sys.path[:] = [p for p in new_path if p not in prev_path] + [ + p for p in new_path if p in prev_path + ] + + sys.real_prefix = sys.prefix + sys.prefix = str(venv_root_dir) + sys.executable = str(bin_path) + + def _ensure_python_exe(self, python_exe_root: Path): + """On some machines in CI venv does not behave consistently. Sometimes + only a "python3" executable is created, but we expect "python". Since + they are functionally identical, we can just copy "python3" to "python" + (and vice-versa) to solve the problem. + """ + python3_exe_path = python_exe_root / "python3" + python_exe_path = python_exe_root / "python" + + if self._is_windows(): + python3_exe_path = python3_exe_path.with_suffix(".exe") + python_exe_path = python_exe_path.with_suffix(".exe") + + if python3_exe_path.exists() and not python_exe_path.exists(): + shutil.copy(str(python3_exe_path), str(python_exe_path)) + + if python_exe_path.exists() and not python3_exe_path.exists(): + shutil.copy(str(python_exe_path), str(python3_exe_path)) + + if not python_exe_path.exists() and not python3_exe_path.exists(): + raise Exception( + f'Neither a "{python_exe_path.name}" or "{python3_exe_path.name}" ' + f"were found. This means something unexpected happened during the " + f"virtual environment creation and we cannot proceed." + ) + + +# This is (sadly) a mixin for logging methods. +class PerfherderResourceOptionsMixin(ScriptMixin): + def perfherder_resource_options(self): + """Obtain a list of extraOptions values to identify the env.""" + opts = [] + + if "TASKCLUSTER_INSTANCE_TYPE" in os.environ: + # Include the instance type so results can be grouped. + opts.append("taskcluster-%s" % os.environ["TASKCLUSTER_INSTANCE_TYPE"]) + else: + # We assume !taskcluster => buildbot. + instance = "unknown" + + # Try to load EC2 instance type from metadata file. This file + # may not exist in many scenarios (including when inside a chroot). + # So treat it as optional. + try: + # This file should exist on Linux in EC2. + with open("/etc/instance_metadata.json", "rb") as fh: + im = json.load(fh) + instance = im.get("aws_instance_type", "unknown").encode("ascii") + except IOError as e: + if e.errno != errno.ENOENT: + raise + self.info( + "instance_metadata.json not found; unable to " + "determine instance type" + ) + except Exception: + self.warning( + "error reading instance_metadata: %s" % traceback.format_exc() + ) + + opts.append("buildbot-%s" % instance) + + return opts + + +class ResourceMonitoringMixin(PerfherderResourceOptionsMixin): + """Provides resource monitoring capabilities to scripts. + + When this class is in the inheritance chain, resource usage stats of the + executing script will be recorded. + + This class requires the VirtualenvMixin in order to install a package used + for recording resource usage. + + While we would like to record resource usage for the entirety of a script, + since we require an external package, we can only record resource usage + after that package is installed (as part of creating the virtualenv). + That's just the way things have to be. + """ + + def __init__(self, *args, **kwargs): + super(ResourceMonitoringMixin, self).__init__(*args, **kwargs) + + self.register_virtualenv_module("psutil>=5.9.0", method="pip", optional=True) + self.register_virtualenv_module( + "mozsystemmonitor==1.0.1", method="pip", optional=True + ) + self.register_virtualenv_module("jsonschema==2.5.1", method="pip") + self._resource_monitor = None + + # 2-tuple of (name, options) to assign Perfherder resource monitor + # metrics to. This needs to be assigned by a script in order for + # Perfherder metrics to be reported. + self.resource_monitor_perfherder_id = None + + @PostScriptAction("create-virtualenv") + def _start_resource_monitoring(self, action, success=None): + self.activate_virtualenv() + + # Resource Monitor requires Python 2.7, however it's currently optional. + # Remove when all machines have had their Python version updated (bug 711299). + if sys.version_info[:2] < (2, 7): + self.warning( + "Resource monitoring will not be enabled! Python 2.7+ required." + ) + return + + try: + from mozsystemmonitor.resourcemonitor import SystemResourceMonitor + + self.info("Starting resource monitoring.") + self._resource_monitor = SystemResourceMonitor(poll_interval=1.0) + self._resource_monitor.start() + except Exception: + self.warning( + "Unable to start resource monitor: %s" % traceback.format_exc() + ) + + @PreScriptAction + def _resource_record_pre_action(self, action): + # Resource monitor isn't available until after create-virtualenv. + if not self._resource_monitor: + return + + self._resource_monitor.begin_phase(action) + + @PostScriptAction + def _resource_record_post_action(self, action, success=None): + # Resource monitor isn't available until after create-virtualenv. + if not self._resource_monitor: + return + + self._resource_monitor.finish_phase(action) + + @PostScriptRun + def _resource_record_post_run(self): + if not self._resource_monitor: + return + + # This should never raise an exception. This is a workaround until + # mozsystemmonitor is fixed. See bug 895388. + try: + self._resource_monitor.stop() + self._log_resource_usage() + + # Upload a JSON file containing the raw resource data. + try: + upload_dir = self.query_abs_dirs()["abs_blob_upload_dir"] + if not os.path.exists(upload_dir): + os.makedirs(upload_dir) + with open(os.path.join(upload_dir, "resource-usage.json"), "w") as fh: + json.dump( + self._resource_monitor.as_dict(), fh, sort_keys=True, indent=4 + ) + except (AttributeError, KeyError): + self.exception("could not upload resource usage JSON", level=WARNING) + + except Exception: + self.warning( + "Exception when reporting resource usage: %s" % traceback.format_exc() + ) + + def _log_resource_usage(self): + # Delay import because not available until virtualenv is populated. + import jsonschema + + rm = self._resource_monitor + + if rm.start_time is None: + return + + def resources(phase): + cpu_percent = rm.aggregate_cpu_percent(phase=phase, per_cpu=False) + cpu_times = rm.aggregate_cpu_times(phase=phase, per_cpu=False) + io = rm.aggregate_io(phase=phase) + + swap_in = sum(m.swap.sin for m in rm.measurements) + swap_out = sum(m.swap.sout for m in rm.measurements) + + return cpu_percent, cpu_times, io, (swap_in, swap_out) + + def log_usage(prefix, duration, cpu_percent, cpu_times, io): + message = ( + "{prefix} - Wall time: {duration:.0f}s; " + "CPU: {cpu_percent}; " + "Read bytes: {io_read_bytes}; Write bytes: {io_write_bytes}; " + "Read time: {io_read_time}; Write time: {io_write_time}" + ) + + # XXX Some test harnesses are complaining about a string being + # being fed into a 'f' formatter. This will help diagnose the + # issue. + if cpu_percent: + # pylint: disable=W1633 + cpu_percent_str = str(round(cpu_percent)) + "%" + else: + cpu_percent_str = "Can't collect data" + + try: + self.info( + message.format( + prefix=prefix, + duration=duration, + cpu_percent=cpu_percent_str, + io_read_bytes=io.read_bytes, + io_write_bytes=io.write_bytes, + io_read_time=io.read_time, + io_write_time=io.write_time, + ) + ) + + except ValueError: + self.warning("Exception when formatting: %s" % traceback.format_exc()) + + cpu_percent, cpu_times, io, (swap_in, swap_out) = resources(None) + duration = rm.end_time - rm.start_time + + # Write out Perfherder data if configured. + if self.resource_monitor_perfherder_id: + perfherder_name, perfherder_options = self.resource_monitor_perfherder_id + + suites = [] + overall = [] + + if cpu_percent: + overall.append( + { + "name": "cpu_percent", + "value": cpu_percent, + } + ) + + overall.extend( + [ + {"name": "io_write_bytes", "value": io.write_bytes}, + {"name": "io.read_bytes", "value": io.read_bytes}, + {"name": "io_write_time", "value": io.write_time}, + {"name": "io_read_time", "value": io.read_time}, + ] + ) + + suites.append( + { + "name": "%s.overall" % perfherder_name, + "extraOptions": perfherder_options + + self.perfherder_resource_options(), + "subtests": overall, + } + ) + + for phase in rm.phases.keys(): + phase_duration = rm.phases[phase][1] - rm.phases[phase][0] + subtests = [ + { + "name": "time", + "value": phase_duration, + } + ] + cpu_percent = rm.aggregate_cpu_percent(phase=phase, per_cpu=False) + if cpu_percent is not None: + subtests.append( + { + "name": "cpu_percent", + "value": rm.aggregate_cpu_percent( + phase=phase, per_cpu=False + ), + } + ) + + # We don't report I/O during each step because measured I/O + # is system I/O and that I/O can be delayed (e.g. writes will + # buffer before being flushed and recorded in our metrics). + suites.append( + { + "name": "%s.%s" % (perfherder_name, phase), + "subtests": subtests, + } + ) + + data = { + "framework": {"name": "job_resource_usage"}, + "suites": suites, + } + + schema_path = os.path.join( + external_tools_path, "performance-artifact-schema.json" + ) + with open(schema_path, "rb") as fh: + schema = json.load(fh) + + # this will throw an exception that causes the job to fail if the + # perfherder data is not valid -- please don't change this + # behaviour, otherwise people will inadvertently break this + # functionality + self.info("Validating Perfherder data against %s" % schema_path) + jsonschema.validate(data, schema) + self.info("PERFHERDER_DATA: %s" % json.dumps(data)) + + log_usage("Total resource usage", duration, cpu_percent, cpu_times, io) + + # Print special messages so usage shows up in Treeherder. + if cpu_percent: + self._tinderbox_print("CPU usage<br/>{:,.1f}%".format(cpu_percent)) + + self._tinderbox_print( + "I/O read bytes / time<br/>{:,} / {:,}".format(io.read_bytes, io.read_time) + ) + self._tinderbox_print( + "I/O write bytes / time<br/>{:,} / {:,}".format( + io.write_bytes, io.write_time + ) + ) + + # Print CPU components having >1%. "cpu_times" is a data structure + # whose attributes are measurements. Ideally we'd have an API that + # returned just the measurements as a dict or something. + cpu_attrs = [] + for attr in sorted(dir(cpu_times)): + if attr.startswith("_"): + continue + if attr in ("count", "index"): + continue + cpu_attrs.append(attr) + + cpu_total = sum(getattr(cpu_times, attr) for attr in cpu_attrs) + + for attr in cpu_attrs: + value = getattr(cpu_times, attr) + # cpu_total can be 0.0. Guard against division by 0. + # pylint --py3k W1619 + percent = value / cpu_total * 100.0 if cpu_total else 0.0 + + if percent > 1.00: + self._tinderbox_print( + "CPU {}<br/>{:,.1f} ({:,.1f}%)".format(attr, value, percent) + ) + + # Swap on Windows isn't reported by psutil. + if not self._is_windows(): + self._tinderbox_print( + "Swap in / out<br/>{:,} / {:,}".format(swap_in, swap_out) + ) + + for phase in rm.phases.keys(): + start_time, end_time = rm.phases[phase] + cpu_percent, cpu_times, io, swap = resources(phase) + log_usage(phase, end_time - start_time, cpu_percent, cpu_times, io) + + def _tinderbox_print(self, message): + self.info("TinderboxPrint: %s" % message) + + +# This needs to be inherited only if you have already inherited ScriptMixin +class Python3Virtualenv(object): + """Support Python3.5+ virtualenv creation.""" + + py3_initialized_venv = False + + def py3_venv_configuration(self, python_path, venv_path): + """We don't use __init__ to allow integrating with other mixins. + + python_path - Path to Python 3 binary. + venv_path - Path to virtual environment to be created. + """ + self.py3_initialized_venv = True + self.py3_python_path = os.path.abspath(python_path) + version = self.get_output_from_command( + [self.py3_python_path, "--version"], env=self.query_env() + ).split()[-1] + # Using -m venv is only used on 3.5+ versions + assert version > "3.5.0" + self.py3_venv_path = os.path.abspath(venv_path) + self.py3_pip_path = os.path.join(self.py3_path_to_executables(), "pip") + + def py3_path_to_executables(self): + platform = self.platform_name() + if platform.startswith("win"): + return os.path.join(self.py3_venv_path, "Scripts") + else: + return os.path.join(self.py3_venv_path, "bin") + + def py3_venv_initialized(func): + def call(self, *args, **kwargs): + if not self.py3_initialized_venv: + raise Exception( + "You need to call py3_venv_configuration() " + "before using this method." + ) + func(self, *args, **kwargs) + + return call + + @py3_venv_initialized + def py3_create_venv(self): + """Create Python environment with python3 -m venv /path/to/venv.""" + if os.path.exists(self.py3_venv_path): + self.info( + "Virtualenv %s appears to already exist; skipping " + "virtualenv creation." % self.py3_venv_path + ) + else: + self.info("Running command...") + self.run_command( + "%s -m venv %s" % (self.py3_python_path, self.py3_venv_path), + error_list=VirtualenvErrorList, + halt_on_failure=True, + env=self.query_env(), + ) + + @py3_venv_initialized + def py3_install_modules(self, modules, use_mozharness_pip_config=True): + if not os.path.exists(self.py3_venv_path): + raise Exception("You need to call py3_create_venv() first.") + + for m in modules: + cmd = [self.py3_pip_path, "install"] + if use_mozharness_pip_config: + cmd += self._mozharness_pip_args() + cmd += [m] + self.run_command(cmd, env=self.query_env()) + + def _mozharness_pip_args(self): + """We have information in Mozharness configs that apply to pip""" + c = self.config + pip_args = [] + # To avoid timeouts with our pypi server, increase default timeout: + # https://bugzilla.mozilla.org/show_bug.cgi?id=1007230#c802 + pip_args += ["--timeout", str(c.get("pip_timeout", 120))] + + if c.get("find_links") and not c["pip_index"]: + pip_args += ["--no-index"] + + # Add --find-links pages to look at. Add --trusted-host automatically if + # the host isn't secure. This allows modern versions of pip to connect + # without requiring an override. + trusted_hosts = set() + for link in c.get("find_links", []): + parsed = urlparse.urlparse(link) + + try: + socket.gethostbyname(parsed.hostname) + except socket.gaierror as e: + self.info("error resolving %s (ignoring): %s" % (parsed.hostname, e)) + continue + + pip_args += ["--find-links", link] + if parsed.scheme != "https": + trusted_hosts.add(parsed.hostname) + + for host in sorted(trusted_hosts): + pip_args += ["--trusted-host", host] + + return pip_args + + @py3_venv_initialized + def py3_install_requirement_files( + self, requirements, pip_args=[], use_mozharness_pip_config=True + ): + """ + requirements - You can specify multiple requirements paths + """ + cmd = [self.py3_pip_path, "install"] + cmd += pip_args + + if use_mozharness_pip_config: + cmd += self._mozharness_pip_args() + + for requirement_path in requirements: + cmd += ["-r", requirement_path] + + self.run_command(cmd, env=self.query_env()) + + +# __main__ {{{1 + +if __name__ == "__main__": + """TODO: unit tests.""" + pass diff --git a/testing/mozharness/mozharness/base/script.py b/testing/mozharness/mozharness/base/script.py new file mode 100644 index 0000000000..0a5622440b --- /dev/null +++ b/testing/mozharness/mozharness/base/script.py @@ -0,0 +1,2551 @@ +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +"""Generic script objects. + +script.py, along with config.py and log.py, represents the core of +mozharness. +""" + +import codecs +import datetime +import errno +import fnmatch +import functools +import gzip +import hashlib +import inspect +import itertools +import os +import platform +import pprint +import re +import shutil +import socket +import ssl +import stat +import subprocess +import sys +import tarfile +import time +import traceback +import zipfile +import zlib +from contextlib import contextmanager +from io import BytesIO + +import mozinfo +import six +from mozprocess import ProcessHandler +from six import binary_type + +from mozharness.base.config import BaseConfig +from mozharness.base.log import ( + DEBUG, + ERROR, + FATAL, + INFO, + WARNING, + ConsoleLogger, + LogMixin, + MultiFileLogger, + OutputParser, + SimpleFileLogger, +) + +try: + import httplib +except ImportError: + import http.client as httplib +try: + import simplejson as json +except ImportError: + import json +try: + from urllib2 import Request, quote, urlopen +except ImportError: + from urllib.request import Request, quote, urlopen +try: + import urlparse +except ImportError: + import urllib.parse as urlparse +if os.name == "nt": + import locale + + try: + import win32api + import win32file + + PYWIN32 = True + except ImportError: + PYWIN32 = False + +try: + from urllib2 import HTTPError, URLError +except ImportError: + from urllib.error import HTTPError, URLError + + +class ContentLengthMismatch(Exception): + pass + + +def _validate_tar_member(member, path): + def _is_within_directory(directory, target): + real_directory = os.path.realpath(directory) + real_target = os.path.realpath(target) + prefix = os.path.commonprefix([real_directory, real_target]) + return prefix == real_directory + + member_path = os.path.join(path, member.name) + if not _is_within_directory(path, member_path): + raise Exception("Attempted path traversal in tar file: " + member.name) + if member.issym(): + link_path = os.path.join(os.path.dirname(member_path), member.linkname) + if not _is_within_directory(path, link_path): + raise Exception("Attempted link path traversal in tar file: " + member.name) + if member.mode & (stat.S_ISUID | stat.S_ISGID): + raise Exception("Attempted setuid or setgid in tar file: " + member.name) + + +def _safe_extract(tar, path=".", *, numeric_owner=False): + def _files(tar, path): + for member in tar: + _validate_tar_member(member, path) + yield member + + tar.extractall(path, members=_files(tar, path), numeric_owner=numeric_owner) + + +def platform_name(): + pm = PlatformMixin() + + if pm._is_linux() and pm._is_64_bit(): + return "linux64" + elif pm._is_linux() and not pm._is_64_bit(): + return "linux" + elif pm._is_darwin(): + return "macosx" + elif pm._is_windows() and pm._is_64_bit(): + return "win64" + elif pm._is_windows() and not pm._is_64_bit(): + return "win32" + else: + return None + + +class PlatformMixin(object): + def _is_windows(self): + """check if the current operating system is Windows. + + Returns: + bool: True if the current platform is Windows, False otherwise + """ + system = platform.system() + if system in ("Windows", "Microsoft"): + return True + if system.startswith("CYGWIN"): + return True + if os.name == "nt": + return True + + def _is_darwin(self): + """check if the current operating system is Darwin. + + Returns: + bool: True if the current platform is Darwin, False otherwise + """ + if platform.system() in ("Darwin"): + return True + if sys.platform.startswith("darwin"): + return True + + def _is_linux(self): + """check if the current operating system is a Linux distribution. + + Returns: + bool: True if the current platform is a Linux distro, False otherwise + """ + if platform.system() in ("Linux"): + return True + if sys.platform.startswith("linux"): + return True + + def _is_debian(self): + """check if the current operating system is explicitly Debian. + This intentionally doesn't count Debian derivatives like Ubuntu. + + Returns: + bool: True if the current platform is debian, False otherwise + """ + if not self._is_linux(): + return False + self.info(mozinfo.linux_distro) + re_debian_distro = re.compile("debian") + return re_debian_distro.match(mozinfo.linux_distro) is not None + + def _is_redhat_based(self): + """check if the current operating system is a Redhat derived Linux distribution. + + Returns: + bool: True if the current platform is a Redhat Linux distro, False otherwise + """ + if not self._is_linux(): + return False + re_redhat_distro = re.compile("Redhat|Fedora|CentOS|Oracle") + return re_redhat_distro.match(mozinfo.linux_distro) is not None + + def _is_64_bit(self): + if self._is_darwin(): + # osx is a special snowflake and to ensure the arch, it is better to use the following + return ( + sys.maxsize > 2 ** 32 + ) # context: https://docs.python.org/2/library/platform.html + else: + # Using machine() gives you the architecture of the host rather + # than the build type of the Python binary + return "64" in platform.machine() + + +# ScriptMixin {{{1 +class ScriptMixin(PlatformMixin): + """This mixin contains simple filesystem commands and the like. + + It also contains some very special but very complex methods that, + together with logging and config, provide the base for all scripts + in this harness. + + WARNING !!! + This class depends entirely on `LogMixin` methods in such a way that it will + only works if a class inherits from both `ScriptMixin` and `LogMixin` + simultaneously. + + Depends on self.config of some sort. + + Attributes: + env (dict): a mapping object representing the string environment. + script_obj (ScriptMixin): reference to a ScriptMixin instance. + """ + + env = None + script_obj = None + ssl_context = None + + def query_filesize(self, file_path): + self.info("Determining filesize for %s" % file_path) + length = os.path.getsize(file_path) + self.info(" %s" % str(length)) + return length + + # TODO this should be parallelized with the to-be-written BaseHelper! + def query_sha512sum(self, file_path): + self.info("Determining sha512sum for %s" % file_path) + m = hashlib.sha512() + contents = self.read_from_file(file_path, verbose=False, open_mode="rb") + m.update(contents) + sha512 = m.hexdigest() + self.info(" %s" % sha512) + return sha512 + + def platform_name(self): + """Return the platform name on which the script is running on. + Returns: + None: for failure to determine the platform. + str: The name of the platform (e.g. linux64) + """ + return platform_name() + + # Simple filesystem commands {{{2 + def mkdir_p(self, path, error_level=ERROR): + """Create a directory if it doesn't exists. + This method also logs the creation, error or current existence of the + directory to be created. + + Args: + path (str): path of the directory to be created. + error_level (str): log level name to be used in case of error. + + Returns: + None: for sucess. + int: -1 on error + """ + + if not os.path.exists(path): + self.info("mkdir: %s" % path) + try: + os.makedirs(path) + except OSError: + self.log("Can't create directory %s!" % path, level=error_level) + return -1 + else: + self.debug("mkdir_p: %s Already exists." % path) + + def rmtree(self, path, log_level=INFO, error_level=ERROR, exit_code=-1): + """Delete an entire directory tree and log its result. + This method also logs the platform rmtree function, its retries, errors, + and current existence of the directory. + + Args: + path (str): path to the directory tree root to remove. + log_level (str, optional): log level name to for this operation. Defaults + to `INFO`. + error_level (str, optional): log level name to use in case of error. + Defaults to `ERROR`. + exit_code (int, optional): useless parameter, not use here. + Defaults to -1 + + Returns: + None: for success + """ + + self.log("rmtree: %s" % path, level=log_level) + error_message = "Unable to remove %s!" % path + if self._is_windows(): + # Call _rmtree_windows() directly, since even checking + # os.path.exists(path) will hang if path is longer than MAX_PATH. + self.info("Using _rmtree_windows ...") + return self.retry( + self._rmtree_windows, + error_level=error_level, + error_message=error_message, + args=(path,), + log_level=log_level, + ) + if os.path.exists(path): + if os.path.isdir(path): + return self.retry( + shutil.rmtree, + error_level=error_level, + error_message=error_message, + retry_exceptions=(OSError,), + args=(path,), + log_level=log_level, + ) + else: + return self.retry( + os.remove, + error_level=error_level, + error_message=error_message, + retry_exceptions=(OSError,), + args=(path,), + log_level=log_level, + ) + else: + self.debug("%s doesn't exist." % path) + + def query_msys_path(self, path): + """replaces the Windows harddrive letter path style with a linux + path style, e.g. C:// --> /C/ + Note: method, not used in any script. + + Args: + path (str?): path to convert to the linux path style. + Returns: + str: in case `path` is a string. The result is the path with the new notation. + type(path): `path` itself is returned in case `path` is not str type. + """ + if not isinstance(path, six.string_types): + return path + path = path.replace("\\", "/") + + def repl(m): + return "/%s/" % m.group(1) + + path = re.sub(r"""^([a-zA-Z]):/""", repl, path) + return path + + def _rmtree_windows(self, path): + """Windows-specific rmtree that handles path lengths longer than MAX_PATH. + Ported from clobberer.py. + + Args: + path (str): directory path to remove. + + Returns: + None: if the path doesn't exists. + int: the return number of calling `self.run_command` + int: in case the path specified is not a directory but a file. + 0 on success, non-zero on error. Note: The returned value + is the result of calling `win32file.DeleteFile` + """ + + assert self._is_windows() + path = os.path.realpath(path) + full_path = "\\\\?\\" + path + if not os.path.exists(full_path): + return + if not PYWIN32: + if not os.path.isdir(path): + return self.run_command('del /F /Q "%s"' % path) + else: + return self.run_command('rmdir /S /Q "%s"' % path) + # Make sure directory is writable + win32file.SetFileAttributesW("\\\\?\\" + path, win32file.FILE_ATTRIBUTE_NORMAL) + # Since we call rmtree() with a file, sometimes + if not os.path.isdir("\\\\?\\" + path): + return win32file.DeleteFile("\\\\?\\" + path) + + for ffrec in win32api.FindFiles("\\\\?\\" + path + "\\*.*"): + file_attr = ffrec[0] + name = ffrec[8] + if name == "." or name == "..": + continue + full_name = os.path.join(path, name) + + if file_attr & win32file.FILE_ATTRIBUTE_DIRECTORY: + self._rmtree_windows(full_name) + else: + try: + win32file.SetFileAttributesW( + "\\\\?\\" + full_name, win32file.FILE_ATTRIBUTE_NORMAL + ) + win32file.DeleteFile("\\\\?\\" + full_name) + except Exception: + # DeleteFile fails on long paths, del /f /q works just fine + self.run_command('del /F /Q "%s"' % full_name) + + win32file.RemoveDirectory("\\\\?\\" + path) + + def get_filename_from_url(self, url): + """parse a filename base on an url. + + Args: + url (str): url to parse for the filename + + Returns: + str: filename parsed from the url, or `netloc` network location part + of the url. + """ + + parsed = urlparse.urlsplit(url.rstrip("/")) + if parsed.path != "": + return parsed.path.rsplit("/", 1)[-1] + else: + return parsed.netloc + + def _urlopen(self, url, **kwargs): + """open the url `url` using `urllib2`.` + This method can be overwritten to extend its complexity + + Args: + url (str | urllib.request.Request): url to open + kwargs: Arbitrary keyword arguments passed to the `urllib.request.urlopen` function. + + Returns: + file-like: file-like object with additional methods as defined in + `urllib.request.urlopen`_. + None: None may be returned if no handler handles the request. + + Raises: + urllib2.URLError: on errors + + .. urillib.request.urlopen: + https://docs.python.org/2/library/urllib2.html#urllib2.urlopen + """ + # http://bugs.python.org/issue13359 - urllib2 does not automatically quote the URL + url_quoted = quote(url, safe="%/:=&?~#+!$,;'@()*[]|") + # windows certificates need to be refreshed (https://bugs.python.org/issue36011) + if self.platform_name() in ("win64",) and platform.architecture()[0] in ( + "x64", + ): + if self.ssl_context is None: + self.ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS) + self.ssl_context.load_default_certs() + return urlopen(url_quoted, context=self.ssl_context, **kwargs) + else: + return urlopen(url_quoted, **kwargs) + + def fetch_url_into_memory(self, url): + """Downloads a file from a url into memory instead of disk. + + Args: + url (str): URL path where the file to be downloaded is located. + + Raises: + IOError: When the url points to a file on disk and cannot be found + ContentLengthMismatch: When the length of the retrieved content does not match the + Content-Length response header. + ValueError: When the scheme of a url is not what is expected. + + Returns: + BytesIO: contents of url + """ + self.info("Fetch {} into memory".format(url)) + parsed_url = urlparse.urlparse(url) + + if parsed_url.scheme in ("", "file"): + path = parsed_url.path + if not os.path.isfile(path): + raise IOError("Could not find file to extract: {}".format(url)) + + content_length = os.stat(path).st_size + + # In case we're referrencing a file without file:// + if parsed_url.scheme == "": + url = "file://%s" % os.path.abspath(url) + parsed_url = urlparse.urlparse(url) + + request = Request(url) + # When calling fetch_url_into_memory() you should retry when we raise + # one of these exceptions: + # * Bug 1300663 - HTTPError: HTTP Error 404: Not Found + # * Bug 1300413 - HTTPError: HTTP Error 500: Internal Server Error + # * Bug 1300943 - HTTPError: HTTP Error 503: Service Unavailable + # * Bug 1300953 - URLError: <urlopen error [Errno -2] Name or service not known> + # * Bug 1301594 - URLError: <urlopen error [Errno 10054] An existing connection was ... + # * Bug 1301597 - URLError: <urlopen error [Errno 8] _ssl.c:504: EOF occurred in ... + # * Bug 1301855 - URLError: <urlopen error [Errno 60] Operation timed out> + # * Bug 1302237 - URLError: <urlopen error [Errno 104] Connection reset by peer> + # * Bug 1301807 - BadStatusLine: '' + # + # Bug 1309912 - Adding timeout in hopes to solve blocking on response.read() (bug 1300413) + response = urlopen(request, timeout=30) + + if parsed_url.scheme in ("http", "https"): + content_length = int(response.headers.get("Content-Length")) + + response_body = response.read() + response_body_size = len(response_body) + + self.info("Content-Length response header: {}".format(content_length)) + self.info("Bytes received: {}".format(response_body_size)) + + if response_body_size != content_length: + raise ContentLengthMismatch( + "The retrieved Content-Length header declares a body length " + "of {} bytes, while we actually retrieved {} bytes".format( + content_length, response_body_size + ) + ) + + if response.info().get("Content-Encoding") == "gzip": + self.info('Content-Encoding is "gzip", so decompressing response body') + # See http://www.zlib.net/manual.html#Advanced + # section "ZEXTERN int ZEXPORT inflateInit2 OF....": + # Add 32 to windowBits to enable zlib and gzip decoding with automatic + # header detection, or add 16 to decode only the gzip format (the zlib + # format will return a Z_DATA_ERROR). + # Adding 16 since we only wish to support gzip encoding. + file_contents = zlib.decompress(response_body, zlib.MAX_WBITS | 16) + else: + file_contents = response_body + + # Use BytesIO instead of StringIO + # http://stackoverflow.com/questions/34162017/unzip-buffer-with-python/34162395#34162395 + return BytesIO(file_contents) + + def _download_file(self, url, file_name): + """Helper function for download_file() + Additionaly this function logs all exceptions as warnings before + re-raising them + + Args: + url (str): string containing the URL with the file location + file_name (str): name of the file where the downloaded file + is written. + + Returns: + str: filename of the written file on disk + + Raises: + urllib2.URLError: on incomplete download. + urllib2.HTTPError: on Http error code + socket.timeout: on connection timeout + socket.error: on socket error + """ + # If our URLs look like files, prefix them with file:// so they can + # be loaded like URLs. + if not (url.startswith("http") or url.startswith("file://")): + if not os.path.isfile(url): + self.fatal("The file %s does not exist" % url) + url = "file://%s" % os.path.abspath(url) + + try: + f_length = None + f = self._urlopen(url, timeout=30) + + if f.info().get("content-length") is not None: + f_length = int(f.info()["content-length"]) + got_length = 0 + if f.info().get("Content-Encoding") == "gzip": + # Note, we'll download the full compressed content into its own + # file, since that allows the gzip library to seek through it. + # Once downloaded, we'll decompress it into the real target + # file, and delete the compressed version. + local_file = open(file_name + ".gz", "wb") + else: + local_file = open(file_name, "wb") + while True: + block = f.read(1024 ** 2) + if not block: + if f_length is not None and got_length != f_length: + raise URLError( + "Download incomplete; content-length was %d, " + "but only received %d" % (f_length, got_length) + ) + break + local_file.write(block) + if f_length is not None: + got_length += len(block) + local_file.close() + if f.info().get("Content-Encoding") == "gzip": + # Decompress file into target location, then remove compressed version + with open(file_name, "wb") as f_out: + # On some execution paths, this could be called with python 2.6 + # whereby gzip.open(...) cannot be used with a 'with' statement. + # So let's do this the python 2.6 way... + try: + f_in = gzip.open(file_name + ".gz", "rb") + shutil.copyfileobj(f_in, f_out) + finally: + f_in.close() + os.remove(file_name + ".gz") + return file_name + except HTTPError as e: + self.warning( + "Server returned status %s %s for %s" % (str(e.code), str(e), url) + ) + raise + except URLError as e: + self.warning("URL Error: %s" % url) + + # Failures due to missing local files won't benefit from retry. + # Raise the original OSError. + if isinstance(e.args[0], OSError) and e.args[0].errno == errno.ENOENT: + raise e.args[0] + + raise + except socket.timeout as e: + self.warning("Timed out accessing %s: %s" % (url, str(e))) + raise + except socket.error as e: + self.warning("Socket error when accessing %s: %s" % (url, str(e))) + raise + + def _retry_download(self, url, error_level, file_name=None, retry_config=None): + """Helper method to retry download methods. + + This method calls `self.retry` on `self._download_file` using the passed + parameters if a file_name is specified. If no file is specified, we will + instead call `self._urlopen`, which grabs the contents of a url but does + not create a file on disk. + + Args: + url (str): URL path where the file is located. + file_name (str): file_name where the file will be written to. + error_level (str): log level to use in case an error occurs. + retry_config (dict, optional): key-value pairs to be passed to + `self.retry`. Defaults to `None` + + Returns: + str: `self._download_file` return value is returned + unknown: `self.retry` `failure_status` is returned on failure, which + defaults to -1 + """ + retry_args = dict( + failure_status=None, + retry_exceptions=( + HTTPError, + URLError, + httplib.HTTPException, + socket.timeout, + socket.error, + ), + error_message="Can't download from %s to %s!" % (url, file_name), + error_level=error_level, + ) + + if retry_config: + retry_args.update(retry_config) + + download_func = self._urlopen + kwargs = {"url": url} + if file_name: + download_func = self._download_file + kwargs = {"url": url, "file_name": file_name} + + return self.retry(download_func, kwargs=kwargs, **retry_args) + + def _filter_entries(self, namelist, extract_dirs): + """Filter entries of the archive based on the specified list of to extract dirs.""" + filter_partial = functools.partial(fnmatch.filter, namelist) + entries = itertools.chain(*map(filter_partial, extract_dirs or ["*"])) + + for entry in entries: + yield entry + + def unzip(self, compressed_file, extract_to, extract_dirs="*", verbose=False): + """This method allows to extract a zip file without writing to disk first. + + Args: + compressed_file (object): File-like object with the contents of a compressed zip file. + extract_to (str): where to extract the compressed file. + extract_dirs (list, optional): directories inside the archive file to extract. + Defaults to '*'. + verbose (bool, optional): whether or not extracted content should be displayed. + Defaults to False. + + Raises: + zipfile.BadZipfile: on contents of zipfile being invalid + """ + with zipfile.ZipFile(compressed_file) as bundle: + entries = self._filter_entries(bundle.namelist(), extract_dirs) + + for entry in entries: + if verbose: + self.info(" {}".format(entry)) + + # Exception to be retried: + # Bug 1301645 - BadZipfile: Bad CRC-32 for file ... + # http://stackoverflow.com/questions/5624669/strange-badzipfile-bad-crc-32-problem/5626098#5626098 + # Bug 1301802 - error: Error -3 while decompressing: invalid stored block lengths + bundle.extract(entry, path=extract_to) + + # ZipFile doesn't preserve permissions during extraction: + # http://bugs.python.org/issue15795 + fname = os.path.realpath(os.path.join(extract_to, entry)) + try: + # getinfo() can raise KeyError + mode = bundle.getinfo(entry).external_attr >> 16 & 0x1FF + # Only set permissions if attributes are available. Otherwise all + # permissions will be removed eg. on Windows. + if mode: + os.chmod(fname, mode) + + except KeyError: + self.warning("{} was not found in the zip file".format(entry)) + + def deflate(self, compressed_file, mode, extract_to=".", *args, **kwargs): + """This method allows to extract a compressed file from a tar, tar.bz2 and tar.gz files. + + Args: + compressed_file (object): File-like object with the contents of a compressed file. + mode (str): string of the form 'filemode[:compression]' (e.g. 'r:gz' or 'r:bz2') + extract_to (str, optional): where to extract the compressed file. + """ + with tarfile.open(fileobj=compressed_file, mode=mode) as t: + _safe_extract(t, path=extract_to) + + def download_unpack(self, url, extract_to=".", extract_dirs="*", verbose=False): + """Generic method to download and extract a compressed file without writing it + to disk first. + + Args: + url (str): URL where the file to be downloaded is located. + extract_to (str, optional): directory where the downloaded file will + be extracted to. + extract_dirs (list, optional): directories inside the archive to extract. + Defaults to `*`. It currently only applies to zip files. + verbose (bool, optional): whether or not extracted content should be displayed. + Defaults to False. + + """ + + def _determine_extraction_method_and_kwargs(url): + EXTENSION_TO_MIMETYPE = { + "bz2": "application/x-bzip2", + "gz": "application/x-gzip", + "tar": "application/x-tar", + "zip": "application/zip", + } + MIMETYPES = { + "application/x-bzip2": { + "function": self.deflate, + "kwargs": {"mode": "r:bz2"}, + }, + "application/x-gzip": { + "function": self.deflate, + "kwargs": {"mode": "r:gz"}, + }, + "application/x-tar": { + "function": self.deflate, + "kwargs": {"mode": "r"}, + }, + "application/zip": { + "function": self.unzip, + }, + "application/x-zip-compressed": { + "function": self.unzip, + }, + } + + filename = url.split("/")[-1] + # XXX: bz2/gz instead of tar.{bz2/gz} + extension = filename[filename.rfind(".") + 1 :] + mimetype = EXTENSION_TO_MIMETYPE[extension] + self.debug("Mimetype: {}".format(mimetype)) + + function = MIMETYPES[mimetype]["function"] + kwargs = { + "compressed_file": compressed_file, + "extract_to": extract_to, + "extract_dirs": extract_dirs, + "verbose": verbose, + } + kwargs.update(MIMETYPES[mimetype].get("kwargs", {})) + + return function, kwargs + + # Many scripts overwrite this method and set extract_dirs to None + extract_dirs = "*" if extract_dirs is None else extract_dirs + self.info( + "Downloading and extracting to {} these dirs {} from {}".format( + extract_to, + ", ".join(extract_dirs), + url, + ) + ) + + # 1) Let's fetch the file + retry_args = dict( + retry_exceptions=( + HTTPError, + URLError, + httplib.HTTPException, + socket.timeout, + socket.error, + ContentLengthMismatch, + ), + sleeptime=30, + attempts=5, + error_message="Can't download from {}".format(url), + error_level=FATAL, + ) + compressed_file = self.retry( + self.fetch_url_into_memory, kwargs={"url": url}, **retry_args + ) + + # 2) We're guaranteed to have download the file with error_level=FATAL + # Let's unpack the file + function, kwargs = _determine_extraction_method_and_kwargs(url) + try: + function(**kwargs) + except zipfile.BadZipfile: + # Dump the exception and exit + self.exception(level=FATAL) + + def load_json_url(self, url, error_level=None, *args, **kwargs): + """Returns a json object from a url (it retries).""" + contents = self._retry_download( + url=url, error_level=error_level, *args, **kwargs + ) + return json.loads(contents.read()) + + # http://www.techniqal.com/blog/2008/07/31/python-file-read-write-with-urllib2/ + # TODO thinking about creating a transfer object. + def download_file( + self, + url, + file_name=None, + parent_dir=None, + create_parent_dir=True, + error_level=ERROR, + exit_code=3, + retry_config=None, + ): + """Python wget. + Download the filename at `url` into `file_name` and put it on `parent_dir`. + On error log with the specified `error_level`, on fatal exit with `exit_code`. + Execute all the above based on `retry_config` parameter. + + Args: + url (str): URL path where the file to be downloaded is located. + file_name (str, optional): file_name where the file will be written to. + Defaults to urls' filename. + parent_dir (str, optional): directory where the downloaded file will + be written to. Defaults to current working + directory + create_parent_dir (bool, optional): create the parent directory if it + doesn't exist. Defaults to `True` + error_level (str, optional): log level to use in case an error occurs. + Defaults to `ERROR` + retry_config (dict, optional): key-value pairs to be passed to + `self.retry`. Defaults to `None` + + Returns: + str: filename where the downloaded file was written to. + unknown: on failure, `failure_status` is returned. + """ + if not file_name: + try: + file_name = self.get_filename_from_url(url) + except AttributeError: + self.log( + "Unable to get filename from %s; bad url?" % url, + level=error_level, + exit_code=exit_code, + ) + return + if parent_dir: + file_name = os.path.join(parent_dir, file_name) + if create_parent_dir: + self.mkdir_p(parent_dir, error_level=error_level) + self.info("Downloading %s to %s" % (url, file_name)) + status = self._retry_download( + url=url, + error_level=error_level, + file_name=file_name, + retry_config=retry_config, + ) + if status == file_name: + self.info("Downloaded %d bytes." % os.path.getsize(file_name)) + return status + + def move(self, src, dest, log_level=INFO, error_level=ERROR, exit_code=-1): + """recursively move a file or directory (src) to another location (dest). + + Args: + src (str): file or directory path to move. + dest (str): file or directory path where to move the content to. + log_level (str): log level to use for normal operation. Defaults to + `INFO` + error_level (str): log level to use on error. Defaults to `ERROR` + + Returns: + int: 0 on success. -1 on error. + """ + self.log("Moving %s to %s" % (src, dest), level=log_level) + try: + shutil.move(src, dest) + # http://docs.python.org/tutorial/errors.html + except IOError as e: + self.log("IO error: %s" % str(e), level=error_level, exit_code=exit_code) + return -1 + except shutil.Error as e: + # ERROR level ends up reporting the failure to treeherder & + # pollutes the failure summary list. + self.log("shutil error: %s" % str(e), level=WARNING, exit_code=exit_code) + return -1 + return 0 + + def chmod(self, path, mode): + """change `path` mode to `mode`. + + Args: + path (str): path whose mode will be modified. + mode (hex): one of the values defined at `stat`_ + + .. _stat: + https://docs.python.org/2/library/os.html#os.chmod + """ + + self.info("Chmoding %s to %s" % (path, str(oct(mode)))) + os.chmod(path, mode) + + def copyfile( + self, + src, + dest, + log_level=INFO, + error_level=ERROR, + copystat=False, + compress=False, + ): + """copy or compress `src` into `dest`. + + Args: + src (str): filepath to copy. + dest (str): filepath where to move the content to. + log_level (str, optional): log level to use for normal operation. Defaults to + `INFO` + error_level (str, optional): log level to use on error. Defaults to `ERROR` + copystat (bool, optional): whether or not to copy the files metadata. + Defaults to `False`. + compress (bool, optional): whether or not to compress the destination file. + Defaults to `False`. + + Returns: + int: -1 on error + None: on success + """ + + if compress: + self.log("Compressing %s to %s" % (src, dest), level=log_level) + try: + infile = open(src, "rb") + outfile = gzip.open(dest, "wb") + outfile.writelines(infile) + outfile.close() + infile.close() + except IOError as e: + self.log( + "Can't compress %s to %s: %s!" % (src, dest, str(e)), + level=error_level, + ) + return -1 + else: + self.log("Copying %s to %s" % (src, dest), level=log_level) + try: + shutil.copyfile(src, dest) + except (IOError, shutil.Error) as e: + self.log( + "Can't copy %s to %s: %s!" % (src, dest, str(e)), level=error_level + ) + return -1 + + if copystat: + try: + shutil.copystat(src, dest) + except (IOError, shutil.Error) as e: + self.log( + "Can't copy attributes of %s to %s: %s!" % (src, dest, str(e)), + level=error_level, + ) + return -1 + + def copytree( + self, src, dest, overwrite="no_overwrite", log_level=INFO, error_level=ERROR + ): + """An implementation of `shutil.copytree` that allows for `dest` to exist + and implements different overwrite levels: + - 'no_overwrite' will keep all(any) existing files in destination tree + - 'overwrite_if_exists' will only overwrite destination paths that have + the same path names relative to the root of the + src and destination tree + - 'clobber' will replace the whole destination tree(clobber) if it exists + + Args: + src (str): directory path to move. + dest (str): directory path where to move the content to. + overwrite (str): string specifying the overwrite level. + log_level (str, optional): log level to use for normal operation. Defaults to + `INFO` + error_level (str, optional): log level to use on error. Defaults to `ERROR` + + Returns: + int: -1 on error + None: on success + """ + + self.info("copying tree: %s to %s" % (src, dest)) + try: + if overwrite == "clobber" or not os.path.exists(dest): + self.rmtree(dest) + shutil.copytree(src, dest) + elif overwrite == "no_overwrite" or overwrite == "overwrite_if_exists": + files = os.listdir(src) + for f in files: + abs_src_f = os.path.join(src, f) + abs_dest_f = os.path.join(dest, f) + if not os.path.exists(abs_dest_f): + if os.path.isdir(abs_src_f): + self.mkdir_p(abs_dest_f) + self.copytree(abs_src_f, abs_dest_f, overwrite="clobber") + else: + shutil.copy2(abs_src_f, abs_dest_f) + elif overwrite == "no_overwrite": # destination path exists + if os.path.isdir(abs_src_f) and os.path.isdir(abs_dest_f): + self.copytree( + abs_src_f, abs_dest_f, overwrite="no_overwrite" + ) + else: + self.debug( + "ignoring path: %s as destination: \ + %s exists" + % (abs_src_f, abs_dest_f) + ) + else: # overwrite == 'overwrite_if_exists' and destination exists + self.debug("overwriting: %s with: %s" % (abs_dest_f, abs_src_f)) + self.rmtree(abs_dest_f) + + if os.path.isdir(abs_src_f): + self.mkdir_p(abs_dest_f) + self.copytree( + abs_src_f, abs_dest_f, overwrite="overwrite_if_exists" + ) + else: + shutil.copy2(abs_src_f, abs_dest_f) + else: + self.fatal( + "%s is not a valid argument for param overwrite" % (overwrite) + ) + except (IOError, shutil.Error): + self.exception( + "There was an error while copying %s to %s!" % (src, dest), + level=error_level, + ) + return -1 + + def write_to_file( + self, + file_path, + contents, + verbose=True, + open_mode="w", + create_parent_dir=False, + error_level=ERROR, + ): + """Write `contents` to `file_path`, according to `open_mode`. + + Args: + file_path (str): filepath where the content will be written to. + contents (str): content to write to the filepath. + verbose (bool, optional): whether or not to log `contents` value. + Defaults to `True` + open_mode (str, optional): open mode to use for openning the file. + Defaults to `w` + create_parent_dir (bool, optional): whether or not to create the + parent directory of `file_path` + error_level (str, optional): log level to use on error. Defaults to `ERROR` + + Returns: + str: `file_path` on success + None: on error. + """ + self.info("Writing to file %s" % file_path) + if verbose: + self.info("Contents:") + for line in contents.splitlines(): + self.info(" %s" % line) + if create_parent_dir: + parent_dir = os.path.dirname(file_path) + self.mkdir_p(parent_dir, error_level=error_level) + try: + fh = open(file_path, open_mode) + try: + fh.write(contents) + except UnicodeEncodeError: + fh.write(contents.encode("utf-8", "replace")) + fh.close() + return file_path + except IOError: + self.log("%s can't be opened for writing!" % file_path, level=error_level) + + @contextmanager + def opened(self, file_path, verbose=True, open_mode="r", error_level=ERROR): + """Create a context manager to use on a with statement. + + Args: + file_path (str): filepath of the file to open. + verbose (bool, optional): useless parameter, not used here. + Defaults to True. + open_mode (str, optional): open mode to use for openning the file. + Defaults to `r` + error_level (str, optional): log level name to use on error. + Defaults to `ERROR` + + Yields: + tuple: (file object, error) pair. In case of error `None` is yielded + as file object, together with the corresponding error. + If there is no error, `None` is returned as the error. + """ + # See opened_w_error in http://www.python.org/dev/peps/pep-0343/ + self.info("Reading from file %s" % file_path) + try: + fh = open(file_path, open_mode) + except IOError as err: + self.log( + "unable to open %s: %s" % (file_path, err.strerror), level=error_level + ) + yield None, err + else: + try: + yield fh, None + finally: + fh.close() + + def read_from_file(self, file_path, verbose=True, open_mode="r", error_level=ERROR): + """Use `self.opened` context manager to open a file and read its + content. + + Args: + file_path (str): filepath of the file to read. + verbose (bool, optional): whether or not to log the file content. + Defaults to True. + open_mode (str, optional): open mode to use for openning the file. + Defaults to `r` + error_level (str, optional): log level name to use on error. + Defaults to `ERROR` + + Returns: + None: on error. + str: file content on success. + """ + with self.opened(file_path, verbose, open_mode, error_level) as (fh, err): + if err: + return None + contents = fh.read() + if verbose: + self.info("Contents:") + for line in contents.splitlines(): + self.info(" %s" % line) + return contents + + def chdir(self, dir_name): + self.log("Changing directory to %s." % dir_name) + os.chdir(dir_name) + + def is_exe(self, fpath): + """ + Determine if fpath is a file and if it is executable. + """ + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + def which(self, program): + """OS independent implementation of Unix's which command + + Args: + program (str): name or path to the program whose executable is + being searched. + + Returns: + None: if the executable was not found. + str: filepath of the executable file. + """ + if self._is_windows() and not program.endswith(".exe"): + program += ".exe" + fpath, fname = os.path.split(program) + if fpath: + if self.is_exe(program): + return program + else: + # If the exe file is defined in the configs let's use that + exe = self.query_exe(program) + if self.is_exe(exe): + return exe + + # If not defined, let's look for it in the $PATH + env = self.query_env() + for path in env["PATH"].split(os.pathsep): + exe_file = os.path.join(path, program) + if self.is_exe(exe_file): + return exe_file + return None + + # More complex commands {{{2 + def retry( + self, + action, + attempts=None, + sleeptime=60, + max_sleeptime=5 * 60, + retry_exceptions=(Exception,), + good_statuses=None, + cleanup=None, + error_level=ERROR, + error_message="%(action)s failed after %(attempts)d tries!", + failure_status=-1, + log_level=INFO, + args=(), + kwargs={}, + ): + """generic retry command. Ported from `util.retry`_ + + Args: + action (func): callable object to retry. + attempts (int, optinal): maximum number of times to call actions. + Defaults to `self.config.get('global_retries', 5)` + sleeptime (int, optional): number of seconds to wait between + attempts. Defaults to 60 and doubles each retry attempt, to + a maximum of `max_sleeptime' + max_sleeptime (int, optional): maximum value of sleeptime. Defaults + to 5 minutes + retry_exceptions (tuple, optional): Exceptions that should be caught. + If exceptions other than those listed in `retry_exceptions' are + raised from `action', they will be raised immediately. Defaults + to (Exception) + good_statuses (object, optional): return values which, if specified, + will result in retrying if the return value isn't listed. + Defaults to `None`. + cleanup (func, optional): If `cleanup' is provided and callable + it will be called immediately after an Exception is caught. + No arguments will be passed to it. If your cleanup function + requires arguments it is recommended that you wrap it in an + argumentless function. + Defaults to `None`. + error_level (str, optional): log level name in case of error. + Defaults to `ERROR`. + error_message (str, optional): string format to use in case + none of the attempts success. Defaults to + '%(action)s failed after %(attempts)d tries!' + failure_status (int, optional): flag to return in case the retries + were not successfull. Defaults to -1. + log_level (str, optional): log level name to use for normal activity. + Defaults to `INFO`. + args (tuple, optional): positional arguments to pass onto `action`. + kwargs (dict, optional): key-value arguments to pass onto `action`. + + Returns: + object: return value of `action`. + int: failure status in case of failure retries. + """ + if not callable(action): + self.fatal("retry() called with an uncallable method %s!" % action) + if cleanup and not callable(cleanup): + self.fatal("retry() called with an uncallable cleanup method %s!" % cleanup) + if not attempts: + attempts = self.config.get("global_retries", 5) + if max_sleeptime < sleeptime: + self.debug( + "max_sleeptime %d less than sleeptime %d" % (max_sleeptime, sleeptime) + ) + n = 0 + while n <= attempts: + retry = False + n += 1 + try: + self.log( + "retry: Calling %s with args: %s, kwargs: %s, attempt #%d" + % (action.__name__, str(args), str(kwargs), n), + level=log_level, + ) + status = action(*args, **kwargs) + if good_statuses and status not in good_statuses: + retry = True + except retry_exceptions as e: + retry = True + error_message = "%s\nCaught exception: %s" % (error_message, str(e)) + self.log( + "retry: attempt #%d caught %s exception: %s" + % (n, type(e).__name__, str(e)), + level=INFO, + ) + + if not retry: + return status + else: + if cleanup: + cleanup() + if n == attempts: + self.log( + error_message % {"action": action, "attempts": n}, + level=error_level, + ) + return failure_status + if sleeptime > 0: + self.log( + "retry: Failed, sleeping %d seconds before retrying" + % sleeptime, + level=log_level, + ) + time.sleep(sleeptime) + sleeptime = sleeptime * 2 + if sleeptime > max_sleeptime: + sleeptime = max_sleeptime + + def query_env( + self, + partial_env=None, + replace_dict=None, + purge_env=(), + set_self_env=None, + log_level=DEBUG, + avoid_host_env=False, + ): + """Environment query/generation method. + The default, self.query_env(), will look for self.config['env'] + and replace any special strings in there ( %(PATH)s ). + It will then store it as self.env for speeding things up later. + + If you specify partial_env, partial_env will be used instead of + self.config['env'], and we don't save self.env as it's a one-off. + + + Args: + partial_env (dict, optional): key-value pairs of the name and value + of different environment variables. Defaults to an empty dictionary. + replace_dict (dict, optional): key-value pairs to replace the old + environment variables. + purge_env (list): environment names to delete from the final + environment dictionary. + set_self_env (boolean, optional): whether or not the environment + variables dictionary should be copied to `self`. + Defaults to True. + log_level (str, optional): log level name to use on normal operation. + Defaults to `DEBUG`. + avoid_host_env (boolean, optional): if set to True, we will not use + any environment variables set on the host except PATH. + Defaults to False. + + Returns: + dict: environment variables names with their values. + """ + if partial_env is None: + if self.env is not None: + return self.env + partial_env = self.config.get("env", None) + if partial_env is None: + partial_env = {} + if set_self_env is None: + set_self_env = True + + env = {"PATH": os.environ["PATH"]} if avoid_host_env else os.environ.copy() + + default_replace_dict = self.query_abs_dirs() + default_replace_dict["PATH"] = os.environ["PATH"] + if not replace_dict: + replace_dict = default_replace_dict + else: + for key in default_replace_dict: + if key not in replace_dict: + replace_dict[key] = default_replace_dict[key] + for key in partial_env.keys(): + env[key] = partial_env[key] % replace_dict + self.log("ENV: %s is now %s" % (key, env[key]), level=log_level) + for k in purge_env: + if k in env: + del env[k] + if os.name == "nt": + pref_encoding = locale.getpreferredencoding() + for k, v in six.iteritems(env): + # When run locally on Windows machines, some environment + # variables may be unicode. + env[k] = six.ensure_str(v, pref_encoding) + if set_self_env: + self.env = env + return env + + def query_exe( + self, + exe_name, + exe_dict="exes", + default=None, + return_type=None, + error_level=FATAL, + ): + """One way to work around PATH rewrites. + + By default, return exe_name, and we'll fall through to searching + os.environ["PATH"]. + However, if self.config[exe_dict][exe_name] exists, return that. + This lets us override exe paths via config file. + + If we need runtime setting, we can build in self.exes support later. + + Args: + exe_name (str): name of the executable to search for. + exe_dict(str, optional): name of the dictionary of executables + present in `self.config`. Defaults to `exes`. + default (str, optional): default name of the executable to search + for. Defaults to `exe_name`. + return_type (str, optional): type to which the original return + value will be turn into. Only 'list', 'string' and `None` are + supported. Defaults to `None`. + error_level (str, optional): log level name to use on error. + + Returns: + list: in case return_type is 'list' + str: in case return_type is 'string' + None: in case return_type is `None` + Any: if the found executable is not of type list, tuple nor str. + """ + if default is None: + default = exe_name + exe = self.config.get(exe_dict, {}).get(exe_name, default) + repl_dict = {} + if hasattr(self.script_obj, "query_abs_dirs"): + # allow for 'make': '%(abs_work_dir)s/...' etc. + dirs = self.script_obj.query_abs_dirs() + repl_dict.update(dirs) + if isinstance(exe, dict): + found = False + # allow for searchable paths of the exe + for name, path in six.iteritems(exe): + if isinstance(path, list) or isinstance(path, tuple): + path = [x % repl_dict for x in path] + if all([os.path.exists(section) for section in path]): + found = True + elif isinstance(path, str): + path = path % repl_dict + if os.path.exists(path): + found = True + else: + self.log( + "a exes %s dict's value is not a string, list, or tuple. Got key " + "%s and value %s" % (exe_name, name, str(path)), + level=error_level, + ) + if found: + exe = path + break + else: + self.log( + "query_exe was a searchable dict but an existing " + "path could not be determined. Tried searching in " + "paths: %s" % (str(exe)), + level=error_level, + ) + return None + elif isinstance(exe, list) or isinstance(exe, tuple): + exe = [x % repl_dict for x in exe] + elif isinstance(exe, str): + exe = exe % repl_dict + else: + self.log( + "query_exe: %s is not a list, tuple, dict, or string: " + "%s!" % (exe_name, str(exe)), + level=error_level, + ) + return exe + if return_type == "list": + if isinstance(exe, str): + exe = [exe] + elif return_type == "string": + if isinstance(exe, list): + exe = subprocess.list2cmdline(exe) + elif return_type is not None: + self.log( + "Unknown return_type type %s requested in query_exe!" % return_type, + level=error_level, + ) + return exe + + def run_command( + self, + command, + cwd=None, + error_list=None, + halt_on_failure=False, + success_codes=None, + env=None, + partial_env=None, + return_type="status", + throw_exception=False, + output_parser=None, + output_timeout=None, + fatal_exit_code=2, + error_level=ERROR, + **kwargs + ): + """Run a command, with logging and error parsing. + TODO: context_lines + + error_list example: + [{'regex': re.compile('^Error: LOL J/K'), level=IGNORE}, + {'regex': re.compile('^Error:'), level=ERROR, contextLines='5:5'}, + {'substr': 'THE WORLD IS ENDING', level=FATAL, contextLines='20:'} + ] + (context_lines isn't written yet) + + Args: + command (str | list | tuple): command or sequence of commands to + execute and log. + cwd (str, optional): directory path from where to execute the + command. Defaults to `None`. + error_list (list, optional): list of errors to pass to + `mozharness.base.log.OutputParser`. Defaults to `None`. + halt_on_failure (bool, optional): whether or not to redefine the + log level as `FATAL` on errors. Defaults to False. + success_codes (int, optional): numeric value to compare against + the command return value. + env (dict, optional): key-value of environment values to use to + run the command. Defaults to None. + partial_env (dict, optional): key-value of environment values to + replace from the current environment values. Defaults to None. + return_type (str, optional): if equal to 'num_errors' then the + amount of errors matched by `error_list` is returned. Defaults + to 'status'. + throw_exception (bool, optional): whether or not to raise an + exception if the return value of the command doesn't match + any of the `success_codes`. Defaults to False. + output_parser (OutputParser, optional): lets you provide an + instance of your own OutputParser subclass. Defaults to `OutputParser`. + output_timeout (int): amount of seconds to wait for output before + the process is killed. + fatal_exit_code (int, optional): call `self.fatal` if the return value + of the command is not in `success_codes`. Defaults to 2. + error_level (str, optional): log level name to use on error. Defaults + to `ERROR`. + **kwargs: Arbitrary keyword arguments. + + Returns: + int: -1 on error. + Any: `command` return value is returned otherwise. + """ + if success_codes is None: + success_codes = [0] + if cwd is not None: + if not os.path.isdir(cwd): + level = error_level + if halt_on_failure: + level = FATAL + self.log( + "Can't run command %s in non-existent directory '%s'!" + % (command, cwd), + level=level, + ) + return -1 + self.info("Running command: %s in %s" % (command, cwd)) + else: + self.info("Running command: %s" % (command,)) + if isinstance(command, list) or isinstance(command, tuple): + self.info("Copy/paste: %s" % subprocess.list2cmdline(command)) + shell = True + if isinstance(command, list) or isinstance(command, tuple): + shell = False + if env is None: + if partial_env: + self.info("Using partial env: %s" % pprint.pformat(partial_env)) + env = self.query_env(partial_env=partial_env) + else: + if hasattr(self, "previous_env") and env == self.previous_env: + self.info("Using env: (same as previous command)") + else: + self.info("Using env: %s" % pprint.pformat(env)) + self.previous_env = env + + if output_parser is None: + parser = OutputParser( + config=self.config, log_obj=self.log_obj, error_list=error_list + ) + else: + parser = output_parser + + try: + if output_timeout: + + def processOutput(line): + parser.add_lines(line) + + def onTimeout(): + self.info( + "Automation Error: mozprocess timed out after " + "%s seconds running %s" % (str(output_timeout), str(command)) + ) + + p = ProcessHandler( + command, + shell=shell, + env=env, + cwd=cwd, + storeOutput=False, + onTimeout=(onTimeout,), + processOutputLine=[processOutput], + ) + self.info( + "Calling %s with output_timeout %d" % (command, output_timeout) + ) + p.run(outputTimeout=output_timeout) + p.wait() + if p.timedOut: + self.log( + "timed out after %s seconds of no output" % output_timeout, + level=error_level, + ) + returncode = int(p.proc.returncode) + else: + p = subprocess.Popen( + command, + shell=shell, + stdout=subprocess.PIPE, + cwd=cwd, + stderr=subprocess.STDOUT, + env=env, + bufsize=0, + ) + loop = True + while loop: + if p.poll() is not None: + """Avoid losing the final lines of the log?""" + loop = False + while True: + line = p.stdout.readline() + if not line: + break + parser.add_lines(line) + returncode = p.returncode + except KeyboardInterrupt: + level = error_level + if halt_on_failure: + level = FATAL + self.log( + "Process interrupted by the user, killing process with pid %s" % p.pid, + level=level, + ) + p.kill() + return -1 + except OSError as e: + level = error_level + if halt_on_failure: + level = FATAL + self.log( + "caught OS error %s: %s while running %s" + % (e.errno, e.strerror, command), + level=level, + ) + return -1 + + if returncode not in success_codes: + if throw_exception: + raise subprocess.CalledProcessError(returncode, command) + # Force level to be INFO as message is not necessary in Treeherder + self.log("Return code: %d" % returncode, level=INFO) + + if halt_on_failure: + _fail = False + if returncode not in success_codes: + self.log( + "%s not in success codes: %s" % (returncode, success_codes), + level=error_level, + ) + _fail = True + if parser.num_errors: + self.log("failures found while parsing output", level=error_level) + _fail = True + if _fail: + self.return_code = fatal_exit_code + self.fatal( + "Halting on failure while running %s" % (command,), + exit_code=fatal_exit_code, + ) + if return_type == "num_errors": + return parser.num_errors + return returncode + + def get_output_from_command( + self, + command, + cwd=None, + halt_on_failure=False, + env=None, + silent=False, + log_level=INFO, + tmpfile_base_path="tmpfile", + return_type="output", + save_tmpfiles=False, + throw_exception=False, + fatal_exit_code=2, + ignore_errors=False, + success_codes=None, + output_filter=None, + ): + """Similar to run_command, but where run_command is an + os.system(command) analog, get_output_from_command is a `command` + analog. + + Less error checking by design, though if we figure out how to + do it without borking the output, great. + + TODO: binary mode? silent is kinda like that. + TODO: since p.wait() can take a long time, optionally log something + every N seconds? + TODO: optionally only keep the first or last (N) line(s) of output? + TODO: optionally only return the tmp_stdout_filename? + + ignore_errors=True is for the case where a command might produce standard + error output, but you don't particularly care; setting to True will + cause standard error to be logged at DEBUG rather than ERROR + + Args: + command (str | list): command or list of commands to + execute and log. + cwd (str, optional): directory path from where to execute the + command. Defaults to `None`. + halt_on_failure (bool, optional): whether or not to redefine the + log level as `FATAL` on error. Defaults to False. + env (dict, optional): key-value of environment values to use to + run the command. Defaults to None. + silent (bool, optional): whether or not to output the stdout of + executing the command. Defaults to False. + log_level (str, optional): log level name to use on normal execution. + Defaults to `INFO`. + tmpfile_base_path (str, optional): base path of the file to which + the output will be writen to. Defaults to 'tmpfile'. + return_type (str, optional): if equal to 'output' then the complete + output of the executed command is returned, otherwise the written + filenames are returned. Defaults to 'output'. + save_tmpfiles (bool, optional): whether or not to save the temporary + files created from the command output. Defaults to False. + throw_exception (bool, optional): whether or not to raise an + exception if the return value of the command is not zero. + Defaults to False. + fatal_exit_code (int, optional): call self.fatal if the return value + of the command match this value. + ignore_errors (bool, optional): whether or not to change the log + level to `ERROR` for the output of stderr. Defaults to False. + success_codes (int, optional): numeric value to compare against + the command return value. + output_filter (func, optional): provide a function to filter output + so that noise is reduced and lines are sanitized. default: None + + Returns: + None: if the cwd is not a directory. + None: on IOError. + tuple: stdout and stderr filenames. + str: stdout output. + """ + if cwd: + if not os.path.isdir(cwd): + level = ERROR + if halt_on_failure: + level = FATAL + self.log( + "Can't run command %s in non-existent directory %s!" + % (command, cwd), + level=level, + ) + return None + self.info("Getting output from command: %s in %s" % (command, cwd)) + else: + self.info("Getting output from command: %s" % command) + if isinstance(command, list): + self.info("Copy/paste: %s" % subprocess.list2cmdline(command)) + # This could potentially return something? + tmp_stdout = None + tmp_stderr = None + tmp_stdout_filename = "%s_stdout" % tmpfile_base_path + tmp_stderr_filename = "%s_stderr" % tmpfile_base_path + if success_codes is None: + success_codes = [0] + + # TODO probably some more elegant solution than 2 similar passes + try: + tmp_stdout = open(tmp_stdout_filename, "w") + except IOError: + level = ERROR + if halt_on_failure: + level = FATAL + self.log( + "Can't open %s for writing!" % tmp_stdout_filename + self.exception(), + level=level, + ) + return None + try: + tmp_stderr = open(tmp_stderr_filename, "w") + except IOError: + level = ERROR + if halt_on_failure: + level = FATAL + self.log( + "Can't open %s for writing!" % tmp_stderr_filename + self.exception(), + level=level, + ) + return None + shell = True + if isinstance(command, list): + shell = False + + p = subprocess.Popen( + command, + shell=shell, + stdout=tmp_stdout, + cwd=cwd, + stderr=tmp_stderr, + env=env, + bufsize=0, + ) + # XXX: changed from self.debug to self.log due to this error: + # TypeError: debug() takes exactly 1 argument (2 given) + self.log( + "Temporary files: %s and %s" % (tmp_stdout_filename, tmp_stderr_filename), + level=DEBUG, + ) + p.wait() + tmp_stdout.close() + tmp_stderr.close() + return_level = DEBUG + output = None + if return_type == "output" or not silent: + if os.path.exists(tmp_stdout_filename) and os.path.getsize( + tmp_stdout_filename + ): + output = self.read_from_file(tmp_stdout_filename, verbose=False) + if output_filter: + output = output_filter(output) + if not silent: + self.log("Output received:", level=log_level) + output_lines = output.rstrip().splitlines() + for line in output_lines: + if not line or line.isspace(): + continue + if isinstance(line, binary_type): + line = line.decode("utf-8") + self.log(" %s" % line, level=log_level) + output = "\n".join(output_lines) + if os.path.exists(tmp_stderr_filename) and os.path.getsize(tmp_stderr_filename): + errors = self.read_from_file(tmp_stderr_filename, verbose=False) + if output_filter: + errors = output_filter(errors) + if errors: + if not ignore_errors: + return_level = ERROR + self.log("Errors received:", level=return_level) + for line in errors.rstrip().splitlines(): + if not line or line.isspace(): + continue + if isinstance(line, binary_type): + line = line.decode("utf-8") + self.log(" %s" % line, level=return_level) + elif p.returncode not in success_codes and not ignore_errors: + return_level = ERROR + # Clean up. + if not save_tmpfiles: + self.rmtree(tmp_stderr_filename, log_level=DEBUG) + self.rmtree(tmp_stdout_filename, log_level=DEBUG) + if p.returncode and throw_exception: + raise subprocess.CalledProcessError(p.returncode, command) + # Force level to be INFO as message is not necessary in Treeherder + self.log("Return code: %d" % p.returncode, level=INFO) + if halt_on_failure and return_level == ERROR: + self.return_code = fatal_exit_code + self.fatal( + "Halting on failure while running %s" % command, + exit_code=fatal_exit_code, + ) + # Hm, options on how to return this? I bet often we'll want + # output_lines[0] with no newline. + if return_type != "output": + return (tmp_stdout_filename, tmp_stderr_filename) + else: + return output + + def _touch_file(self, file_name, times=None, error_level=FATAL): + """touch a file. + + Args: + file_name (str): name of the file to touch. + times (tuple, optional): 2-tuple as specified by `os.utime`_ + Defaults to None. + error_level (str, optional): log level name in case of error. + Defaults to `FATAL`. + + .. _`os.utime`: + https://docs.python.org/3.4/library/os.html?highlight=os.utime#os.utime + """ + self.info("Touching: %s" % file_name) + try: + os.utime(file_name, times) + except OSError: + try: + open(file_name, "w").close() + except IOError as e: + msg = "I/O error(%s): %s" % (e.errno, e.strerror) + self.log(msg, error_level=error_level) + os.utime(file_name, times) + + def unpack( + self, + filename, + extract_to, + extract_dirs=None, + error_level=ERROR, + fatal_exit_code=2, + verbose=False, + ): + """The method allows to extract a file regardless of its extension. + + Args: + filename (str): filename of the compressed file. + extract_to (str): where to extract the compressed file. + extract_dirs (list, optional): directories inside the archive file to extract. + Defaults to `None`. + fatal_exit_code (int, optional): call `self.fatal` if the return value + of the command is not in `success_codes`. Defaults to 2. + verbose (bool, optional): whether or not extracted content should be displayed. + Defaults to False. + + Raises: + IOError: on `filename` file not found. + + """ + if not os.path.isfile(filename): + raise IOError("Could not find file to extract: %s" % filename) + + if zipfile.is_zipfile(filename): + try: + self.info( + "Using ZipFile to extract {} to {}".format(filename, extract_to) + ) + with zipfile.ZipFile(filename) as bundle: + for entry in self._filter_entries(bundle.namelist(), extract_dirs): + if verbose: + self.info(" %s" % entry) + bundle.extract(entry, path=extract_to) + + # ZipFile doesn't preserve permissions during extraction: + # http://bugs.python.org/issue15795 + fname = os.path.realpath(os.path.join(extract_to, entry)) + mode = bundle.getinfo(entry).external_attr >> 16 & 0x1FF + # Only set permissions if attributes are available. Otherwise all + # permissions will be removed eg. on Windows. + if mode: + os.chmod(fname, mode) + except zipfile.BadZipfile as e: + self.log( + "%s (%s)" % (str(e), filename), + level=error_level, + exit_code=fatal_exit_code, + ) + + # Bug 1211882 - is_tarfile cannot be trusted for dmg files + elif tarfile.is_tarfile(filename) and not filename.lower().endswith(".dmg"): + try: + self.info( + "Using TarFile to extract {} to {}".format(filename, extract_to) + ) + with tarfile.open(filename) as bundle: + for entry in self._filter_entries(bundle.getnames(), extract_dirs): + _validate_tar_member(bundle.getmember(entry), extract_to) + if verbose: + self.info(" %s" % entry) + bundle.extract(entry, path=extract_to) + except tarfile.TarError as e: + self.log( + "%s (%s)" % (str(e), filename), + level=error_level, + exit_code=fatal_exit_code, + ) + else: + self.log( + "No extraction method found for: %s" % filename, + level=error_level, + exit_code=fatal_exit_code, + ) + + def is_taskcluster(self): + """Returns boolean indicating if we're running in TaskCluster.""" + # This may need expanding in the future to work on + return "TASKCLUSTER_WORKER_TYPE" in os.environ + + +def PreScriptRun(func): + """Decorator for methods that will be called before script execution. + + Each method on a BaseScript having this decorator will be called at the + beginning of BaseScript.run(). + + The return value is ignored. Exceptions will abort execution. + """ + func._pre_run_listener = True + return func + + +def PostScriptRun(func): + """Decorator for methods that will be called after script execution. + + This is similar to PreScriptRun except it is called at the end of + execution. The method will always be fired, even if execution fails. + """ + func._post_run_listener = True + return func + + +def PreScriptAction(action=None): + """Decorator for methods that will be called at the beginning of each action. + + Each method on a BaseScript having this decorator will be called during + BaseScript.run() before an individual action is executed. The method will + receive the action's name as an argument. + + If no values are passed to the decorator, it will be applied to every + action. If a string is passed, the decorated function will only be called + for the action of that name. + + The return value of the method is ignored. Exceptions will abort execution. + """ + + def _wrapped(func): + func._pre_action_listener = action + return func + + def _wrapped_none(func): + func._pre_action_listener = None + return func + + if type(action) == type(_wrapped): + return _wrapped_none(action) + + return _wrapped + + +def PostScriptAction(action=None): + """Decorator for methods that will be called at the end of each action. + + This behaves similarly to PreScriptAction. It varies in that it is called + after execution of the action. + + The decorated method will receive the action name as a positional argument. + It will then receive the following named arguments: + + success - Bool indicating whether the action finished successfully. + + The decorated method will always be called, even if the action threw an + exception. + + The return value is ignored. + """ + + def _wrapped(func): + func._post_action_listener = action + return func + + def _wrapped_none(func): + func._post_action_listener = None + return func + + if type(action) == type(_wrapped): + return _wrapped_none(action) + + return _wrapped + + +# BaseScript {{{1 +class BaseScript(ScriptMixin, LogMixin, object): + def __init__( + self, + config_options=None, + ConfigClass=BaseConfig, + default_log_level="info", + **kwargs + ): + self._return_code = 0 + super(BaseScript, self).__init__() + + self.log_obj = None + self.abs_dirs = None + if config_options is None: + config_options = [] + self.summary_list = [] + self.failures = [] + rw_config = ConfigClass(config_options=config_options, **kwargs) + self.config = rw_config.get_read_only_config() + self.actions = tuple(rw_config.actions) + self.all_actions = tuple(rw_config.all_actions) + self.env = None + self.new_log_obj(default_log_level=default_log_level) + self.script_obj = self + + # Indicate we're a source checkout if VCS directory is present at the + # appropriate place. This code will break if this file is ever moved + # to another directory. + self.topsrcdir = None + + srcreldir = "testing/mozharness/mozharness/base" + here = os.path.normpath(os.path.dirname(__file__)) + if here.replace("\\", "/").endswith(srcreldir): + topsrcdir = os.path.normpath(os.path.join(here, "..", "..", "..", "..")) + hg_dir = os.path.join(topsrcdir, ".hg") + git_dir = os.path.join(topsrcdir, ".git") + if os.path.isdir(hg_dir) or os.path.isdir(git_dir): + self.topsrcdir = topsrcdir + + # Set self.config to read-only. + # + # We can create intermediate config info programmatically from + # this in a repeatable way, with logs; this is how we straddle the + # ideal-but-not-user-friendly static config and the + # easy-to-write-hard-to-debug writable config. + # + # To allow for other, script-specific configurations + # (e.g., props json parsing), before locking, + # call self._pre_config_lock(). If needed, this method can + # alter self.config. + self._pre_config_lock(rw_config) + self._config_lock() + + self.info("Run as %s" % rw_config.command_line) + if self.config.get("dump_config_hierarchy"): + # we only wish to dump and display what self.config is made up of, + # against the current script + args, without actually running any + # actions + self._dump_config_hierarchy(rw_config.all_cfg_files_and_dicts) + if self.config.get("dump_config"): + self.dump_config(exit_on_finish=True) + + # Collect decorated methods. We simply iterate over the attributes of + # the current class instance and look for signatures deposited by + # the decorators. + self._listeners = dict( + pre_run=[], + pre_action=[], + post_action=[], + post_run=[], + ) + for k in dir(self): + try: + item = self._getattr(k) + except Exception as e: + item = None + self.warning( + "BaseScript collecting decorated methods: " + "failure to get attribute {}: {}".format(k, str(e)) + ) + if not item: + continue + + # We only decorate methods, so ignore other types. + if not inspect.ismethod(item): + continue + + if hasattr(item, "_pre_run_listener"): + self._listeners["pre_run"].append(k) + + if hasattr(item, "_pre_action_listener"): + self._listeners["pre_action"].append((k, item._pre_action_listener)) + + if hasattr(item, "_post_action_listener"): + self._listeners["post_action"].append((k, item._post_action_listener)) + + if hasattr(item, "_post_run_listener"): + self._listeners["post_run"].append(k) + + def _getattr(self, name): + # `getattr(self, k)` will call the method `k` for any property + # access. If the property depends upon a module which has not + # been imported at the time the BaseScript initializer is + # executed, this property access will result in an + # Exception. Until Python 3's `inspect.getattr_static` is + # available, the simplest approach is to ignore the specific + # properties which are known to cause issues. Currently + # adb_path and device are ignored since they require the + # availablity of the mozdevice package which is not guaranteed + # when BaseScript is called. + property_list = set(["adb_path", "device"]) + if six.PY2: + if name in property_list: + item = None + else: + item = getattr(self, name) + else: + item = inspect.getattr_static(self, name) + if type(item) == property: + item = None + else: + item = getattr(self, name) + return item + + def _dump_config_hierarchy(self, cfg_files): + """interpret each config file used. + + This will show which keys/values are being added or overwritten by + other config files depending on their hierarchy (when they were added). + """ + # go through each config_file. We will start with the lowest and + # print its keys/values that are being used in self.config. If any + # keys/values are present in a config file with a higher precedence, + # ignore those. + dirs = self.query_abs_dirs() + cfg_files_dump_config = {} # we will dump this to file + # keep track of keys that did not come from a config file + keys_not_from_file = set(self.config.keys()) + if not cfg_files: + cfg_files = [] + self.info("Total config files: %d" % (len(cfg_files))) + if len(cfg_files): + self.info("cfg files used from lowest precedence to highest:") + for i, (target_file, target_dict) in enumerate(cfg_files): + unique_keys = set(target_dict.keys()) + unique_dict = {} + # iterate through the target_dicts remaining 'higher' cfg_files + remaining_cfgs = cfg_files[slice(i + 1, len(cfg_files))] + # where higher == more precedent + for ii, (higher_file, higher_dict) in enumerate(remaining_cfgs): + # now only keep keys/values that are not overwritten by a + # higher config + unique_keys = unique_keys.difference(set(higher_dict.keys())) + # unique_dict we know now has only keys/values that are unique to + # this config file. + unique_dict = dict((key, target_dict.get(key)) for key in unique_keys) + cfg_files_dump_config[target_file] = unique_dict + self.action_message("Config File %d: %s" % (i + 1, target_file)) + self.info(pprint.pformat(unique_dict)) + # let's also find out which keys/values from self.config are not + # from each target config file dict + keys_not_from_file = keys_not_from_file.difference(set(target_dict.keys())) + not_from_file_dict = dict( + (key, self.config.get(key)) for key in keys_not_from_file + ) + cfg_files_dump_config["not_from_cfg_file"] = not_from_file_dict + self.action_message( + "Not from any config file (default_config, " "cmd line options, etc)" + ) + self.info(pprint.pformat(not_from_file_dict)) + + # finally, let's dump this output as JSON and exit early + self.dump_config( + os.path.join(dirs["abs_log_dir"], "localconfigfiles.json"), + cfg_files_dump_config, + console_output=False, + exit_on_finish=True, + ) + + def _pre_config_lock(self, rw_config): + """This empty method can allow for config checking and manipulation + before the config lock, when overridden in scripts. + """ + pass + + def _config_lock(self): + """After this point, the config is locked and should not be + manipulated (based on mozharness.base.config.ReadOnlyDict) + """ + self.config.lock() + + def _possibly_run_method(self, method_name, error_if_missing=False): + """This is here for run().""" + if hasattr(self, method_name) and callable(self._getattr(method_name)): + return getattr(self, method_name)() + elif error_if_missing: + self.error("No such method %s!" % method_name) + + def run_action(self, action): + if action not in self.actions: + self.action_message("Skipping %s step." % action) + return + + method_name = action.replace("-", "_") + self.action_message("Running %s step." % action) + + # An exception during a pre action listener should abort execution. + for fn, target in self._listeners["pre_action"]: + if target is not None and target != action: + continue + + try: + self.info("Running pre-action listener: %s" % fn) + method = getattr(self, fn) + method(action) + except Exception: + self.error( + "Exception during pre-action for %s: %s" + % (action, traceback.format_exc()) + ) + + for fn, target in self._listeners["post_action"]: + if target is not None and target != action: + continue + + try: + self.info("Running post-action listener: %s" % fn) + method = getattr(self, fn) + method(action, success=False) + except Exception: + self.error( + "An additional exception occurred during " + "post-action for %s: %s" % (action, traceback.format_exc()) + ) + + self.fatal("Aborting due to exception in pre-action listener.") + + # We always run post action listeners, even if the main routine failed. + success = False + try: + self.info("Running main action method: %s" % method_name) + self._possibly_run_method("preflight_%s" % method_name) + self._possibly_run_method(method_name, error_if_missing=True) + self._possibly_run_method("postflight_%s" % method_name) + success = True + finally: + post_success = True + for fn, target in self._listeners["post_action"]: + if target is not None and target != action: + continue + + try: + self.info("Running post-action listener: %s" % fn) + method = getattr(self, fn) + method(action, success=success and self.return_code == 0) + except Exception: + post_success = False + self.error( + "Exception during post-action for %s: %s" + % (action, traceback.format_exc()) + ) + + step_result = "success" if success else "failed" + self.action_message("Finished %s step (%s)" % (action, step_result)) + + if not post_success: + self.fatal("Aborting due to failure in post-action listener.") + + def run(self): + """Default run method. + This is the "do everything" method, based on actions and all_actions. + + First run self.dump_config() if it exists. + Second, go through the list of all_actions. + If they're in the list of self.actions, try to run + self.preflight_ACTION(), self.ACTION(), and self.postflight_ACTION(). + + Preflight is sanity checking before doing anything time consuming or + destructive. + + Postflight is quick testing for success after an action. + + """ + for fn in self._listeners["pre_run"]: + try: + self.info("Running pre-run listener: %s" % fn) + method = getattr(self, fn) + method() + except Exception: + self.error( + "Exception during pre-run listener: %s" % traceback.format_exc() + ) + + for fn in self._listeners["post_run"]: + try: + method = getattr(self, fn) + method() + except Exception: + self.error( + "An additional exception occurred during a " + "post-run listener: %s" % traceback.format_exc() + ) + + self.fatal("Aborting due to failure in pre-run listener.") + + self.dump_config() + try: + for action in self.all_actions: + self.run_action(action) + except Exception: + self.fatal("Uncaught exception: %s" % traceback.format_exc()) + finally: + post_success = True + for fn in self._listeners["post_run"]: + try: + self.info("Running post-run listener: %s" % fn) + method = getattr(self, fn) + method() + except Exception: + post_success = False + self.error( + "Exception during post-run listener: %s" + % traceback.format_exc() + ) + + if not post_success: + self.fatal("Aborting due to failure in post-run listener.") + + return self.return_code + + def run_and_exit(self): + """Runs the script and exits the current interpreter.""" + rc = self.run() + if rc != 0: + self.warning("returning nonzero exit status %d" % rc) + sys.exit(rc) + + def clobber(self): + """ + Delete the working directory + """ + dirs = self.query_abs_dirs() + self.rmtree(dirs["abs_work_dir"], error_level=FATAL) + + def query_abs_dirs(self): + """We want to be able to determine where all the important things + are. Absolute paths lend themselves well to this, though I wouldn't + be surprised if this causes some issues somewhere. + + This should be overridden in any script that has additional dirs + to query. + + The query_* methods tend to set self.VAR variables as their + runtime cache. + """ + if self.abs_dirs: + return self.abs_dirs + c = self.config + dirs = {} + dirs["base_work_dir"] = c["base_work_dir"] + dirs["abs_work_dir"] = os.path.join(c["base_work_dir"], c["work_dir"]) + dirs["abs_log_dir"] = os.path.join(c["base_work_dir"], c.get("log_dir", "logs")) + if "GECKO_PATH" in os.environ: + dirs["abs_src_dir"] = os.environ["GECKO_PATH"] + self.abs_dirs = dirs + return self.abs_dirs + + def dump_config( + self, file_path=None, config=None, console_output=True, exit_on_finish=False + ): + """Dump self.config to localconfig.json""" + config = config or self.config + dirs = self.query_abs_dirs() + if not file_path: + file_path = os.path.join(dirs["abs_log_dir"], "localconfig.json") + self.info("Dumping config to %s." % file_path) + self.mkdir_p(os.path.dirname(file_path)) + json_config = json.dumps(config, sort_keys=True, indent=4) + fh = codecs.open(file_path, encoding="utf-8", mode="w+") + fh.write(json_config) + fh.close() + if console_output: + self.info(pprint.pformat(config)) + if exit_on_finish: + sys.exit() + + # logging {{{2 + def new_log_obj(self, default_log_level="info"): + c = self.config + log_dir = os.path.join(c["base_work_dir"], c.get("log_dir", "logs")) + log_config = { + "logger_name": "Simple", + "log_name": "log", + "log_dir": log_dir, + "log_level": default_log_level, + "log_format": "%(asctime)s %(levelname)8s - %(message)s", + "log_to_console": True, + "append_to_log": False, + } + log_type = self.config.get("log_type", "console") + for key in log_config.keys(): + value = self.config.get(key, None) + if value is not None: + log_config[key] = value + if log_type == "multi": + self.log_obj = MultiFileLogger(**log_config) + elif log_type == "simple": + self.log_obj = SimpleFileLogger(**log_config) + else: + self.log_obj = ConsoleLogger(**log_config) + + def action_message(self, message): + self.info( + "[mozharness: %sZ] %s" + % (datetime.datetime.utcnow().isoformat(" "), message) + ) + + def summary(self): + """Print out all the summary lines added via add_summary() + throughout the script. + + I'd like to revisit how to do this in a prettier fashion. + """ + self.action_message("%s summary:" % self.__class__.__name__) + if self.summary_list: + for item in self.summary_list: + try: + self.log(item["message"], level=item["level"]) + except ValueError: + """log is closed; print as a default. Ran into this + when calling from __del__()""" + print("### Log is closed! (%s)" % item["message"]) + + def add_summary(self, message, level=INFO): + self.summary_list.append({"message": message, "level": level}) + # TODO write to a summary-only log? + # Summaries need a lot more love. + self.log(message, level=level) + + def summarize_success_count( + self, success_count, total_count, message="%d of %d successful.", level=None + ): + if level is None: + level = INFO + if success_count < total_count: + level = ERROR + self.add_summary(message % (success_count, total_count), level=level) + + def get_hash_for_file(self, file_path, hash_type="sha512"): + bs = 65536 + hasher = hashlib.new(hash_type) + with open(file_path, "rb") as fh: + buf = fh.read(bs) + while len(buf) > 0: + hasher.update(buf) + buf = fh.read(bs) + return hasher.hexdigest() + + @property + def return_code(self): + return self._return_code + + @return_code.setter + def return_code(self, code): + old_return_code, self._return_code = self._return_code, code + if old_return_code != code: + self.warning("setting return code to %d" % code) diff --git a/testing/mozharness/mozharness/base/transfer.py b/testing/mozharness/mozharness/base/transfer.py new file mode 100755 index 0000000000..610e93ecc9 --- /dev/null +++ b/testing/mozharness/mozharness/base/transfer.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +"""Generic ways to upload + download files. +""" + +import pprint + +try: + from urllib2 import urlopen +except ImportError: + from urllib.request import urlopen + +import json + +from mozharness.base.log import DEBUG + + +# TransferMixin {{{1 +class TransferMixin(object): + """ + Generic transfer methods. + + Dependent on BaseScript. + """ + + def load_json_from_url(self, url, timeout=30, log_level=DEBUG): + self.log( + "Attempting to download %s; timeout=%i" % (url, timeout), level=log_level + ) + try: + r = urlopen(url, timeout=timeout) + j = json.load(r) + self.log(pprint.pformat(j), level=log_level) + except BaseException: + self.exception(message="Unable to download %s!" % url) + raise + return j diff --git a/testing/mozharness/mozharness/base/vcs/__init__.py b/testing/mozharness/mozharness/base/vcs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/mozharness/mozharness/base/vcs/__init__.py diff --git a/testing/mozharness/mozharness/base/vcs/gittool.py b/testing/mozharness/mozharness/base/vcs/gittool.py new file mode 100644 index 0000000000..e9d0c0e2c9 --- /dev/null +++ b/testing/mozharness/mozharness/base/vcs/gittool.py @@ -0,0 +1,107 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import re + +try: + import urlparse +except ImportError: + import urllib.parse as urlparse + +from mozharness.base.errors import GitErrorList, VCSException +from mozharness.base.log import LogMixin, OutputParser +from mozharness.base.script import ScriptMixin + + +class GittoolParser(OutputParser): + """ + A class that extends OutputParser such that it can find the "Got revision" + string from gittool.py output + """ + + got_revision_exp = re.compile(r"Got revision (\w+)") + got_revision = None + + def parse_single_line(self, line): + m = self.got_revision_exp.match(line) + if m: + self.got_revision = m.group(1) + super(GittoolParser, self).parse_single_line(line) + + +class GittoolVCS(ScriptMixin, LogMixin): + def __init__(self, log_obj=None, config=None, vcs_config=None, script_obj=None): + super(GittoolVCS, self).__init__() + + self.log_obj = log_obj + self.script_obj = script_obj + if config: + self.config = config + else: + self.config = {} + # vcs_config = { + # repo: repository, + # branch: branch, + # revision: revision, + # ssh_username: ssh_username, + # ssh_key: ssh_key, + # } + self.vcs_config = vcs_config + self.gittool = self.query_exe("gittool.py", return_type="list") + + def ensure_repo_and_revision(self): + """Makes sure that `dest` is has `revision` or `branch` checked out + from `repo`. + + Do what it takes to make that happen, including possibly clobbering + dest. + """ + c = self.vcs_config + for conf_item in ("dest", "repo"): + assert self.vcs_config[conf_item] + dest = os.path.abspath(c["dest"]) + repo = c["repo"] + revision = c.get("revision") + branch = c.get("branch") + clean = c.get("clean") + share_base = c.get("vcs_share_base", os.environ.get("GIT_SHARE_BASE_DIR", None)) + env = {"PATH": os.environ.get("PATH")} + env.update(c.get("env", {})) + if self._is_windows(): + # git.exe is not in the PATH by default + env["PATH"] = "%s;C:/mozilla-build/Git/bin" % env["PATH"] + # SYSTEMROOT is needed for 'import random' + if "SYSTEMROOT" not in env: + env["SYSTEMROOT"] = os.environ.get("SYSTEMROOT") + if share_base is not None: + env["GIT_SHARE_BASE_DIR"] = share_base + + cmd = self.gittool[:] + if branch: + cmd.extend(["-b", branch]) + if revision: + cmd.extend(["-r", revision]) + if clean: + cmd.append("--clean") + + for base_mirror_url in self.config.get( + "gittool_base_mirror_urls", self.config.get("vcs_base_mirror_urls", []) + ): + bits = urlparse.urlparse(repo) + mirror_url = urlparse.urljoin(base_mirror_url, bits.path) + cmd.extend(["--mirror", mirror_url]) + + cmd.extend([repo, dest]) + parser = GittoolParser( + config=self.config, log_obj=self.log_obj, error_list=GitErrorList + ) + retval = self.run_command( + cmd, error_list=GitErrorList, env=env, output_parser=parser + ) + + if retval != 0: + raise VCSException("Unable to checkout") + + return parser.got_revision diff --git a/testing/mozharness/mozharness/base/vcs/mercurial.py b/testing/mozharness/mozharness/base/vcs/mercurial.py new file mode 100755 index 0000000000..63b0d27c34 --- /dev/null +++ b/testing/mozharness/mozharness/base/vcs/mercurial.py @@ -0,0 +1,478 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +"""Mercurial VCS support. +""" + +import hashlib +import os +import re +import subprocess +import sys +from collections import namedtuple + +try: + from urlparse import urlsplit +except ImportError: + from urllib.parse import urlsplit + +import mozharness +from mozharness.base.errors import HgErrorList, VCSException +from mozharness.base.log import LogMixin, OutputParser +from mozharness.base.script import ScriptMixin +from mozharness.base.transfer import TransferMixin + +sys.path.insert(1, os.path.dirname(os.path.dirname(os.path.dirname(sys.path[0])))) + + +external_tools_path = os.path.join( + os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))), + "external_tools", +) + + +HG_OPTIONS = ["--config", "ui.merge=internal:merge"] + +# MercurialVCS {{{1 +# TODO Make the remaining functions more mozharness-friendly. +# TODO Add the various tag functionality that are currently in +# build/tools/scripts to MercurialVCS -- generic tagging logic belongs here. +REVISION, BRANCH = 0, 1 + + +class RepositoryUpdateRevisionParser(OutputParser): + """Parse `hg pull` output for "repository unrelated" errors.""" + + revision = None + RE_UPDATED = re.compile("^updated to ([a-f0-9]{40})$") + + def parse_single_line(self, line): + m = self.RE_UPDATED.match(line) + if m: + self.revision = m.group(1) + + return super(RepositoryUpdateRevisionParser, self).parse_single_line(line) + + +def make_hg_url(hg_host, repo_path, protocol="http", revision=None, filename=None): + """Helper function. + + Construct a valid hg url from a base hg url (hg.mozilla.org), + repo_path, revision and possible filename + """ + base = "%s://%s" % (protocol, hg_host) + repo = "/".join(p.strip("/") for p in [base, repo_path]) + if not filename: + if not revision: + return repo + else: + return "/".join([p.strip("/") for p in [repo, "rev", revision]]) + else: + assert revision + return "/".join([p.strip("/") for p in [repo, "raw-file", revision, filename]]) + + +class MercurialVCS(ScriptMixin, LogMixin, TransferMixin): + # For the most part, scripts import mercurial, update + # tag-release.py imports + # apply_and_push, update, get_revision, out, BRANCH, REVISION, + # get_branches, cleanOutgoingRevs + + def __init__(self, log_obj=None, config=None, vcs_config=None, script_obj=None): + super(MercurialVCS, self).__init__() + self.can_share = None + self.log_obj = log_obj + self.script_obj = script_obj + if config: + self.config = config + else: + self.config = {} + # vcs_config = { + # hg_host: hg_host, + # repo: repository, + # branch: branch, + # revision: revision, + # ssh_username: ssh_username, + # ssh_key: ssh_key, + # } + self.vcs_config = vcs_config or {} + self.hg = self.query_exe("hg", return_type="list") + HG_OPTIONS + + def _make_absolute(self, repo): + if repo.startswith("file://"): + path = repo[len("file://") :] + repo = "file://%s" % os.path.abspath(path) + elif "://" not in repo: + repo = os.path.abspath(repo) + return repo + + def get_repo_name(self, repo): + return repo.rstrip("/").split("/")[-1] + + def get_repo_path(self, repo): + repo = self._make_absolute(repo) + if repo.startswith("/"): + return repo.lstrip("/") + else: + return urlsplit(repo).path.lstrip("/") + + def get_revision_from_path(self, path): + """Returns which revision directory `path` currently has checked out.""" + return self.get_output_from_command( + self.hg + ["parent", "--template", "{node}"], cwd=path + ) + + def get_branch_from_path(self, path): + branch = self.get_output_from_command(self.hg + ["branch"], cwd=path) + return str(branch).strip() + + def get_branches_from_path(self, path): + branches = [] + for line in self.get_output_from_command( + self.hg + ["branches", "-c"], cwd=path + ).splitlines(): + branches.append(line.split()[0]) + return branches + + def hg_ver(self): + """Returns the current version of hg, as a tuple of + (major, minor, build)""" + ver_string = self.get_output_from_command(self.hg + ["-q", "version"]) + match = re.search(r"\(version ([0-9.]+)\)", ver_string) + if match: + bits = match.group(1).split(".") + if len(bits) < 3: + bits += (0,) + ver = tuple(int(b) for b in bits) + else: + ver = (0, 0, 0) + self.debug("Running hg version %s" % str(ver)) + return ver + + def update(self, dest, branch=None, revision=None): + """Updates working copy `dest` to `branch` or `revision`. + If revision is set, branch will be ignored. + If neither is set then the working copy will be updated to the + latest revision on the current branch. Local changes will be + discarded. + """ + # If we have a revision, switch to that + msg = "Updating %s" % dest + if branch: + msg += " to branch %s" % branch + if revision: + msg += " revision %s" % revision + self.info("%s." % msg) + if revision is not None: + cmd = self.hg + ["update", "-C", "-r", revision] + if self.run_command(cmd, cwd=dest, error_list=HgErrorList): + raise VCSException("Unable to update %s to %s!" % (dest, revision)) + else: + # Check & switch branch + local_branch = self.get_branch_from_path(dest) + + cmd = self.hg + ["update", "-C"] + + # If this is different, checkout the other branch + if branch and branch != local_branch: + cmd.append(branch) + + if self.run_command(cmd, cwd=dest, error_list=HgErrorList): + raise VCSException("Unable to update %s!" % dest) + return self.get_revision_from_path(dest) + + def clone(self, repo, dest, branch=None, revision=None, update_dest=True): + """Clones hg repo and places it at `dest`, replacing whatever else + is there. The working copy will be empty. + + If `revision` is set, only the specified revision and its ancestors + will be cloned. If revision is set, branch is ignored. + + If `update_dest` is set, then `dest` will be updated to `revision` + if set, otherwise to `branch`, otherwise to the head of default. + """ + msg = "Cloning %s to %s" % (repo, dest) + if branch: + msg += " on branch %s" % branch + if revision: + msg += " to revision %s" % revision + self.info("%s." % msg) + parent_dest = os.path.dirname(dest) + if parent_dest and not os.path.exists(parent_dest): + self.mkdir_p(parent_dest) + if os.path.exists(dest): + self.info("Removing %s before clone." % dest) + self.rmtree(dest) + + cmd = self.hg + ["clone"] + if not update_dest: + cmd.append("-U") + + if revision: + cmd.extend(["-r", revision]) + elif branch: + # hg >= 1.6 supports -b branch for cloning + ver = self.hg_ver() + if ver >= (1, 6, 0): + cmd.extend(["-b", branch]) + + cmd.extend([repo, dest]) + output_timeout = self.config.get( + "vcs_output_timeout", self.vcs_config.get("output_timeout") + ) + if ( + self.run_command(cmd, error_list=HgErrorList, output_timeout=output_timeout) + != 0 + ): + raise VCSException("Unable to clone %s to %s!" % (repo, dest)) + + if update_dest: + return self.update(dest, branch, revision) + + def common_args(self, revision=None, branch=None, ssh_username=None, ssh_key=None): + """Fill in common hg arguments, encapsulating logic checks that + depend on mercurial versions and provided arguments + """ + args = [] + if ssh_username or ssh_key: + opt = ["-e", "ssh"] + if ssh_username: + opt[1] += " -l %s" % ssh_username + if ssh_key: + opt[1] += " -i %s" % ssh_key + args.extend(opt) + if revision: + args.extend(["-r", revision]) + elif branch: + if self.hg_ver() >= (1, 6, 0): + args.extend(["-b", branch]) + return args + + def pull(self, repo, dest, update_dest=True, **kwargs): + """Pulls changes from hg repo and places it in `dest`. + + If `revision` is set, only the specified revision and its ancestors + will be pulled. + + If `update_dest` is set, then `dest` will be updated to `revision` + if set, otherwise to `branch`, otherwise to the head of default. + """ + msg = "Pulling %s to %s" % (repo, dest) + if update_dest: + msg += " and updating" + self.info("%s." % msg) + if not os.path.exists(dest): + # Error or clone? + # If error, should we have a halt_on_error=False above? + self.error("Can't hg pull in nonexistent directory %s." % dest) + return -1 + # Convert repo to an absolute path if it's a local repository + repo = self._make_absolute(repo) + cmd = self.hg + ["pull"] + cmd.extend(self.common_args(**kwargs)) + cmd.append(repo) + output_timeout = self.config.get( + "vcs_output_timeout", self.vcs_config.get("output_timeout") + ) + if ( + self.run_command( + cmd, cwd=dest, error_list=HgErrorList, output_timeout=output_timeout + ) + != 0 + ): + raise VCSException("Can't pull in %s!" % dest) + + if update_dest: + branch = self.vcs_config.get("branch") + revision = self.vcs_config.get("revision") + return self.update(dest, branch=branch, revision=revision) + + # Defines the places of attributes in the tuples returned by `out' + + def out(self, src, remote, **kwargs): + """Check for outgoing changesets present in a repo""" + self.info("Checking for outgoing changesets from %s to %s." % (src, remote)) + cmd = self.hg + ["-q", "out", "--template", "{node} {branches}\n"] + cmd.extend(self.common_args(**kwargs)) + cmd.append(remote) + if os.path.exists(src): + try: + revs = [] + for line in ( + self.get_output_from_command(cmd, cwd=src, throw_exception=True) + .rstrip() + .split("\n") + ): + try: + rev, branch = line.split() + # Mercurial displays no branch at all if the revision + # is on "default" + except ValueError: + rev = line.rstrip() + branch = "default" + revs.append((rev, branch)) + return revs + except subprocess.CalledProcessError as inst: + # In some situations, some versions of Mercurial return "1" + # if no changes are found, so we need to ignore this return + # code + if inst.returncode == 1: + return [] + raise + + def push(self, src, remote, push_new_branches=True, **kwargs): + # This doesn't appear to work with hg_ver < (1, 6, 0). + # Error out, or let you try? + self.info("Pushing new changes from %s to %s." % (src, remote)) + cmd = self.hg + ["push"] + cmd.extend(self.common_args(**kwargs)) + if push_new_branches and self.hg_ver() >= (1, 6, 0): + cmd.append("--new-branch") + cmd.append(remote) + status = self.run_command( + cmd, + cwd=src, + error_list=HgErrorList, + success_codes=(0, 1), + return_type="num_errors", + ) + if status: + raise VCSException("Can't push %s to %s!" % (src, remote)) + return status + + @property + def robustcheckout_path(self): + """Path to the robustcheckout extension.""" + ext = os.path.join(external_tools_path, "robustcheckout.py") + if os.path.exists(ext): + return ext + + def ensure_repo_and_revision(self): + """Makes sure that `dest` is has `revision` or `branch` checked out + from `repo`. + + Do what it takes to make that happen, including possibly clobbering + dest. + """ + c = self.vcs_config + dest = c["dest"] + repo_url = c["repo"] + rev = c.get("revision") + branch = c.get("branch") + purge = c.get("clone_with_purge", False) + upstream = c.get("clone_upstream_url") + + # The API here is kind of bad because we're relying on state in + # self.vcs_config instead of passing arguments. This confuses + # scripts that have multiple repos. This includes the clone_tools() + # step :( + + if not rev and not branch: + self.warning('did not specify revision or branch; assuming "default"') + branch = "default" + + share_base = c.get("vcs_share_base") or os.environ.get("HG_SHARE_BASE_DIR") + if share_base and c.get("use_vcs_unique_share"): + # Bug 1277041 - update migration scripts to support robustcheckout + # fake a share but don't really share + share_base = os.path.join(share_base, hashlib.md5(dest).hexdigest()) + + # We require shared storage is configured because it guarantees we + # only have 1 local copy of logical repo stores. + if not share_base: + raise VCSException( + "vcs share base not defined; " "refusing to operate sub-optimally" + ) + + if not self.robustcheckout_path: + raise VCSException("could not find the robustcheckout Mercurial extension") + + # Log HG version and install info to aid debugging. + self.run_command(self.hg + ["--version"]) + self.run_command(self.hg + ["debuginstall", "--config=ui.username=worker"]) + + args = self.hg + [ + "--config", + "extensions.robustcheckout=%s" % self.robustcheckout_path, + "robustcheckout", + repo_url, + dest, + "--sharebase", + share_base, + ] + if purge: + args.append("--purge") + if upstream: + args.extend(["--upstream", upstream]) + + if rev: + args.extend(["--revision", rev]) + if branch: + args.extend(["--branch", branch]) + + parser = RepositoryUpdateRevisionParser( + config=self.config, log_obj=self.log_obj + ) + if self.run_command(args, output_parser=parser): + raise VCSException("repo checkout failed!") + + if not parser.revision: + raise VCSException("could not identify revision updated to") + + return parser.revision + + def cleanOutgoingRevs(self, reponame, remote, username, sshKey): + # TODO retry + self.info("Wiping outgoing local changes from %s to %s." % (reponame, remote)) + outgoingRevs = self.out( + src=reponame, remote=remote, ssh_username=username, ssh_key=sshKey + ) + for r in reversed(outgoingRevs): + self.run_command( + self.hg + ["strip", "-n", r[REVISION]], + cwd=reponame, + error_list=HgErrorList, + ) + + def query_pushinfo(self, repository, revision): + """Query the pushdate and pushid of a repository/revision. + This is intended to be used on hg.mozilla.org/mozilla-central and + similar. It may or may not work for other hg repositories. + """ + PushInfo = namedtuple("PushInfo", ["pushid", "pushdate"]) + + try: + url = "%s/json-pushes?changeset=%s" % (repository, revision) + self.info("Pushdate URL is: %s" % url) + contents = self.retry(self.load_json_from_url, args=(url,)) + + # The contents should be something like: + # { + # "28537": { + # "changesets": [ + # "1d0a914ae676cc5ed203cdc05c16d8e0c22af7e5", + # ], + # "date": 1428072488, + # "user": "user@mozilla.com" + # } + # } + # + # So we grab the first element ("28537" in this case) and then pull + # out the 'date' field. + pushid = next(contents.keys()) + self.info("Pushid is: %s" % pushid) + pushdate = contents[pushid]["date"] + self.info("Pushdate is: %s" % pushdate) + return PushInfo(pushid, pushdate) + + except Exception: + self.exception("Failed to get push info from hg.mozilla.org") + raise + + +# __main__ {{{1 +if __name__ == "__main__": + pass diff --git a/testing/mozharness/mozharness/base/vcs/vcsbase.py b/testing/mozharness/mozharness/base/vcs/vcsbase.py new file mode 100755 index 0000000000..c587a8b1ca --- /dev/null +++ b/testing/mozharness/mozharness/base/vcs/vcsbase.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +"""Generic VCS support. +""" + +import os +import sys +from copy import deepcopy + +from mozharness.base.errors import VCSException +from mozharness.base.log import FATAL +from mozharness.base.script import BaseScript +from mozharness.base.vcs.gittool import GittoolVCS +from mozharness.base.vcs.mercurial import MercurialVCS + +sys.path.insert(1, os.path.dirname(os.path.dirname(os.path.dirname(sys.path[0])))) + + +# Update this with supported VCS name : VCS object +VCS_DICT = { + "hg": MercurialVCS, + "gittool": GittoolVCS, +} + + +# VCSMixin {{{1 +class VCSMixin(object): + """Basic VCS methods that are vcs-agnostic. + The vcs_class handles all the vcs-specific tasks. + """ + + def query_dest(self, kwargs): + if "dest" in kwargs: + return kwargs["dest"] + dest = os.path.basename(kwargs["repo"]) + # Git fun + if dest.endswith(".git"): + dest = dest.replace(".git", "") + return dest + + def _get_revision(self, vcs_obj, dest): + try: + got_revision = vcs_obj.ensure_repo_and_revision() + if got_revision: + return got_revision + except VCSException: + self.rmtree(dest) + raise + + def _get_vcs_class(self, vcs): + vcs = vcs or self.config.get("default_vcs", getattr(self, "default_vcs", None)) + vcs_class = VCS_DICT.get(vcs) + return vcs_class + + def vcs_checkout(self, vcs=None, error_level=FATAL, **kwargs): + """Check out a single repo.""" + c = self.config + vcs_class = self._get_vcs_class(vcs) + if not vcs_class: + self.error("Running vcs_checkout with kwargs %s" % str(kwargs)) + raise VCSException("No VCS set!") + # need a better way to do this. + if "dest" not in kwargs: + kwargs["dest"] = self.query_dest(kwargs) + if "vcs_share_base" not in kwargs: + kwargs["vcs_share_base"] = c.get( + "%s_share_base" % vcs, c.get("vcs_share_base") + ) + vcs_obj = vcs_class( + log_obj=self.log_obj, + config=self.config, + vcs_config=kwargs, + script_obj=self, + ) + return self.retry( + self._get_revision, + error_level=error_level, + error_message="Automation Error: Can't checkout %s!" % kwargs["repo"], + args=(vcs_obj, kwargs["dest"]), + ) + + def vcs_checkout_repos( + self, repo_list, parent_dir=None, tag_override=None, **kwargs + ): + """Check out a list of repos.""" + orig_dir = os.getcwd() + c = self.config + if not parent_dir: + parent_dir = os.path.join(c["base_work_dir"], c["work_dir"]) + self.mkdir_p(parent_dir) + self.chdir(parent_dir) + revision_dict = {} + kwargs_orig = deepcopy(kwargs) + for repo_dict in repo_list: + kwargs = deepcopy(kwargs_orig) + kwargs.update(repo_dict) + if tag_override: + kwargs["branch"] = tag_override + dest = self.query_dest(kwargs) + revision_dict[dest] = {"repo": kwargs["repo"]} + revision_dict[dest]["revision"] = self.vcs_checkout(**kwargs) + self.chdir(orig_dir) + return revision_dict + + def vcs_query_pushinfo(self, repository, revision, vcs=None): + """Query the pushid/pushdate of a repository/revision + Returns a namedtuple with "pushid" and "pushdate" elements + """ + vcs_class = self._get_vcs_class(vcs) + if not vcs_class: + raise VCSException("No VCS set in vcs_query_pushinfo!") + vcs_obj = vcs_class( + log_obj=self.log_obj, + config=self.config, + script_obj=self, + ) + return vcs_obj.query_pushinfo(repository, revision) + + +class VCSScript(VCSMixin, BaseScript): + def __init__(self, **kwargs): + super(VCSScript, self).__init__(**kwargs) + + def pull(self, repos=None, parent_dir=None): + repos = repos or self.config.get("repos") + if not repos: + self.info("Pull has nothing to do!") + return + dirs = self.query_abs_dirs() + parent_dir = parent_dir or dirs["abs_work_dir"] + return self.vcs_checkout_repos(repos, parent_dir=parent_dir) + + +# Specific VCS stubs {{{1 +# For ease of use. +# This is here instead of mercurial.py because importing MercurialVCS into +# vcsbase from mercurial, and importing VCSScript into mercurial from +# vcsbase, was giving me issues. +class MercurialScript(VCSScript): + default_vcs = "hg" + + +# __main__ {{{1 +if __name__ == "__main__": + pass diff --git a/testing/mozharness/mozharness/lib/__init__.py b/testing/mozharness/mozharness/lib/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/mozharness/mozharness/lib/__init__.py diff --git a/testing/mozharness/mozharness/lib/python/__init__.py b/testing/mozharness/mozharness/lib/python/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/mozharness/mozharness/lib/python/__init__.py diff --git a/testing/mozharness/mozharness/lib/python/authentication.py b/testing/mozharness/mozharness/lib/python/authentication.py new file mode 100644 index 0000000000..5d7330e357 --- /dev/null +++ b/testing/mozharness/mozharness/lib/python/authentication.py @@ -0,0 +1,60 @@ +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** + +"""module for http authentication operations""" +import getpass +import os + +CREDENTIALS_PATH = os.path.expanduser("~/.mozilla/credentials.cfg") +DIRNAME = os.path.dirname(CREDENTIALS_PATH) +LDAP_PASSWORD = None + + +def get_credentials(): + """Returns http credentials. + + The user's email address is stored on disk (for convenience in the future) + while the password is requested from the user on first invocation. + """ + global LDAP_PASSWORD + if not os.path.exists(DIRNAME): + os.makedirs(DIRNAME) + + if os.path.isfile(CREDENTIALS_PATH): + with open(CREDENTIALS_PATH, "r") as file_handler: + content = file_handler.read().splitlines() + + https_username = content[0].strip() + + if len(content) > 1: + # We want to remove files which contain the password + os.remove(CREDENTIALS_PATH) + else: + try: + # pylint: disable=W1609 + input_method = raw_input + except NameError: + input_method = input + + https_username = input_method("Please enter your full LDAP email address: ") + + with open(CREDENTIALS_PATH, "w+") as file_handler: + file_handler.write("%s\n" % https_username) + + os.chmod(CREDENTIALS_PATH, 0o600) + + if not LDAP_PASSWORD: + print("Please enter your LDAP password (we won't store it):") + LDAP_PASSWORD = getpass.getpass() + + return https_username, LDAP_PASSWORD + + +def get_credentials_path(): + if os.path.isfile(CREDENTIALS_PATH): + get_credentials() + + return CREDENTIALS_PATH diff --git a/testing/mozharness/mozharness/mozilla/__init__.py b/testing/mozharness/mozharness/mozilla/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/__init__.py diff --git a/testing/mozharness/mozharness/mozilla/automation.py b/testing/mozharness/mozharness/mozilla/automation.py new file mode 100644 index 0000000000..0158f800ed --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/automation.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +"""Code to integration with automation. +""" + +try: + import simplejson as json + + assert json +except ImportError: + import json + +from mozharness.base.log import ERROR, INFO, WARNING + +TBPL_SUCCESS = "SUCCESS" +TBPL_WARNING = "WARNING" +TBPL_FAILURE = "FAILURE" +TBPL_EXCEPTION = "EXCEPTION" +TBPL_RETRY = "RETRY" +TBPL_STATUS_DICT = { + TBPL_SUCCESS: INFO, + TBPL_WARNING: WARNING, + TBPL_FAILURE: ERROR, + TBPL_EXCEPTION: ERROR, + TBPL_RETRY: WARNING, +} +EXIT_STATUS_DICT = { + TBPL_SUCCESS: 0, + TBPL_WARNING: 1, + TBPL_FAILURE: 2, + TBPL_EXCEPTION: 3, + TBPL_RETRY: 4, +} +TBPL_WORST_LEVEL_TUPLE = ( + TBPL_RETRY, + TBPL_EXCEPTION, + TBPL_FAILURE, + TBPL_WARNING, + TBPL_SUCCESS, +) + + +class AutomationMixin(object): + worst_status = TBPL_SUCCESS + properties = {} + + def tryserver_email(self): + pass + + def record_status(self, tbpl_status, level=None, set_return_code=True): + if tbpl_status not in TBPL_STATUS_DICT: + self.error("record_status() doesn't grok the status %s!" % tbpl_status) + else: + if not level: + level = TBPL_STATUS_DICT[tbpl_status] + self.worst_status = self.worst_level( + tbpl_status, self.worst_status, TBPL_WORST_LEVEL_TUPLE + ) + if self.worst_status != tbpl_status: + self.info( + "Current worst status %s is worse; keeping it." % self.worst_status + ) + if set_return_code: + self.return_code = EXIT_STATUS_DICT[self.worst_status] + + def add_failure(self, key, message="%(key)s failed.", level=ERROR): + if key not in self.failures: + self.failures.append(key) + self.add_summary(message % {"key": key}, level=level) + self.record_status(TBPL_FAILURE) + + def query_failure(self, key): + return key in self.failures + + def query_is_nightly(self): + """returns whether or not the script should run as a nightly build.""" + return bool(self.config.get("nightly_build")) diff --git a/testing/mozharness/mozharness/mozilla/bouncer/__init__.py b/testing/mozharness/mozharness/mozilla/bouncer/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/bouncer/__init__.py diff --git a/testing/mozharness/mozharness/mozilla/bouncer/submitter.py b/testing/mozharness/mozharness/mozilla/bouncer/submitter.py new file mode 100644 index 0000000000..e9289d2be6 --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/bouncer/submitter.py @@ -0,0 +1,134 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import base64 +import socket +import sys +import traceback +from xml.dom.minidom import parseString + +from mozharness.base.log import FATAL + +try: + import httplib +except ImportError: + import http.client as httplib +try: + from urllib import quote, urlencode +except ImportError: + from urllib.parse import quote, urlencode +try: + from urllib2 import HTTPError, Request, URLError, urlopen +except ImportError: + from urllib.request import HTTPError, Request, URLError, urlopen + + +class BouncerSubmitterMixin(object): + def query_credentials(self): + if self.credentials: + return self.credentials + global_dict = {} + local_dict = {} + exec( + compile( + open(self.config["credentials_file"], "rb").read(), + self.config["credentials_file"], + "exec", + ), + global_dict, + local_dict, + ) + self.credentials = (local_dict["tuxedoUsername"], local_dict["tuxedoPassword"]) + return self.credentials + + def api_call(self, route, data, error_level=FATAL, retry_config=None): + retry_args = dict( + failure_status=None, + retry_exceptions=( + HTTPError, + URLError, + httplib.BadStatusLine, + socket.timeout, + socket.error, + ), + error_message="call to %s failed" % (route), + error_level=error_level, + ) + + if retry_config: + retry_args.update(retry_config) + + return self.retry(self._api_call, args=(route, data), **retry_args) + + def _api_call(self, route, data): + api_prefix = self.config["bouncer-api-prefix"] + api_url = "%s/%s" % (api_prefix, route) + request = Request(api_url) + if data: + post_data = urlencode(data, doseq=True) + request.add_data(post_data) + self.info("POST data: %s" % post_data) + credentials = self.query_credentials() + if credentials: + auth = base64.encodestring("%s:%s" % credentials) + request.add_header("Authorization", "Basic %s" % auth.strip()) + try: + self.info("Submitting to %s" % api_url) + res = urlopen(request, timeout=60).read() + self.info("Server response") + self.info(res) + return res + except HTTPError as e: + self.warning("Cannot access %s" % api_url) + traceback.print_exc(file=sys.stdout) + self.warning("Returned page source:") + self.warning(e.read()) + raise + except URLError: + traceback.print_exc(file=sys.stdout) + self.warning("Cannot access %s" % api_url) + raise + except socket.timeout as e: + self.warning("Timed out accessing %s: %s" % (api_url, e)) + raise + except socket.error as e: + self.warning("Socket error when accessing %s: %s" % (api_url, e)) + raise + except httplib.BadStatusLine as e: + self.warning("BadStatusLine accessing %s: %s" % (api_url, e)) + raise + + def product_exists(self, product_name): + self.info("Checking if %s already exists" % product_name) + res = self.api_call("product_show?product=%s" % quote(product_name), data=None) + try: + xml = parseString(res) + # API returns <products/> if the product doesn't exist + products_found = len(xml.getElementsByTagName("product")) + self.info("Products found: %s" % products_found) + return bool(products_found) + except Exception as e: + self.warning("Error parsing XML: %s" % e) + self.warning("Assuming %s does not exist" % product_name) + # ignore XML parsing errors + return False + + def api_add_product(self, product_name, add_locales, ssl_only=False): + data = { + "product": product_name, + } + if self.locales and add_locales: + data["languages"] = self.locales + if ssl_only: + # Send "true" as a string + data["ssl_only"] = "true" + self.api_call("product_add/", data) + + def api_add_location(self, product_name, bouncer_platform, path): + data = { + "product": product_name, + "os": bouncer_platform, + "path": path, + } + self.api_call("location_add/", data) diff --git a/testing/mozharness/mozharness/mozilla/building/__init__.py b/testing/mozharness/mozharness/mozilla/building/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/building/__init__.py diff --git a/testing/mozharness/mozharness/mozilla/building/buildbase.py b/testing/mozharness/mozharness/mozilla/building/buildbase.py new file mode 100755 index 0000000000..01392325a0 --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py @@ -0,0 +1,1527 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +""" buildbase.py. + +provides a base class for fx desktop builds +author: Jordan Lund + +""" +import copy +import json +import os +import re +import sys +import time +import uuid +from datetime import datetime + +import six +import yaml +from yaml import YAMLError + +from mozharness.base.config import DEFAULT_CONFIG_PATH, BaseConfig, parse_config_file +from mozharness.base.errors import MakefileErrorList +from mozharness.base.log import ERROR, FATAL, OutputParser +from mozharness.base.python import PerfherderResourceOptionsMixin, VirtualenvMixin +from mozharness.base.script import PostScriptRun +from mozharness.base.vcs.vcsbase import MercurialScript +from mozharness.mozilla.automation import ( + EXIT_STATUS_DICT, + TBPL_FAILURE, + TBPL_RETRY, + TBPL_STATUS_DICT, + TBPL_SUCCESS, + TBPL_WORST_LEVEL_TUPLE, + AutomationMixin, +) +from mozharness.mozilla.secrets import SecretsMixin + +AUTOMATION_EXIT_CODES = sorted(EXIT_STATUS_DICT.values()) + +MISSING_CFG_KEY_MSG = "The key '%s' could not be determined \ +Please add this to your config." + +ERROR_MSGS = { + "comments_undetermined": '"comments" could not be determined. This may be \ +because it was a forced build.', + "tooltool_manifest_undetermined": '"tooltool_manifest_src" not set, \ +Skipping run_tooltool...', +} + + +# Output Parsers + +TBPL_UPLOAD_ERRORS = [ + { + "regex": re.compile("Connection timed out"), + "level": TBPL_RETRY, + }, + { + "regex": re.compile("Connection reset by peer"), + "level": TBPL_RETRY, + }, + { + "regex": re.compile("Connection refused"), + "level": TBPL_RETRY, + }, +] + + +class MakeUploadOutputParser(OutputParser): + tbpl_error_list = TBPL_UPLOAD_ERRORS + + def __init__(self, **kwargs): + super(MakeUploadOutputParser, self).__init__(**kwargs) + self.tbpl_status = TBPL_SUCCESS + + def parse_single_line(self, line): + # let's check for retry errors which will give log levels: + # tbpl status as RETRY and mozharness status as WARNING + for error_check in self.tbpl_error_list: + if error_check["regex"].search(line): + self.num_warnings += 1 + self.warning(line) + self.tbpl_status = self.worst_level( + error_check["level"], + self.tbpl_status, + levels=TBPL_WORST_LEVEL_TUPLE, + ) + break + else: + self.info(line) + + +class MozconfigPathError(Exception): + """ + There was an error getting a mozconfig path from a mozharness config. + """ + + +def get_mozconfig_path(script, config, dirs): + """ + Get the path to the mozconfig file to use from a mozharness config. + + :param script: The object to interact with the filesystem through. + :type script: ScriptMixin: + + :param config: The mozharness config to inspect. + :type config: dict + + :param dirs: The directories specified for this build. + :type dirs: dict + """ + COMPOSITE_KEYS = {"mozconfig_variant", "app_name", "mozconfig_platform"} + have_composite_mozconfig = COMPOSITE_KEYS <= set(config.keys()) + have_partial_composite_mozconfig = len(COMPOSITE_KEYS & set(config.keys())) > 0 + have_src_mozconfig = "src_mozconfig" in config + have_src_mozconfig_manifest = "src_mozconfig_manifest" in config + + # first determine the mozconfig path + if have_partial_composite_mozconfig and not have_composite_mozconfig: + raise MozconfigPathError( + "All or none of 'app_name', 'mozconfig_platform' and `mozconfig_variant' must be " + "in the config in order to determine the mozconfig." + ) + elif have_composite_mozconfig and have_src_mozconfig: + raise MozconfigPathError( + "'src_mozconfig' or 'mozconfig_variant' must be " + "in the config but not both in order to determine the mozconfig." + ) + elif have_composite_mozconfig and have_src_mozconfig_manifest: + raise MozconfigPathError( + "'src_mozconfig_manifest' or 'mozconfig_variant' must be " + "in the config but not both in order to determine the mozconfig." + ) + elif have_src_mozconfig and have_src_mozconfig_manifest: + raise MozconfigPathError( + "'src_mozconfig' or 'src_mozconfig_manifest' must be " + "in the config but not both in order to determine the mozconfig." + ) + elif have_composite_mozconfig: + src_mozconfig = "%(app_name)s/config/mozconfigs/%(platform)s/%(variant)s" % { + "app_name": config["app_name"], + "platform": config["mozconfig_platform"], + "variant": config["mozconfig_variant"], + } + abs_mozconfig_path = os.path.join(dirs["abs_src_dir"], src_mozconfig) + elif have_src_mozconfig: + abs_mozconfig_path = os.path.join( + dirs["abs_src_dir"], config.get("src_mozconfig") + ) + elif have_src_mozconfig_manifest: + manifest = os.path.join(dirs["abs_work_dir"], config["src_mozconfig_manifest"]) + if not os.path.exists(manifest): + raise MozconfigPathError( + 'src_mozconfig_manifest: "%s" not found. Does it exist?' % (manifest,) + ) + else: + with script.opened(manifest, error_level=ERROR) as (fh, err): + if err: + raise MozconfigPathError( + "%s exists but coud not read properties" % manifest + ) + abs_mozconfig_path = os.path.join( + dirs["abs_src_dir"], json.load(fh)["gecko_path"] + ) + else: + raise MozconfigPathError( + "Must provide 'app_name', 'mozconfig_platform' and 'mozconfig_variant'; " + "or one of 'src_mozconfig' or 'src_mozconfig_manifest' in the config " + "in order to determine the mozconfig." + ) + + return abs_mozconfig_path + + +class BuildingConfig(BaseConfig): + # TODO add nosetests for this class + def get_cfgs_from_files(self, all_config_files, options): + """ + Determine the configuration from the normal options and from + `--branch`, `--build-pool`, and `--custom-build-variant-cfg`. If the + files for any of the latter options are also given with `--config-file` + or `--opt-config-file`, they are only parsed once. + + The build pool has highest precedence, followed by branch, build + variant, and any normally-specified configuration files. + """ + # override from BaseConfig + + # this is what we will return. It will represent each config + # file name and its associated dict + # eg ('builds/branch_specifics.py', {'foo': 'bar'}) + all_config_dicts = [] + # important config files + variant_cfg_file = pool_cfg_file = "" + + # we want to make the order in which the options were given + # not matter. ie: you can supply --branch before --build-pool + # or vice versa and the hierarchy will not be different + + # ### The order from highest precedence to lowest is: + # # There can only be one of these... + # 1) build_pool: this can be either staging, pre-prod, and prod cfgs + # 2) build_variant: these could be known like asan and debug + # or a custom config + # + # # There can be many of these + # 3) all other configs: these are any configs that are passed with + # --cfg and --opt-cfg. There order is kept in + # which they were passed on the cmd line. This + # behaviour is maintains what happens by default + # in mozharness + + # so, let's first assign the configs that hold a known position of + # importance (1 through 3) + for i, cf in enumerate(all_config_files): + if options.build_pool: + if cf == BuildOptionParser.build_pool_cfg_file: + pool_cfg_file = all_config_files[i] + + if cf == options.build_variant: + variant_cfg_file = all_config_files[i] + + # now remove these from the list if there was any. + # we couldn't pop() these in the above loop as mutating a list while + # iterating through it causes spurious results :) + for cf in [pool_cfg_file, variant_cfg_file]: + if cf: + all_config_files.remove(cf) + + # now let's update config with the remaining config files. + # this functionality is the same as the base class + all_config_dicts.extend( + super(BuildingConfig, self).get_cfgs_from_files(all_config_files, options) + ) + + # stack variant, branch, and pool cfg files on top of that, + # if they are present, in that order + if variant_cfg_file: + # take the whole config + all_config_dicts.append( + (variant_cfg_file, parse_config_file(variant_cfg_file)) + ) + config_paths = options.config_paths or ["."] + if pool_cfg_file: + # take only the specific pool. If we are here, the pool + # must be present + build_pool_configs = parse_config_file( + pool_cfg_file, search_path=config_paths + [DEFAULT_CONFIG_PATH] + ) + all_config_dicts.append( + (pool_cfg_file, build_pool_configs[options.build_pool]) + ) + return all_config_dicts + + +# noinspection PyUnusedLocal +class BuildOptionParser(object): + # TODO add nosetests for this class + platform = None + bits = None + + # add to this list and you can automagically do things like + # --custom-build-variant-cfg asan + # and the script will pull up the appropriate path for the config + # against the current platform and bits. + # *It will warn and fail if there is not a config for the current + # platform/bits + path_base = "builds/releng_sub_%s_configs/" + build_variants = { + "add-on-devel": path_base + "%s_add-on-devel.py", + "asan": path_base + "%s_asan.py", + "asan-tc": path_base + "%s_asan_tc.py", + "asan-reporter-tc": path_base + "%s_asan_reporter_tc.py", + "fuzzing-asan-tc": path_base + "%s_fuzzing_asan_tc.py", + "tsan-tc": path_base + "%s_tsan_tc.py", + "fuzzing-tsan-tc": path_base + "%s_fuzzing_tsan_tc.py", + "cross-debug": path_base + "%s_cross_debug.py", + "cross-debug-searchfox": path_base + "%s_cross_debug_searchfox.py", + "cross-noopt-debug": path_base + "%s_cross_noopt_debug.py", + "cross-fuzzing-asan": path_base + "%s_cross_fuzzing_asan.py", + "cross-fuzzing-debug": path_base + "%s_cross_fuzzing_debug.py", + "debug": path_base + "%s_debug.py", + "fuzzing-debug": path_base + "%s_fuzzing_debug.py", + "asan-and-debug": path_base + "%s_asan_and_debug.py", + "asan-tc-and-debug": path_base + "%s_asan_tc_and_debug.py", + "stat-and-debug": path_base + "%s_stat_and_debug.py", + "code-coverage-debug": path_base + "%s_code_coverage_debug.py", + "code-coverage-opt": path_base + "%s_code_coverage_opt.py", + "source": path_base + "%s_source.py", + "noopt-debug": path_base + "%s_noopt_debug.py", + "arm-gradle-dependencies": path_base + + "%s_arm_gradle_dependencies.py", # NOQA: E501 + "arm": path_base + "%s_arm.py", + "arm-lite": path_base + "%s_arm_lite.py", + "arm-beta": path_base + "%s_arm_beta.py", + "arm-beta-debug": path_base + "%s_arm_beta_debug.py", + "arm-debug": path_base + "%s_arm_debug.py", + "arm-lite-debug": path_base + "%s_arm_debug_lite.py", + "arm-debug-ccov": path_base + "%s_arm_debug_ccov.py", + "arm-debug-searchfox": path_base + "%s_arm_debug_searchfox.py", + "arm-gradle": path_base + "%s_arm_gradle.py", + "rusttests": path_base + "%s_rusttests.py", + "rusttests-debug": path_base + "%s_rusttests_debug.py", + "x86": path_base + "%s_x86.py", + "x86-lite": path_base + "%s_x86_lite.py", + "x86-beta": path_base + "%s_x86_beta.py", + "x86-beta-debug": path_base + "%s_x86_beta_debug.py", + "x86-debug": path_base + "%s_x86_debug.py", + "x86-lite-debug": path_base + "%s_x86_debug_lite.py", + "x86-profile-generate": path_base + "%s_x86_profile_generate.py", + "x86_64": path_base + "%s_x86_64.py", + "x86_64-lite": path_base + "%s_x86_64_lite.py", + "x86_64-beta": path_base + "%s_x86_64_beta.py", + "x86_64-beta-debug": path_base + "%s_x86_64_beta_debug.py", + "x86_64-debug": path_base + "%s_x86_64_debug.py", + "x86_64-lite-debug": path_base + "%s_x86_64_debug_lite.py", + "x86_64-debug-isolated-process": path_base + + "%s_x86_64_debug_isolated_process.py", + "x86_64-profile-generate": path_base + "%s_x86_64_profile_generate.py", + "arm-partner-sample1": path_base + "%s_arm_partner_sample1.py", + "aarch64": path_base + "%s_aarch64.py", + "aarch64-lite": path_base + "%s_aarch64_lite.py", + "aarch64-beta": path_base + "%s_aarch64_beta.py", + "aarch64-beta-debug": path_base + "%s_aarch64_beta_debug.py", + "aarch64-pgo": path_base + "%s_aarch64_pgo.py", + "aarch64-debug": path_base + "%s_aarch64_debug.py", + "aarch64-lite-debug": path_base + "%s_aarch64_debug_lite.py", + "android-geckoview-docs": path_base + "%s_geckoview_docs.py", + "valgrind": path_base + "%s_valgrind.py", + } + build_pool_cfg_file = "builds/build_pool_specifics.py" + + @classmethod + def _query_pltfrm_and_bits(cls, target_option, options): + """determine platform and bits + + This can be from either from a supplied --platform and --bits + or parsed from given config file names. + """ + error_msg = ( + "Whoops!\nYou are trying to pass a shortname for " + "%s. \nHowever, I need to know the %s to find the appropriate " + 'filename. You can tell me by passing:\n\t"%s" or a config ' + 'filename via "--config" with %s in it. \nIn either case, these ' + "option arguments must come before --custom-build-variant." + ) + current_config_files = options.config_files or [] + if not cls.bits: + # --bits has not been supplied + # lets parse given config file names for 32 or 64 + for cfg_file_name in current_config_files: + if "32" in cfg_file_name: + cls.bits = "32" + break + if "64" in cfg_file_name: + cls.bits = "64" + break + else: + sys.exit(error_msg % (target_option, "bits", "--bits", '"32" or "64"')) + + if not cls.platform: + # --platform has not been supplied + # lets parse given config file names for platform + for cfg_file_name in current_config_files: + if "windows" in cfg_file_name: + cls.platform = "windows" + break + if "mac" in cfg_file_name: + cls.platform = "mac" + break + if "linux" in cfg_file_name: + cls.platform = "linux" + break + if "android" in cfg_file_name: + cls.platform = "android" + break + else: + sys.exit( + error_msg + % ( + target_option, + "platform", + "--platform", + '"linux", "windows", "mac", or "android"', + ) + ) + return cls.bits, cls.platform + + @classmethod + def find_variant_cfg_path(cls, opt, value, parser): + valid_variant_cfg_path = None + # first let's see if we were given a valid short-name + if cls.build_variants.get(value): + bits, pltfrm = cls._query_pltfrm_and_bits(opt, parser.values) + prospective_cfg_path = cls.build_variants[value] % (pltfrm, bits) + else: + # this is either an incomplete path or an invalid key in + # build_variants + prospective_cfg_path = value + + if os.path.exists(prospective_cfg_path): + # now let's see if we were given a valid pathname + valid_variant_cfg_path = value + else: + # FIXME: We should actually wait until we have parsed all arguments + # before looking at this, otherwise the behavior will depend on the + # order of arguments. But that isn't a problem as long as --extra-config-path + # is always passed first. + extra_config_paths = parser.values.config_paths or [] + config_paths = extra_config_paths + [DEFAULT_CONFIG_PATH] + # let's take our prospective_cfg_path and see if we can + # determine an existing file + for path in config_paths: + if os.path.exists(os.path.join(path, prospective_cfg_path)): + # success! we found a config file + valid_variant_cfg_path = os.path.join(path, prospective_cfg_path) + break + return valid_variant_cfg_path, prospective_cfg_path + + @classmethod + def set_build_variant(cls, option, opt, value, parser): + """sets an extra config file. + + This is done by either taking an existing filepath or by taking a valid + shortname coupled with known platform/bits. + """ + valid_variant_cfg_path, prospective_cfg_path = cls.find_variant_cfg_path( + "--custom-build-variant-cfg", value, parser + ) + + if not valid_variant_cfg_path: + # either the value was an indeterminable path or an invalid short + # name + sys.exit( + "Whoops!\n'--custom-build-variant' was passed but an " + "appropriate config file could not be determined. Tried " + "using: '%s' but it was not:" + "\n\t-- a valid shortname: %s " + "\n\t-- a valid variant for the given platform and bits." + % (prospective_cfg_path, str(list(cls.build_variants.keys()))) + ) + parser.values.config_files.append(valid_variant_cfg_path) + setattr(parser.values, option.dest, value) # the pool + + @classmethod + def set_build_pool(cls, option, opt, value, parser): + # first let's add the build pool file where there may be pool + # specific keys/values. Then let's store the pool name + parser.values.config_files.append(cls.build_pool_cfg_file) + setattr(parser.values, option.dest, value) # the pool + + @classmethod + def set_build_branch(cls, option, opt, value, parser): + # Store the branch name we are using + setattr(parser.values, option.dest, value) # the branch name + + @classmethod + def set_platform(cls, option, opt, value, parser): + cls.platform = value + setattr(parser.values, option.dest, value) + + @classmethod + def set_bits(cls, option, opt, value, parser): + cls.bits = value + setattr(parser.values, option.dest, value) + + +# this global depends on BuildOptionParser and therefore can not go at the +# top of the file +BUILD_BASE_CONFIG_OPTIONS = [ + [ + ["--developer-run"], + { + "action": "store_false", + "dest": "is_automation", + "default": True, + "help": "If this is running outside of Mozilla's build" + "infrastructure, use this option. It ignores actions" + "that are not needed and adds config checks.", + }, + ], + [ + ["--platform"], + { + "action": "callback", + "callback": BuildOptionParser.set_platform, + "type": "string", + "dest": "platform", + "help": "Sets the platform we are running this against" + " valid values: 'windows', 'mac', 'linux'", + }, + ], + [ + ["--bits"], + { + "action": "callback", + "callback": BuildOptionParser.set_bits, + "type": "string", + "dest": "bits", + "help": "Sets which bits we are building this against" + " valid values: '32', '64'", + }, + ], + [ + ["--custom-build-variant-cfg"], + { + "action": "callback", + "callback": BuildOptionParser.set_build_variant, + "type": "string", + "dest": "build_variant", + "help": "Sets the build type and will determine appropriate" + " additional config to use. Either pass a config path" + " or use a valid shortname from: " + "%s" % (list(BuildOptionParser.build_variants.keys()),), + }, + ], + [ + ["--build-pool"], + { + "action": "callback", + "callback": BuildOptionParser.set_build_pool, + "type": "string", + "dest": "build_pool", + "help": "This will update the config with specific pool" + " environment keys/values. The dicts for this are" + " in %s\nValid values: staging or" + " production" % ("builds/build_pool_specifics.py",), + }, + ], + [ + ["--branch"], + { + "action": "callback", + "callback": BuildOptionParser.set_build_branch, + "type": "string", + "dest": "branch", + "help": "This sets the branch we will be building this for.", + }, + ], + [ + ["--enable-nightly"], + { + "action": "store_true", + "dest": "nightly_build", + "default": False, + "help": "Sets the build to run in nightly mode", + }, + ], + [ + ["--who"], + { + "dest": "who", + "default": "", + "help": "stores who made the created the change.", + }, + ], +] + + +def generate_build_ID(): + return time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())) + + +def generate_build_UID(): + return uuid.uuid4().hex + + +class BuildScript( + AutomationMixin, + VirtualenvMixin, + MercurialScript, + SecretsMixin, + PerfherderResourceOptionsMixin, +): + def __init__(self, **kwargs): + # objdir is referenced in _query_abs_dirs() so let's make sure we + # have that attribute before calling BaseScript.__init__ + self.objdir = None + super(BuildScript, self).__init__(**kwargs) + # epoch is only here to represent the start of the build + # that this mozharn script came from. until I can grab bbot's + # status.build.gettime()[0] this will have to do as a rough estimate + # although it is about 4s off from the time it would be if it was + # done through MBF. + # TODO find out if that time diff matters or if we just use it to + # separate each build + self.epoch_timestamp = int(time.mktime(datetime.now().timetuple())) + self.branch = self.config.get("branch") + self.stage_platform = self.config.get("stage_platform") + if not self.branch or not self.stage_platform: + if not self.branch: + self.error("'branch' not determined and is required") + if not self.stage_platform: + self.error("'stage_platform' not determined and is required") + self.fatal("Please add missing items to your config") + self.client_id = None + self.access_token = None + + # Call this before creating the virtualenv so that we can support + # substituting config values with other config values. + self.query_build_env() + + # We need to create the virtualenv directly (without using an action) in + # order to use python modules in PreScriptRun/Action listeners + self.create_virtualenv() + + def _pre_config_lock(self, rw_config): + c = self.config + cfg_files_and_dicts = rw_config.all_cfg_files_and_dicts + build_pool = c.get("build_pool", "") + build_variant = c.get("build_variant", "") + variant_cfg = "" + if build_variant: + variant_cfg = BuildOptionParser.build_variants[build_variant] % ( + BuildOptionParser.platform, + BuildOptionParser.bits, + ) + build_pool_cfg = BuildOptionParser.build_pool_cfg_file + + cfg_match_msg = "Script was run with '%(option)s %(type)s' and \ +'%(type)s' matches a key in '%(type_config_file)s'. Updating self.config with \ +items from that key's value." + + for i, (target_file, target_dict) in enumerate(cfg_files_and_dicts): + if build_pool_cfg and build_pool_cfg in target_file: + self.info( + cfg_match_msg + % { + "option": "--build-pool", + "type": build_pool, + "type_config_file": build_pool_cfg, + } + ) + if variant_cfg and variant_cfg in target_file: + self.info( + cfg_match_msg + % { + "option": "--custom-build-variant-cfg", + "type": build_variant, + "type_config_file": variant_cfg, + } + ) + self.info( + "To generate a config file based upon options passed and " + "config files used, run script as before but extend options " + 'with "--dump-config"' + ) + self.info( + "For a diff of where self.config got its items, " + "run the script again as before but extend options with: " + '"--dump-config-hierarchy"' + ) + self.info( + "Both --dump-config and --dump-config-hierarchy don't " + "actually run any actions." + ) + + def _query_objdir(self): + if self.objdir: + return self.objdir + + if not self.config.get("objdir"): + return self.fatal(MISSING_CFG_KEY_MSG % ("objdir",)) + self.objdir = self.config["objdir"] + return self.objdir + + def query_is_nightly_promotion(self): + platform_enabled = self.config.get("enable_nightly_promotion") + branch_enabled = self.branch in self.config.get("nightly_promotion_branches") + return platform_enabled and branch_enabled + + def query_build_env(self, **kwargs): + c = self.config + + # let's evoke the base query_env and make a copy of it + # as we don't always want every key below added to the same dict + env = copy.deepcopy(super(BuildScript, self).query_env(**kwargs)) + + if self.query_is_nightly() or self.query_is_nightly_promotion(): + # taskcluster sets the update channel for shipping builds + # explicitly + if c.get("update_channel"): + update_channel = c["update_channel"] + if six.PY2 and isinstance(update_channel, six.text_type): + update_channel = update_channel.encode("utf-8") + env["MOZ_UPDATE_CHANNEL"] = update_channel + else: # let's just give the generic channel based on branch + env["MOZ_UPDATE_CHANNEL"] = "nightly-%s" % (self.branch,) + self.info("Update channel set to: {}".format(env["MOZ_UPDATE_CHANNEL"])) + + return env + + def query_mach_build_env(self, multiLocale=None): + c = self.config + if multiLocale is None and self.query_is_nightly(): + multiLocale = c.get("multi_locale", False) + mach_env = {} + if c.get("upload_env"): + mach_env.update(c["upload_env"]) + + # this prevents taskcluster from overwriting the target files with + # the multilocale files. Put everything from the en-US build in a + # separate folder. + if multiLocale and self.config.get("taskcluster_nightly"): + if "UPLOAD_PATH" in mach_env: + mach_env["UPLOAD_PATH"] = os.path.join(mach_env["UPLOAD_PATH"], "en-US") + return mach_env + + def _get_mozconfig(self): + """assign mozconfig.""" + dirs = self.query_abs_dirs() + + try: + abs_mozconfig_path = get_mozconfig_path( + script=self, config=self.config, dirs=dirs + ) + except MozconfigPathError as e: + if six.PY2: + self.fatal(e.message) + else: + self.fatal(e.msg) + + self.info("Use mozconfig: {}".format(abs_mozconfig_path)) + + # print its contents + content = self.read_from_file(abs_mozconfig_path, error_level=FATAL) + + extra_content = self.config.get("extra_mozconfig_content") + if extra_content: + content += "\n".join(extra_content) + + self.info("mozconfig content:") + self.info(content) + + # finally, copy the mozconfig to a path that 'mach build' expects it to + # be + with open(os.path.join(dirs["abs_src_dir"], ".mozconfig"), "w") as fh: + fh.write(content) + + def _run_tooltool(self): + env = self.query_build_env() + env.update(self.query_mach_build_env()) + + c = self.config + dirs = self.query_abs_dirs() + manifest_src = os.environ.get("TOOLTOOL_MANIFEST") + if not manifest_src: + manifest_src = c.get("tooltool_manifest_src") + if not manifest_src: + return self.warning(ERROR_MSGS["tooltool_manifest_undetermined"]) + cmd = [ + sys.executable, + "-u", + os.path.join(dirs["abs_src_dir"], "mach"), + "artifact", + "toolchain", + "-v", + "--retry", + "4", + "--artifact-manifest", + os.path.join(dirs["abs_src_dir"], "toolchains.json"), + ] + if manifest_src: + cmd.extend( + [ + "--tooltool-manifest", + os.path.join(dirs["abs_src_dir"], manifest_src), + ] + ) + cache = c["env"].get("TOOLTOOL_CACHE") + if cache: + cmd.extend(["--cache-dir", cache]) + self.info(str(cmd)) + self.run_command(cmd, cwd=dirs["abs_src_dir"], halt_on_failure=True, env=env) + + def _create_mozbuild_dir(self, mozbuild_path=None): + if not mozbuild_path: + env = self.query_build_env() + mozbuild_path = env.get("MOZBUILD_STATE_PATH") + if mozbuild_path: + self.mkdir_p(mozbuild_path) + else: + self.warning( + "mozbuild_path could not be determined. skipping " "creating it." + ) + + def preflight_build(self): + """set up machine state for a complete build.""" + self._get_mozconfig() + self._run_tooltool() + self._create_mozbuild_dir() + self._ensure_upload_path() + + def build(self): + """builds application.""" + + args = ["build", "-v"] + + # This will error on non-0 exit code. + self._run_mach_command_in_build_env(args) + + self._generate_build_stats() + + def static_analysis_autotest(self): + """Run mach static-analysis autotest, in order to make sure we dont regress""" + self.preflight_build() + self._run_mach_command_in_build_env(["configure"]) + self._run_mach_command_in_build_env( + ["static-analysis", "autotest", "--intree-tool"], use_subprocess=True + ) + + def _query_mach(self): + return [sys.executable, "mach"] + + def _run_mach_command_in_build_env(self, args, use_subprocess=False): + """Run a mach command in a build context.""" + env = self.query_build_env() + env.update(self.query_mach_build_env()) + + dirs = self.query_abs_dirs() + + mach = self._query_mach() + + # XXX See bug 1483883 + # Work around an interaction between Gradle and mozharness + # Not using `subprocess` causes gradle to hang + if use_subprocess: + import subprocess + + return_code = subprocess.call( + mach + ["--log-no-times"] + args, env=env, cwd=dirs["abs_src_dir"] + ) + else: + return_code = self.run_command( + command=mach + ["--log-no-times"] + args, + cwd=dirs["abs_src_dir"], + env=env, + error_list=MakefileErrorList, + output_timeout=self.config.get("max_build_output_timeout", 60 * 40), + ) + + if return_code: + self.return_code = self.worst_level( + EXIT_STATUS_DICT[TBPL_FAILURE], + self.return_code, + AUTOMATION_EXIT_CODES[::-1], + ) + self.fatal( + "'mach %s' did not run successfully. Please check " + "log for errors." % " ".join(args) + ) + + def multi_l10n(self): + if not self.query_is_nightly(): + self.info("Not a nightly build, skipping multi l10n.") + return + + dirs = self.query_abs_dirs() + base_work_dir = dirs["base_work_dir"] + work_dir = dirs["abs_work_dir"] + objdir = dirs["abs_obj_dir"] + branch = self.branch + + # Building a nightly with the try repository fails because a + # config-file does not exist for try. Default to mozilla-central + # settings (arbitrarily). + if branch == "try": + branch = "mozilla-central" + + multil10n_path = os.path.join( + dirs["abs_src_dir"], + "testing/mozharness/scripts/multil10n.py", + ) + + cmd = [ + sys.executable, + multil10n_path, + "--work-dir", + work_dir, + "--config-file", + "multi_locale/android-mozharness-build.json", + "--pull-locale-source", + "--package-multi", + "--summary", + ] + + self.run_command( + cmd, env=self.query_build_env(), cwd=base_work_dir, halt_on_failure=True + ) + + package_cmd = [ + "make", + "echo-variable-PACKAGE", + "AB_CD=multi", + ] + package_filename = self.get_output_from_command( + package_cmd, + cwd=objdir, + ) + if not package_filename: + self.fatal( + "Unable to determine the package filename for the multi-l10n build. " + "Was trying to run: %s" % package_cmd + ) + + self.info("Multi-l10n package filename is: %s" % package_filename) + + parser = MakeUploadOutputParser( + config=self.config, + log_obj=self.log_obj, + ) + upload_cmd = ["make", "upload", "AB_CD=multi"] + self.run_command( + upload_cmd, + partial_env=self.query_mach_build_env(multiLocale=False), + cwd=objdir, + halt_on_failure=True, + output_parser=parser, + ) + upload_files_cmd = [ + "make", + "echo-variable-UPLOAD_FILES", + "AB_CD=multi", + ] + self.get_output_from_command( + upload_files_cmd, + cwd=objdir, + ) + + def postflight_build(self): + """grabs properties from post build and calls ccache -s""" + # A list of argument lists. Better names gratefully accepted! + mach_commands = self.config.get("postflight_build_mach_commands", []) + for mach_command in mach_commands: + self._execute_postflight_build_mach_command(mach_command) + + def _execute_postflight_build_mach_command(self, mach_command_args): + env = self.query_build_env() + env.update(self.query_mach_build_env()) + + command = [sys.executable, "mach", "--log-no-times"] + command.extend(mach_command_args) + + self.run_command( + command=command, + cwd=self.query_abs_dirs()["abs_src_dir"], + env=env, + output_timeout=self.config.get("max_build_output_timeout", 60 * 20), + halt_on_failure=True, + ) + + def preflight_package_source(self): + self._get_mozconfig() + + def package_source(self): + """generates source archives and uploads them""" + env = self.query_build_env() + env.update(self.query_mach_build_env()) + dirs = self.query_abs_dirs() + + self.run_command( + command=[sys.executable, "mach", "--log-no-times", "configure"], + cwd=dirs["abs_src_dir"], + env=env, + output_timeout=60 * 3, + halt_on_failure=True, + ) + self.run_command( + command=[ + "make", + "source-package", + "source-upload", + ], + cwd=dirs["abs_obj_dir"], + env=env, + output_timeout=60 * 45, + halt_on_failure=True, + ) + + def _is_configuration_shipped(self): + """Determine if the current build configuration is shipped to users. + + This is used to drive alerting so we don't see alerts for build + configurations we care less about. + """ + # Ideally this would be driven by a config option. However, our + # current inheritance mechanism of using a base config and then + # one-off configs for variants isn't conducive to this since derived + # configs we need to be reset and we don't like requiring boilerplate + # in derived configs. + + # Debug builds are never shipped. + if self.config.get("debug_build"): + return False + + # OS X opt builds without a variant are shipped. + if self.config.get("platform") == "macosx64": + if not self.config.get("build_variant"): + return True + + # Android opt builds without a variant are shipped. + if self.config.get("platform") == "android": + if not self.config.get("build_variant"): + return True + + return False + + def _load_build_resources(self): + p = self.config.get("build_resources_path") % self.query_abs_dirs() + if not os.path.exists(p): + self.info("%s does not exist; not loading build resources" % p) + return None + + with open(p, "r") as fh: + resources = json.load(fh) + + if "duration" not in resources: + self.info("resource usage lacks duration; ignoring") + return None + + # We want to always collect metrics. But alerts with sccache enabled + # we should disable automatic alerting + should_alert = False if os.environ.get("USE_SCCACHE") == "1" else True + + data = { + "name": "build times", + "value": resources["duration"], + "extraOptions": self.perfherder_resource_options(), + "shouldAlert": should_alert, + "subtests": [], + } + + for phase in resources["phases"]: + if "duration" not in phase: + continue + data["subtests"].append( + { + "name": phase["name"], + "value": phase["duration"], + } + ) + + return data + + def _load_sccache_stats(self): + stats_file = os.path.join( + self.query_abs_dirs()["abs_obj_dir"], "sccache-stats.json" + ) + if not os.path.exists(stats_file): + self.info("%s does not exist; not loading sccache stats" % stats_file) + return + + with open(stats_file, "r") as fh: + stats = json.load(fh) + + def get_stat(key): + val = stats["stats"][key] + # Future versions of sccache will distinguish stats by language + # and store them as a dict. + if isinstance(val, dict): + val = sum(val["counts"].values()) + return val + + total = get_stat("requests_executed") + hits = get_stat("cache_hits") + if total > 0: + hits /= float(total) + + yield { + "name": "sccache hit rate", + "value": hits, + "subtests": [], + "alertThreshold": 50.0, + "lowerIsBetter": False, + # We want to always collect metrics. + # But disable automatic alerting on it + "shouldAlert": False, + } + + yield { + "name": "sccache cache_write_errors", + "value": stats["stats"]["cache_write_errors"], + "alertThreshold": 50.0, + "subtests": [], + } + + yield { + "name": "sccache requests_not_cacheable", + "value": stats["stats"]["requests_not_cacheable"], + "alertThreshold": 50.0, + "subtests": [], + } + + def _get_package_metrics(self): + import tarfile + import zipfile + + dirs = self.query_abs_dirs() + + dist_dir = os.path.join(dirs["abs_obj_dir"], "dist") + for ext in ["apk", "dmg", "tar.bz2", "zip"]: + name = "target." + ext + if os.path.exists(os.path.join(dist_dir, name)): + packageName = name + break + else: + self.fatal("could not determine packageName") + + interests = ["libxul.so", "classes.dex", "omni.ja", "xul.dll"] + installer = os.path.join(dist_dir, packageName) + installer_size = 0 + size_measurements = [] + + def paths_with_sizes(installer): + if zipfile.is_zipfile(installer): + with zipfile.ZipFile(installer, "r") as zf: + for zi in zf.infolist(): + yield zi.filename, zi.file_size + elif tarfile.is_tarfile(installer): + with tarfile.open(installer, "r:*") as tf: + for ti in tf: + yield ti.name, ti.size + + if os.path.exists(installer): + installer_size = self.query_filesize(installer) + self.info("Size of %s: %s bytes" % (packageName, installer_size)) + try: + subtests = {} + for path, size in paths_with_sizes(installer): + name = os.path.basename(path) + if name in interests: + # We have to be careful here: desktop Firefox installers + # contain two omni.ja files: one for the general runtime, + # and one for the browser proper. + if name == "omni.ja": + containing_dir = os.path.basename(os.path.dirname(path)) + if containing_dir == "browser": + name = "browser-omni.ja" + if name in subtests: + self.fatal( + "should not see %s (%s) multiple times!" % (name, path) + ) + subtests[name] = size + for name in subtests: + self.info("Size of %s: %s bytes" % (name, subtests[name])) + size_measurements.append({"name": name, "value": subtests[name]}) + except Exception: + self.info("Unable to search %s for component sizes." % installer) + size_measurements = [] + + if not installer_size and not size_measurements: + return + + # We want to always collect metrics. But alerts for installer size are + # only use for builds with ship. So nix the alerts for builds we don't + # ship. + def filter_alert(alert): + if not self._is_configuration_shipped(): + alert["shouldAlert"] = False + + return alert + + if installer.endswith(".apk"): # Android + yield filter_alert( + { + "name": "installer size", + "value": installer_size, + "alertChangeType": "absolute", + "alertThreshold": (200 * 1024), + "subtests": size_measurements, + } + ) + else: + yield filter_alert( + { + "name": "installer size", + "value": installer_size, + "alertChangeType": "absolute", + "alertThreshold": (100 * 1024), + "subtests": size_measurements, + } + ) + + def _get_sections(self, file, filter=None): + """ + Returns a dictionary of sections and their sizes. + """ + # Check for `rust_size`, our cross platform version of size. It should + # be fetched by run-task in $MOZ_FETCHES_DIR/rust-size/rust-size + rust_size = os.path.join( + os.environ["MOZ_FETCHES_DIR"], "rust-size", "rust-size" + ) + size_prog = self.which(rust_size) + if not size_prog: + self.info("Couldn't find `rust-size` program") + return {} + + self.info("Using %s" % size_prog) + cmd = [size_prog, file] + output = self.get_output_from_command(cmd) + if not output: + self.info("`rust-size` failed") + return {} + + # Format is JSON: + # { + # "section_type": { + # "section_name": size, .... + # }, + # ... + # } + try: + parsed = json.loads(output) + except ValueError: + self.info("`rust-size` failed: %s" % output) + return {} + + sections = {} + for sec_type in list(parsed.values()): + for name, size in list(sec_type.items()): + if not filter or name in filter: + sections[name] = size + + return sections + + def _get_binary_metrics(self): + """ + Provides metrics on interesting compenents of the built binaries. + Currently just the sizes of interesting sections. + """ + lib_interests = { + "XUL": ("libxul.so", "xul.dll", "XUL"), + "NSS": ("libnss3.so", "nss3.dll", "libnss3.dylib"), + "NSPR": ("libnspr4.so", "nspr4.dll", "libnspr4.dylib"), + "avcodec": ("libmozavcodec.so", "mozavcodec.dll", "libmozavcodec.dylib"), + "avutil": ("libmozavutil.so", "mozavutil.dll", "libmozavutil.dylib"), + } + section_interests = ( + ".text", + ".data", + ".rodata", + ".rdata", + ".cstring", + ".data.rel.ro", + ".bss", + ) + lib_details = [] + + dirs = self.query_abs_dirs() + dist_dir = os.path.join(dirs["abs_obj_dir"], "dist") + bin_dir = os.path.join(dist_dir, "bin") + + for lib_type, lib_names in list(lib_interests.items()): + for lib_name in lib_names: + lib = os.path.join(bin_dir, lib_name) + if os.path.exists(lib): + lib_size = 0 + section_details = self._get_sections(lib, section_interests) + section_measurements = [] + # Build up the subtests + + # Lump rodata sections together + # - Mach-O separates out read-only string data as .cstring + # - PE really uses .rdata, but XUL at least has a .rodata as well + for ro_alias in (".cstring", ".rdata"): + if ro_alias in section_details: + if ".rodata" in section_details: + section_details[".rodata"] += section_details[ro_alias] + else: + section_details[".rodata"] = section_details[ro_alias] + del section_details[ro_alias] + + for k, v in list(section_details.items()): + section_measurements.append({"name": k, "value": v}) + lib_size += v + lib_details.append( + { + "name": lib_type, + "size": lib_size, + "sections": section_measurements, + } + ) + + for lib_detail in lib_details: + yield { + "name": "%s section sizes" % lib_detail["name"], + "value": lib_detail["size"], + "shouldAlert": False, + "subtests": lib_detail["sections"], + } + + def _generate_build_stats(self): + """grab build stats following a compile. + + This action handles all statistics from a build: 'count_ctors' + and then posts to graph server the results. + We only post to graph server for non nightly build + """ + self.info("Collecting build metrics") + + if os.environ.get("USE_ARTIFACT"): + self.info("Skipping due to forced artifact build.") + return + + c = self.config + + # Report some important file sizes for display in treeherder + + perfherder_data = { + "framework": {"name": "build_metrics"}, + "suites": [], + } + + if not c.get("debug_build") and not c.get("disable_package_metrics"): + perfherder_data["suites"].extend(self._get_package_metrics()) + perfherder_data["suites"].extend(self._get_binary_metrics()) + + # Extract compiler warnings count. + warnings = self.get_output_from_command( + command=[sys.executable, "mach", "warnings-list"], + cwd=self.query_abs_dirs()["abs_src_dir"], + env=self.query_build_env(), + # No need to pollute the log. + silent=True, + # Fail fast. + halt_on_failure=True, + ) + + if warnings is not None: + perfherder_data["suites"].append( + { + "name": "compiler warnings", + "value": len(warnings.strip().splitlines()), + "alertThreshold": 100.0, + "subtests": [], + } + ) + + build_metrics = self._load_build_resources() + if build_metrics: + perfherder_data["suites"].append(build_metrics) + perfherder_data["suites"].extend(self._load_sccache_stats()) + + # Ensure all extra options for this configuration are present. + for opt in os.environ.get("PERFHERDER_EXTRA_OPTIONS", "").split(): + for suite in perfherder_data["suites"]: + if opt not in suite.get("extraOptions", []): + suite.setdefault("extraOptions", []).append(opt) + + if self.query_is_nightly(): + for suite in perfherder_data["suites"]: + suite.setdefault("extraOptions", []).insert(0, "nightly") + + if perfherder_data["suites"]: + self.info("PERFHERDER_DATA: %s" % json.dumps(perfherder_data)) + + def valgrind_test(self): + """Execute mach's valgrind-test for memory leaks""" + env = self.query_build_env() + env.update(self.query_mach_build_env()) + + return_code = self.run_command( + command=[sys.executable, "mach", "valgrind-test"], + cwd=self.query_abs_dirs()["abs_src_dir"], + env=env, + output_timeout=self.config.get("max_build_output_timeout", 60 * 40), + ) + if return_code: + self.return_code = self.worst_level( + EXIT_STATUS_DICT[TBPL_FAILURE], + self.return_code, + AUTOMATION_EXIT_CODES[::-1], + ) + self.fatal( + "'mach valgrind-test' did not run successfully. Please check " + "log for errors." + ) + + def _ensure_upload_path(self): + env = self.query_mach_build_env() + + # Some Taskcluster workers don't like it if an artifacts directory + # is defined but no artifacts are uploaded. Guard against this by always + # ensuring the artifacts directory exists. + if "UPLOAD_PATH" in env and not os.path.exists(env["UPLOAD_PATH"]): + self.mkdir_p(env["UPLOAD_PATH"]) + + def _post_fatal(self, message=None, exit_code=None): + if not self.return_code: # only overwrite return_code if it's 0 + self.error("setting return code to 2 because fatal was called") + self.return_code = 2 + + @PostScriptRun + def _summarize(self): + """If this is run in automation, ensure the return code is valid and + set it to one if it's not. Finally, log any summaries we collected + from the script run. + """ + if self.config.get("is_automation"): + # let's ignore all mention of tbpl status until this + # point so it will be easier to manage + if self.return_code not in AUTOMATION_EXIT_CODES: + self.error( + "Return code is set to: %s and is outside of " + "automation's known values. Setting to 2(failure). " + "Valid return codes %s" % (self.return_code, AUTOMATION_EXIT_CODES) + ) + self.return_code = 2 + for status, return_code in list(EXIT_STATUS_DICT.items()): + if return_code == self.return_code: + self.record_status(status, TBPL_STATUS_DICT[status]) + self.summary() + + @PostScriptRun + def _parse_build_tests_ccov(self): + if "MOZ_FETCHES_DIR" not in os.environ: + return + + dirs = self.query_abs_dirs() + topsrcdir = dirs["abs_src_dir"] + base_work_dir = dirs["base_work_dir"] + + env = self.query_build_env() + + grcov_path = os.path.join(os.environ["MOZ_FETCHES_DIR"], "grcov", "grcov") + if not os.path.isabs(grcov_path): + grcov_path = os.path.join(base_work_dir, grcov_path) + if self._is_windows(): + grcov_path += ".exe" + env["GRCOV_PATH"] = grcov_path + + cmd = self._query_mach() + [ + "python", + os.path.join("testing", "parse_build_tests_ccov.py"), + ] + self.run_command(command=cmd, cwd=topsrcdir, env=env, halt_on_failure=True) + + @PostScriptRun + def _relocate_artifacts(self): + """Move certain artifacts out of the default upload directory. + + These artifacts will be moved to a secondary directory called `cidata`. + Then they will be uploaded with different expiration values.""" + dirs = self.query_abs_dirs() + topsrcdir = dirs["abs_src_dir"] + base_work_dir = dirs["base_work_dir"] + + build_platform = os.environ.get("MOZ_ARTIFACT_PLATFORM") + if build_platform is not None: + build_platform = build_platform.lower() + else: + return + try: + upload_dir = os.environ["UPLOAD_DIR"] + except KeyError: + self.fatal("The env. var. UPLOAD_DIR is not set.") + + artifact_yml_path = os.path.join( + topsrcdir, "taskcluster/gecko_taskgraph/transforms/artifacts.yml" + ) + + upload_short_dir = os.path.join(base_work_dir, "cidata") + + # Choose artifacts based on build platform + if build_platform.startswith("win"): + main_platform = "win" + elif build_platform.startswith("linux"): + main_platform = "linux" + elif build_platform.startswith("mac"): + main_platform = "macos" + elif build_platform.startswith("android"): + if build_platform == "android-geckoview-docs": + return + main_platform = "android" + else: + err = "Build platform {} didn't start with 'mac', 'linux', 'win', or 'android'".format( + build_platform + ) + self.fatal(err) + try: + with open(artifact_yml_path) as artfile: + arts = [] + platforms = yaml.safe_load(artfile.read()) + for artifact in platforms[main_platform]: + arts.append(artifact) + except FileNotFoundError: + self.fatal("Could not read artifacts.yml; file not found. Exiting.") + except PermissionError: + self.fatal("Could not read artifacts.yml; permission error.") + except YAMLError as ye: + self.fatal(f"Failed to parse artifacts.yml with error:\n{ye}") + + try: + os.makedirs(upload_short_dir) + except FileExistsError: + pass + except PermissionError: + self.fatal(f'Failed to create dir. "{upload_short_dir}"; permission error.') + + for art in arts: + source_file = os.path.join(upload_dir, art) + if not os.path.exists(source_file): + self.info( + f"The artifact {source_file} is not present in this build. Skipping" + ) + continue + dest_file = os.path.join(upload_short_dir, art) + try: + os.rename(source_file, dest_file) + if os.path.exists(dest_file): + self.info( + f"Successfully moved artifact {source_file} to {dest_file}" + ) + else: + self.fatal( + f"Move of {source_file} to {dest_file} was not successful." + ) + except (PermissionError, FileNotFoundError) as err: + self.fatal( + f'Failed to move file "{art}" from {source_file} to {dest_file}:\n{err}' + ) + continue diff --git a/testing/mozharness/mozharness/mozilla/checksums.py b/testing/mozharness/mozharness/mozilla/checksums.py new file mode 100644 index 0000000000..e7071d506a --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/checksums.py @@ -0,0 +1,41 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import six + + +def parse_checksums_file(checksums): + """ + Parses checksums files that the build system generates and uploads: + https://hg.mozilla.org/mozilla-central/file/default/build/checksums.py + """ + fileInfo = {} + for line in checksums.splitlines(): + hash_, type_, size, file_ = line.split(None, 3) + type_ = six.ensure_str(type_) + file_ = six.ensure_str(file_) + size = int(size) + if size < 0: + raise ValueError("Found negative value (%d) for size." % size) + if file_ not in fileInfo: + fileInfo[file_] = {"hashes": {}} + # If the file already exists, make sure that the size matches the + # previous entry. + elif fileInfo[file_]["size"] != size: + raise ValueError( + "Found different sizes for same file %s (%s and %s)" + % (file_, fileInfo[file_]["size"], size) + ) + # Same goes for the hash. + elif ( + type_ in fileInfo[file_]["hashes"] + and fileInfo[file_]["hashes"][type_] != hash_ + ): + raise ValueError( + "Found different %s hashes for same file %s (%s and %s)" + % (type_, file_, fileInfo[file_]["hashes"][type_], hash_) + ) + fileInfo[file_]["size"] = size + fileInfo[file_]["hashes"][type_] = hash_ + return fileInfo diff --git a/testing/mozharness/mozharness/mozilla/firefox/__init__.py b/testing/mozharness/mozharness/mozilla/firefox/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/firefox/__init__.py diff --git a/testing/mozharness/mozharness/mozilla/firefox/autoconfig.py b/testing/mozharness/mozharness/mozilla/firefox/autoconfig.py new file mode 100644 index 0000000000..476277e661 --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/firefox/autoconfig.py @@ -0,0 +1,72 @@ +""" This module helps modifying Firefox with autoconfig files.""" +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import os + +from mozharness.base.script import platform_name + +AUTOCONFIG_TEXT = """// Any comment. You must start the file with a comment! +// This entry tells the browser to load a mozilla.cfg +pref("general.config.sandbox_enabled", false); +pref("general.config.filename", "mozilla.cfg"); +pref("general.config.obscure_value", 0); +""" + + +def write_autoconfig_files( + fx_install_dir, cfg_contents, autoconfig_contents=AUTOCONFIG_TEXT +): + """Generate autoconfig files to modify Firefox's set up + + Read documentation in here: + https://developer.mozilla.org/en-US/Firefox/Enterprise_deployment#Configuration + + fx_install_dir - path to Firefox installation + cfg_contents - .cfg file containing JavaScript changes for Firefox + autoconfig_contents - autoconfig.js content to refer to .cfg gile + """ + with open(_cfg_file_path(fx_install_dir), "w") as fd: + fd.write(cfg_contents) + with open(_autoconfig_path(fx_install_dir), "w") as fd: + fd.write(autoconfig_contents) + + +def read_autoconfig_file(fx_install_dir): + """Read autoconfig file that modifies Firefox startup + + fx_install_dir - path to Firefox installation + """ + with open(_cfg_file_path(fx_install_dir), "r") as fd: + return fd.read() + + +def _autoconfig_path(fx_install_dir): + platform = platform_name() + if platform in ("win32", "win64"): + return os.path.join(fx_install_dir, "defaults", "pref", "autoconfig.js") + elif platform in ("linux", "linux64"): + return os.path.join(fx_install_dir, "defaults/pref/autoconfig.js") + elif platform in ("macosx"): + return os.path.join( + fx_install_dir, "Contents/Resources/defaults/pref/autoconfig.js" + ) + else: + raise Exception("Invalid platform.") + + +def _cfg_file_path(fx_install_dir): + """ + Windows: defaults\pref + Mac: Firefox.app/Contents/Resources/defaults/pref + Linux: defaults/pref + """ + platform = platform_name() + if platform in ("win32", "win64"): + return os.path.join(fx_install_dir, "mozilla.cfg") + elif platform in ("linux", "linux64"): + return os.path.join(fx_install_dir, "mozilla.cfg") + elif platform in ("macosx"): + return os.path.join(fx_install_dir, "Contents/Resources/mozilla.cfg") + else: + raise Exception("Invalid platform.") diff --git a/testing/mozharness/mozharness/mozilla/l10n/__init__.py b/testing/mozharness/mozharness/mozilla/l10n/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/l10n/__init__.py diff --git a/testing/mozharness/mozharness/mozilla/l10n/locales.py b/testing/mozharness/mozharness/mozilla/l10n/locales.py new file mode 100755 index 0000000000..83fadd0133 --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/l10n/locales.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +"""Localization. +""" + +import os +import pprint + +from mozharness.base.config import parse_config_file + + +# LocalesMixin {{{1 +class LocalesMixin(object): + def __init__(self, **kwargs): + """Mixins generally don't have an __init__. + This breaks super().__init__() for children. + However, this is needed to override the query_abs_dirs() + """ + self.abs_dirs = None + self.locales = None + self.gecko_locale_revisions = None + self.l10n_revisions = {} + + def query_locales(self): + if self.locales is not None: + return self.locales + c = self.config + ignore_locales = c.get("ignore_locales", []) + additional_locales = c.get("additional_locales", []) + # List of locales can be set by using different methods in the + # following order: + # 1. "MOZ_LOCALES" env variable: a string of locale:revision separated + # by space + # 2. self.config["locales"] which can be either coming from the config + # or from --locale command line argument + # 3. using self.config["locales_file"] l10n changesets file + locales = None + + # Environment variable + if not locales and "MOZ_LOCALES" in os.environ: + self.debug("Using locales from environment: %s" % os.environ["MOZ_LOCALES"]) + locales = os.environ["MOZ_LOCALES"].split() + + # Command line or config + if not locales and c.get("locales", []): + locales = c["locales"] + self.debug("Using locales from config/CLI: %s" % ", ".join(locales)) + + # parse locale:revision if set + if locales: + for l in locales: + if ":" in l: + # revision specified in locale string + locale, revision = l.split(":", 1) + self.debug("Using %s:%s" % (locale, revision)) + self.l10n_revisions[locale] = revision + # clean up locale by removing revisions + locales = [l.split(":")[0] for l in locales] + + if not locales and "locales_file" in c: + abs_dirs = self.query_abs_dirs() + locales_file = os.path.join(abs_dirs["abs_src_dir"], c["locales_file"]) + locales = self.parse_locales_file(locales_file) + + if not locales: + self.fatal("No locales set!") + + for locale in ignore_locales: + if locale in locales: + self.debug("Ignoring locale %s." % locale) + locales.remove(locale) + if locale in self.l10n_revisions: + del self.l10n_revisions[locale] + + for locale in additional_locales: + if locale not in locales: + self.debug("Adding locale %s." % locale) + locales.append(locale) + + if not locales: + return None + self.locales = locales + return self.locales + + def list_locales(self): + """Stub action method.""" + self.info("Locale list: %s" % str(self.query_locales())) + + def parse_locales_file(self, locales_file): + locales = [] + c = self.config + self.info("Parsing locales file %s" % locales_file) + platform = c.get("locales_platform", None) + + if locales_file.endswith("json"): + locales_json = parse_config_file(locales_file) + for locale in sorted(locales_json.keys()): + if isinstance(locales_json[locale], dict): + if platform and platform not in locales_json[locale]["platforms"]: + continue + self.l10n_revisions[locale] = locales_json[locale]["revision"] + else: + # some other way of getting this? + self.l10n_revisions[locale] = "default" + locales.append(locale) + else: + locales = self.read_from_file(locales_file).split() + self.info("self.l10n_revisions: %s" % pprint.pformat(self.l10n_revisions)) + self.info("locales: %s" % locales) + return locales + + def query_abs_dirs(self): + if self.abs_dirs: + return self.abs_dirs + abs_dirs = super(LocalesMixin, self).query_abs_dirs() + c = self.config + dirs = {} + dirs["abs_work_dir"] = os.path.join(c["base_work_dir"], c["work_dir"]) + dirs["abs_l10n_dir"] = os.path.abspath( + os.path.join(abs_dirs["abs_src_dir"], "../l10n-central") + ) + dirs["abs_locales_src_dir"] = os.path.join( + abs_dirs["abs_src_dir"], + c["locales_dir"], + ) + + dirs["abs_obj_dir"] = os.path.join(dirs["abs_work_dir"], c["objdir"]) + dirs["abs_locales_dir"] = os.path.join(dirs["abs_obj_dir"], c["locales_dir"]) + + for key in list(dirs.keys()): + if key not in abs_dirs: + abs_dirs[key] = dirs[key] + self.abs_dirs = abs_dirs + return self.abs_dirs + + # This requires self to inherit a VCSMixin. + def pull_locale_source(self, hg_l10n_base=None, parent_dir=None, vcs="hg"): + c = self.config + if not hg_l10n_base: + hg_l10n_base = c["hg_l10n_base"] + if parent_dir is None: + parent_dir = self.query_abs_dirs()["abs_l10n_dir"] + self.mkdir_p(parent_dir) + # This block is to allow for pulling buildbot-configs in Fennec + # release builds, since we don't pull it in MBF anymore. + if c.get("l10n_repos"): + repos = c.get("l10n_repos") + self.vcs_checkout_repos(repos, tag_override=c.get("tag_override")) + # Pull locales + locales = self.query_locales() + locale_repos = [] + for locale in locales: + tag = c.get("hg_l10n_tag", "default") + if self.l10n_revisions.get(locale): + tag = self.l10n_revisions[locale] + locale_repos.append( + {"repo": "%s/%s" % (hg_l10n_base, locale), "branch": tag, "vcs": vcs} + ) + revs = self.vcs_checkout_repos( + repo_list=locale_repos, + parent_dir=parent_dir, + tag_override=c.get("tag_override"), + ) + self.gecko_locale_revisions = revs + + +# __main__ {{{1 + +if __name__ == "__main__": + pass diff --git a/testing/mozharness/mozharness/mozilla/l10n/multi_locale_build.py b/testing/mozharness/mozharness/mozilla/l10n/multi_locale_build.py new file mode 100755 index 0000000000..6b1f8c4782 --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/l10n/multi_locale_build.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +"""multi_locale_build.py + +This should be a mostly generic multilocale build script. +""" + +import os +import sys + +from mozharness.base.errors import MakefileErrorList +from mozharness.base.vcs.vcsbase import MercurialScript +from mozharness.mozilla.l10n.locales import LocalesMixin + +sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0]))) + + +# MultiLocaleBuild {{{1 +class MultiLocaleBuild(LocalesMixin, MercurialScript): + """This class targets Fennec multilocale builds. + We were considering this for potential Firefox desktop multilocale. + Now that we have a different approach for B2G multilocale, + it's most likely misnamed.""" + + config_options = [ + [ + ["--locale"], + { + "action": "extend", + "dest": "locales", + "type": "string", + "help": "Specify the locale(s) to repack", + }, + ], + [ + ["--objdir"], + { + "action": "store", + "dest": "objdir", + "type": "string", + "default": "objdir", + "help": "Specify the objdir", + }, + ], + [ + ["--l10n-base"], + { + "action": "store", + "dest": "hg_l10n_base", + "type": "string", + "help": "Specify the L10n repo base directory", + }, + ], + [ + ["--l10n-tag"], + { + "action": "store", + "dest": "hg_l10n_tag", + "type": "string", + "help": "Specify the L10n tag", + }, + ], + [ + ["--tag-override"], + { + "action": "store", + "dest": "tag_override", + "type": "string", + "help": "Override the tags set for all repos", + }, + ], + ] + + def __init__(self, require_config_file=True): + LocalesMixin.__init__(self) + MercurialScript.__init__( + self, + config_options=self.config_options, + all_actions=["pull-locale-source", "package-multi", "summary"], + require_config_file=require_config_file, + ) + + # pull_locale_source() defined in LocalesMixin. + + def _run_mach_command(self, args): + dirs = self.query_abs_dirs() + + mach = [sys.executable, "mach"] + + return_code = self.run_command( + command=mach + ["--log-no-times"] + args, + cwd=dirs["abs_src_dir"], + ) + + if return_code: + self.fatal( + "'mach %s' did not run successfully. Please check " + "log for errors." % " ".join(args) + ) + + def package_multi(self): + dirs = self.query_abs_dirs() + objdir = dirs["abs_obj_dir"] + + # This will error on non-0 exit code. + locales = list(sorted(self.query_locales())) + self._run_mach_command(["package-multi-locale", "--locales"] + locales) + + command = "make package-tests AB_CD=multi" + self.run_command( + command, cwd=objdir, error_list=MakefileErrorList, halt_on_failure=True + ) + # TODO deal with buildsymbols + + +# __main__ {{{1 +if __name__ == "__main__": + pass diff --git a/testing/mozharness/mozharness/mozilla/merkle.py b/testing/mozharness/mozharness/mozilla/merkle.py new file mode 100644 index 0000000000..dba780b73a --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/merkle.py @@ -0,0 +1,190 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import struct + + +def _round2(n): + k = 1 + while k < n: + k <<= 1 + return k >> 1 + + +def _leaf_hash(hash_fn, leaf): + return hash_fn(b"\x00" + leaf).digest() + + +def _pair_hash(hash_fn, left, right): + return hash_fn(b"\x01" + left + right).digest() + + +class InclusionProof: + """ + Represents a Merkle inclusion proof for purposes of serialization, + deserialization, and verification of the proof. The format for inclusion + proofs in RFC 6962-bis is as follows: + + opaque LogID<2..127>; + opaque NodeHash<32..2^8-1>; + + struct { + LogID log_id; + uint64 tree_size; + uint64 leaf_index; + NodeHash inclusion_path<1..2^16-1>; + } InclusionProofDataV2; + + In other words: + - 1 + N octets of log_id (currently zero) + - 8 octets of tree_size = self.n + - 8 octets of leaf_index = m + - 2 octets of path length, followed by + * 1 + N octets of NodeHash + """ + + # Pre-generated 'log ID'. Not used by Firefox; it is only needed because + # there's a slot in the RFC 6962-bis format that requires a value at least + # two bytes long (plus a length byte). + LOG_ID = b"\x02\x00\x00" + + def __init__(self, tree_size, leaf_index, path_elements): + self.tree_size = tree_size + self.leaf_index = leaf_index + self.path_elements = path_elements + + @staticmethod + def from_rfc6962_bis(serialized): + start = 0 + read = 1 + if len(serialized) < start + read: + raise Exception("Inclusion proof too short for log ID header") + (log_id_len,) = struct.unpack("B", serialized[start : start + read]) + start += read + start += log_id_len # Ignore the log ID itself + + read = 8 + 8 + 2 + if len(serialized) < start + read: + raise Exception("Inclusion proof too short for middle section") + tree_size, leaf_index, path_len = struct.unpack( + "!QQH", serialized[start : start + read] + ) + start += read + + path_elements = [] + end = 1 + log_id_len + 8 + 8 + 2 + path_len + while start < end: + read = 1 + if len(serialized) < start + read: + raise Exception("Inclusion proof too short for middle section") + (elem_len,) = struct.unpack("!B", serialized[start : start + read]) + start += read + + read = elem_len + if len(serialized) < start + read: + raise Exception("Inclusion proof too short for middle section") + if end < start + read: + raise Exception("Inclusion proof element exceeds declared length") + path_elements.append(serialized[start : start + read]) + start += read + + return InclusionProof(tree_size, leaf_index, path_elements) + + def to_rfc6962_bis(self): + inclusion_path = b"" + for step in self.path_elements: + step_len = struct.pack("B", len(step)) + inclusion_path += step_len + step + + middle = struct.pack( + "!QQH", self.tree_size, self.leaf_index, len(inclusion_path) + ) + return self.LOG_ID + middle + inclusion_path + + def _expected_head(self, hash_fn, leaf, leaf_index, tree_size): + node = _leaf_hash(hash_fn, leaf) + + # Compute indicators of which direction the pair hashes should be done. + # Derived from the PATH logic in draft-ietf-trans-rfc6962-bis + lr = [] + while tree_size > 1: + k = _round2(tree_size) + left = leaf_index < k + lr = [left] + lr + + if left: + tree_size = k + else: + tree_size = tree_size - k + leaf_index = leaf_index - k + + assert len(lr) == len(self.path_elements) + for i, elem in enumerate(self.path_elements): + if lr[i]: + node = _pair_hash(hash_fn, node, elem) + else: + node = _pair_hash(hash_fn, elem, node) + + return node + + def verify(self, hash_fn, leaf, leaf_index, tree_size, tree_head): + return self._expected_head(hash_fn, leaf, leaf_index, tree_size) == tree_head + + +class MerkleTree: + """ + Implements a Merkle tree on a set of data items following the + structure defined in RFC 6962-bis. This allows us to create a + single hash value that summarizes the data (the 'head'), and an + 'inclusion proof' for each element that connects it to the head. + + https://tools.ietf.org/html/draft-ietf-trans-rfc6962-bis-24 + """ + + def __init__(self, hash_fn, data): + self.n = len(data) + self.hash_fn = hash_fn + + # We cache intermediate node values, as a dictionary of dictionaries, + # where the node representing data elements data[m:n] is represented by + # nodes[m][n]. This corresponds to the 'D[m:n]' notation in RFC + # 6962-bis. In particular, the leaves are stored in nodes[i][i+1] and + # the head is nodes[0][n]. + self.nodes = {} + for i in range(self.n): + self.nodes[i, i + 1] = _leaf_hash(self.hash_fn, data[i]) + + def _node(self, start, end): + if (start, end) in self.nodes: + return self.nodes[start, end] + + k = _round2(end - start) + left = self._node(start, start + k) + right = self._node(start + k, end) + node = _pair_hash(self.hash_fn, left, right) + + self.nodes[start, end] = node + return node + + def head(self): + return self._node(0, self.n) + + def _relative_proof(self, target, start, end): + n = end - start + k = _round2(n) + + if n == 1: + return [] + elif target - start < k: + return self._relative_proof(target, start, start + k) + [ + self._node(start + k, end) + ] + elif target - start >= k: + return self._relative_proof(target, start + k, end) + [ + self._node(start, start + k) + ] + + def inclusion_proof(self, leaf_index): + path_elements = self._relative_proof(leaf_index, 0, self.n) + return InclusionProof(self.n, leaf_index, path_elements) diff --git a/testing/mozharness/mozharness/mozilla/mozbase.py b/testing/mozharness/mozharness/mozilla/mozbase.py new file mode 100644 index 0000000000..552ffd850c --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/mozbase.py @@ -0,0 +1,32 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +from mozharness.base.script import PreScriptAction + + +class MozbaseMixin(object): + """Automatically set virtualenv requirements to use mozbase + from test package. + """ + + def __init__(self, *args, **kwargs): + super(MozbaseMixin, self).__init__(*args, **kwargs) + + @PreScriptAction("create-virtualenv") + def _install_mozbase(self, action): + dirs = self.query_abs_dirs() + + requirements = os.path.join( + dirs["abs_test_install_dir"], + "config", + self.config.get("mozbase_requirements", "mozbase_requirements.txt"), + ) + if not os.path.isfile(requirements): + self.fatal( + "Could not find mozbase requirements file: {}".format(requirements) + ) + + self.register_virtualenv_module(requirements=[requirements], two_pass=True) diff --git a/testing/mozharness/mozharness/mozilla/repo_manipulation.py b/testing/mozharness/mozharness/mozilla/repo_manipulation.py new file mode 100644 index 0000000000..3a5712fadb --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/repo_manipulation.py @@ -0,0 +1,222 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import six + +# pylint --py3k: W1648 +if six.PY2: + from ConfigParser import ConfigParser +else: + from configparser import ConfigParser + +import json +import os + +from mozharness.base.errors import HgErrorList +from mozharness.base.log import FATAL, INFO +from mozharness.base.vcs.mercurial import MercurialVCS + + +class MercurialRepoManipulationMixin(object): + def get_version(self, repo_root, version_file="browser/config/version.txt"): + version_path = os.path.join(repo_root, version_file) + contents = self.read_from_file(version_path, error_level=FATAL) + lines = [l for l in contents.splitlines() if l and not l.startswith("#")] + return lines[-1].split(".") + + def replace(self, file_name, from_, to_): + """Replace text in a file.""" + text = self.read_from_file(file_name, error_level=FATAL) + new_text = text.replace(from_, to_) + if text == new_text: + self.fatal("Cannot replace '%s' to '%s' in '%s'" % (from_, to_, file_name)) + self.write_to_file(file_name, new_text, error_level=FATAL) + + def query_hg_revision(self, path): + """Avoid making 'pull' a required action every run, by being able + to fall back to figuring out the revision from the cloned repo + """ + m = MercurialVCS(log_obj=self.log_obj, config=self.config) + revision = m.get_revision_from_path(path) + return revision + + def hg_commit(self, cwd, message, user=None, ignore_no_changes=False): + """Commit changes to hg.""" + cmd = self.query_exe("hg", return_type="list") + ["commit", "-m", message] + if user: + cmd.extend(["-u", user]) + success_codes = [0] + if ignore_no_changes: + success_codes.append(1) + self.run_command( + cmd, + cwd=cwd, + error_list=HgErrorList, + halt_on_failure=True, + success_codes=success_codes, + ) + return self.query_hg_revision(cwd) + + def clean_repos(self): + """We may end up with contaminated local repos at some point, but + we don't want to have to clobber and reclone from scratch every + time. + + This is an attempt to clean up the local repos without needing a + clobber. + """ + dirs = self.query_abs_dirs() + hg = self.query_exe("hg", return_type="list") + hg_repos = self.query_repos() + hg_strip_error_list = [ + { + "substr": r"""abort: empty revision set""", + "level": INFO, + "explanation": "Nothing to clean up; we're good!", + } + ] + HgErrorList + for repo_config in hg_repos: + repo_name = repo_config["dest"] + repo_path = os.path.join(dirs["abs_work_dir"], repo_name) + if os.path.exists(repo_path): + # hg up -C to discard uncommitted changes + self.run_command( + hg + ["up", "-C", "-r", repo_config["branch"]], + cwd=repo_path, + error_list=HgErrorList, + halt_on_failure=True, + ) + # discard unpushed commits + status = self.retry( + self.run_command, + args=( + hg + + [ + "--config", + "extensions.mq=", + "strip", + "--no-backup", + "outgoing()", + ], + ), + kwargs={ + "cwd": repo_path, + "error_list": hg_strip_error_list, + "return_type": "num_errors", + "success_codes": (0, 255), + }, + ) + if status not in [0, 255]: + self.fatal("Issues stripping outgoing revisions!") + # 2nd hg up -C to make sure we're not on a stranded head + # which can happen when reverting debugsetparents + self.run_command( + hg + ["up", "-C", "-r", repo_config["branch"]], + cwd=repo_path, + error_list=HgErrorList, + halt_on_failure=True, + ) + + def commit_changes(self): + """Do the commit.""" + hg = self.query_exe("hg", return_type="list") + for cwd in self.query_commit_dirs(): + self.run_command(hg + ["diff"], cwd=cwd) + self.hg_commit( + cwd, + user=self.config["hg_user"], + message=self.query_commit_message(), + ignore_no_changes=self.config.get("ignore_no_changes", False), + ) + self.info( + "Now verify |hg out| and |hg out --patch| if you're paranoid, and --push" + ) + + def hg_tag( + self, + cwd, + tags, + user=None, + message=None, + revision=None, + force=None, + halt_on_failure=True, + ): + if isinstance(tags, six.string_types): + tags = [tags] + cmd = self.query_exe("hg", return_type="list") + ["tag"] + if not message: + message = "No bug - Tagging %s" % os.path.basename(cwd) + if revision: + message = "%s %s" % (message, revision) + message = "%s with %s" % (message, ", ".join(tags)) + message += " a=release DONTBUILD CLOSED TREE" + self.info(message) + cmd.extend(["-m", message]) + if user: + cmd.extend(["-u", user]) + if revision: + cmd.extend(["-r", revision]) + if force: + cmd.append("-f") + cmd.extend(tags) + return self.run_command( + cmd, cwd=cwd, halt_on_failure=halt_on_failure, error_list=HgErrorList + ) + + def query_existing_tags(self, cwd, halt_on_failure=True): + cmd = self.query_exe("hg", return_type="list") + ["tags"] + existing_tags = {} + output = self.get_output_from_command( + cmd, cwd=cwd, halt_on_failure=halt_on_failure + ) + for line in output.splitlines(): + parts = line.split(" ") + if len(parts) > 1: + # existing_tags = {TAG: REVISION, ...} + existing_tags[parts[0]] = parts[-1].split(":")[-1] + self.info( + "existing_tags:\n{}".format( + json.dumps(existing_tags, sort_keys=True, indent=4) + ) + ) + return existing_tags + + def push(self): + """""" + error_message = """Push failed! If there was a push race, try rerunning +the script (--clean-repos --pull --migrate). The second run will be faster.""" + hg = self.query_exe("hg", return_type="list") + for cwd in self.query_push_dirs(): + if not cwd: + self.warning("Skipping %s" % cwd) + continue + push_cmd = hg + ["push"] + self.query_push_args(cwd) + if self.config.get("push_dest"): + push_cmd.append(self.config["push_dest"]) + status = self.run_command( + push_cmd, + cwd=cwd, + error_list=HgErrorList, + success_codes=[0, 1], + ) + if status == 1: + self.warning("No changes for %s!" % cwd) + elif status: + self.fatal(error_message) + + def edit_repo_hg_rc(self, cwd, section, key, value): + hg_rc = self.read_repo_hg_rc(cwd) + hg_rc.set(section, key, value) + + with open(self._get_hg_rc_path(cwd), "wb") as f: + hg_rc.write(f) + + def read_repo_hg_rc(self, cwd): + hg_rc = ConfigParser() + hg_rc.read(self._get_hg_rc_path(cwd)) + return hg_rc + + def _get_hg_rc_path(self, cwd): + return os.path.join(cwd, ".hg", "hgrc") diff --git a/testing/mozharness/mozharness/mozilla/secrets.py b/testing/mozharness/mozharness/mozilla/secrets.py new file mode 100644 index 0000000000..7ec4c8a2e9 --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/secrets.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +"""Support for fetching secrets from the secrets API +""" + +import json +import os + +import six +from six.moves import urllib + + +class SecretsMixin(object): + def _fetch_secret(self, secret_name): + self.info("fetching secret {} from API".format(secret_name)) + # fetch from TASKCLUSTER_PROXY_URL, which points to the taskcluster proxy + # within a taskcluster task. Outside of that environment, do not + # use this action. + proxy = os.environ.get("TASKCLUSTER_PROXY_URL", "http://taskcluster") + proxy = proxy.rstrip("/") + url = proxy + "/secrets/v1/secret/" + secret_name + res = urllib.request.urlopen(url) + if res.getcode() != 200: + self.fatal("Error fetching from secrets API:" + res.read()) + + return json.loads(six.ensure_str(res.read()))["secret"]["content"] + + def get_secrets(self): + """ + Get the secrets specified by the `secret_files` configuration. This is + a list of dictionaries, one for each secret. The `secret_name` key + names the key in the TaskCluster secrets API to fetch (see + http://docs.taskcluster.net/services/secrets/). It can contain + %-substitutions based on the `subst` dictionary below. + + Since secrets must be JSON objects, the `content` property of the + secret is used as the value to be written to disk. + + The `filename` key in the dictionary gives the filename to which the + secret should be written. + + The optional `min_scm_level` key gives a minimum SCM level at which + this secret is required. For lower levels, the value of the 'default` + key or the contents of the file specified by `default-file` is used, or + no secret is written. + + The optional 'mode' key allows a mode change (chmod) after the file is written + """ + dirs = self.query_abs_dirs() + secret_files = self.config.get("secret_files", []) + + scm_level = int(os.environ.get("MOZ_SCM_LEVEL", "1")) + subst = { + "scm-level": scm_level, + } + + for sf in secret_files: + filename = os.path.abspath(sf["filename"]) + secret_name = sf["secret_name"] % subst + min_scm_level = sf.get("min_scm_level", 0) + if scm_level < min_scm_level: + if "default" in sf: + self.info("Using default value for " + filename) + secret = sf["default"] + elif "default-file" in sf: + default_path = sf["default-file"].format(**dirs) + with open(default_path, "r") as f: + secret = f.read() + else: + self.info("No default for secret; not writing " + filename) + continue + else: + secret = self._fetch_secret(secret_name) + + open(filename, "w").write(secret) + + if sf.get("mode"): + os.chmod(filename, sf["mode"]) diff --git a/testing/mozharness/mozharness/mozilla/structuredlog.py b/testing/mozharness/mozharness/mozilla/structuredlog.py new file mode 100644 index 0000000000..e77722854a --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/structuredlog.py @@ -0,0 +1,306 @@ +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +import json +from collections import defaultdict, namedtuple + +from mozharness.base import log +from mozharness.base.log import ERROR, INFO, WARNING, OutputParser +from mozharness.mozilla.automation import ( + TBPL_FAILURE, + TBPL_RETRY, + TBPL_SUCCESS, + TBPL_WARNING, + TBPL_WORST_LEVEL_TUPLE, +) +from mozharness.mozilla.testing.errors import TinderBoxPrintRe +from mozharness.mozilla.testing.unittest import tbox_print_summary + + +class StructuredOutputParser(OutputParser): + # The script class using this must inherit the MozbaseMixin to ensure + # that mozlog is available. + def __init__(self, **kwargs): + """Object that tracks the overall status of the test run""" + # The 'strict' argument dictates whether the presence of output + # from the harness process other than line-delimited json indicates + # failure. If it does not, the errors_list parameter may be used + # to detect additional failure output from the harness process. + if "strict" in kwargs: + self.strict = kwargs.pop("strict") + else: + self.strict = True + + self.suite_category = kwargs.pop("suite_category", None) + + tbpl_compact = kwargs.pop("log_compact", False) + super(StructuredOutputParser, self).__init__(**kwargs) + self.allow_crashes = kwargs.pop("allow_crashes", False) + + mozlog = self._get_mozlog_module() + self.formatter = mozlog.formatters.TbplFormatter(compact=tbpl_compact) + self.handler = mozlog.handlers.StatusHandler() + self.log_actions = mozlog.structuredlog.log_actions() + + self.worst_log_level = INFO + self.tbpl_status = TBPL_SUCCESS + self.harness_retry_re = TinderBoxPrintRe["harness_error"]["retry_regex"] + self.prev_was_unstructured = False + + def _get_mozlog_module(self): + try: + import mozlog + except ImportError: + self.fatal( + "A script class using structured logging must inherit " + "from the MozbaseMixin to ensure that mozlog is available." + ) + return mozlog + + def _handle_unstructured_output(self, line, log_output=True): + self.log_output = log_output + return super(StructuredOutputParser, self).parse_single_line(line) + + def parse_single_line(self, line): + """Parses a line of log output from the child process and passes + it to mozlog to update the overall status of the run. + Re-emits the logged line in human-readable format. + """ + level = INFO + tbpl_level = TBPL_SUCCESS + + data = None + try: + candidate_data = json.loads(line) + if ( + isinstance(candidate_data, dict) + and "action" in candidate_data + and candidate_data["action"] in self.log_actions + ): + data = candidate_data + except ValueError: + pass + + if data is None: + if self.strict: + if not self.prev_was_unstructured: + self.info( + "Test harness output was not a valid structured log message" + ) + self.info(line) + else: + self.info(line) + self.prev_was_unstructured = True + else: + self._handle_unstructured_output(line) + return + + self.prev_was_unstructured = False + + self.handler(data) + + action = data["action"] + if action in ("log", "process_output"): + if action == "log": + message = data["message"] + level = getattr(log, data["level"].upper()) + else: + message = data["data"] + + # Run log and process_output actions through the error lists, but make sure + # the super parser doesn't print them to stdout (they should go through the + # log formatter). + error_level = self._handle_unstructured_output(message, log_output=False) + if error_level is not None: + level = self.worst_level(error_level, level) + + if self.harness_retry_re.search(message): + self.update_levels(TBPL_RETRY, log.CRITICAL) + tbpl_level = TBPL_RETRY + level = log.CRITICAL + + log_data = self.formatter(data) + if log_data is not None: + self.log(log_data, level=level) + self.update_levels(tbpl_level, level) + + def _subtract_tuples(self, old, new): + items = set(list(old.keys()) + list(new.keys())) + merged = defaultdict(int) + for item in items: + merged[item] = new.get(item, 0) - old.get(item, 0) + if merged[item] <= 0: + del merged[item] + return merged + + def evaluate_parser(self, return_code, success_codes=None, previous_summary=None): + success_codes = success_codes or [0] + summary = self.handler.summarize() + + """ + We can run evaluate_parser multiple times, it will duplicate failures + and status which can mean that future tests will fail if a previous test fails. + When we have a previous summary, we want to do 2 things: + 1) Remove previous data from the new summary to only look at new data + 2) Build a joined summary to include the previous + new data + """ + RunSummary = namedtuple( + "RunSummary", + ( + "unexpected_statuses", + "expected_statuses", + "known_intermittent_statuses", + "log_level_counts", + "action_counts", + ), + ) + if previous_summary == {}: + previous_summary = RunSummary( + defaultdict(int), + defaultdict(int), + defaultdict(int), + defaultdict(int), + defaultdict(int), + ) + if previous_summary: + # Always preserve retry status: if any failure triggers retry, the script + # must exit with TBPL_RETRY to trigger task retry. + if self.tbpl_status != TBPL_RETRY: + self.tbpl_status = TBPL_SUCCESS + joined_summary = summary + + # Remove previously known status messages + if "ERROR" in summary.log_level_counts: + summary.log_level_counts["ERROR"] -= self.handler.no_tests_run_count + + summary = RunSummary( + self._subtract_tuples( + previous_summary.unexpected_statuses, summary.unexpected_statuses + ), + self._subtract_tuples( + previous_summary.expected_statuses, summary.expected_statuses + ), + self._subtract_tuples( + previous_summary.known_intermittent_statuses, + summary.known_intermittent_statuses, + ), + self._subtract_tuples( + previous_summary.log_level_counts, summary.log_level_counts + ), + summary.action_counts, + ) + + # If we have previous data to ignore, + # cache it so we don't parse the log multiple times + self.summary = summary + else: + joined_summary = summary + + fail_pair = TBPL_WARNING, WARNING + error_pair = TBPL_FAILURE, ERROR + + # These are warning/orange statuses. + failure_conditions = [ + (sum(summary.unexpected_statuses.values()), 0, "statuses", False), + ( + summary.action_counts.get("crash", 0), + summary.expected_statuses.get("CRASH", 0), + "crashes", + self.allow_crashes, + ), + ( + summary.action_counts.get("valgrind_error", 0), + 0, + "valgrind errors", + False, + ), + ] + for value, limit, type_name, allow in failure_conditions: + if value > limit: + msg = "%d unexpected %s" % (value, type_name) + if limit != 0: + msg += " expected at most %d" % (limit) + if not allow: + self.update_levels(*fail_pair) + msg = "Got " + msg + # Force level to be WARNING as message is not necessary in Treeherder + self.warning(msg) + else: + msg = "Ignored " + msg + self.warning(msg) + + # These are error/red statuses. A message is output here every time something + # wouldn't otherwise be highlighted in the UI. + required_actions = { + "suite_end": "No suite end message was emitted by this harness.", + "test_end": "No checks run.", + } + for action, diagnostic_message in required_actions.items(): + if action not in summary.action_counts: + self.log(diagnostic_message, ERROR) + self.update_levels(*error_pair) + + failure_log_levels = ["ERROR", "CRITICAL"] + for level in failure_log_levels: + if level in summary.log_level_counts: + self.update_levels(*error_pair) + + # If a superclass was used to detect errors with a regex based output parser, + # this will be reflected in the status here. + if self.num_errors: + self.update_levels(*error_pair) + + # Harnesses typically return non-zero on test failure, so don't promote + # to error if we already have a failing status. + if return_code not in success_codes and self.tbpl_status == TBPL_SUCCESS: + self.update_levels(*error_pair) + + return self.tbpl_status, self.worst_log_level, joined_summary + + def update_levels(self, tbpl_level, log_level): + self.worst_log_level = self.worst_level(log_level, self.worst_log_level) + self.tbpl_status = self.worst_level( + tbpl_level, self.tbpl_status, levels=TBPL_WORST_LEVEL_TUPLE + ) + + def print_summary(self, suite_name): + # Summary text provided for compatibility. Counts are currently + # in the format <pass count>/<fail count>/<todo count>, + # <expected count>/<unexpected count>/<expected fail count> will yield the + # expected info from a structured log (fail count from the prior implementation + # includes unexpected passes from "todo" assertions). + try: + summary = self.summary + except AttributeError: + summary = self.handler.summarize() + + unexpected_count = sum(summary.unexpected_statuses.values()) + expected_count = sum(summary.expected_statuses.values()) + expected_failures = summary.expected_statuses.get("FAIL", 0) + + if unexpected_count: + fail_text = '<em class="testfail">%s</em>' % unexpected_count + else: + fail_text = "0" + + text_summary = "%s/%s/%s" % (expected_count, fail_text, expected_failures) + self.info("TinderboxPrint: %s<br/>%s\n" % (suite_name, text_summary)) + + def append_tinderboxprint_line(self, suite_name): + try: + summary = self.summary + except AttributeError: + summary = self.handler.summarize() + + unexpected_count = sum(summary.unexpected_statuses.values()) + expected_count = sum(summary.expected_statuses.values()) + expected_failures = summary.expected_statuses.get("FAIL", 0) + crashed = 0 + if "crash" in summary.action_counts: + crashed = summary.action_counts["crash"] + text_summary = tbox_print_summary( + expected_count, unexpected_count, expected_failures, crashed > 0, False + ) + self.info("TinderboxPrint: %s<br/>%s\n" % (suite_name, text_summary)) diff --git a/testing/mozharness/mozharness/mozilla/testing/__init__.py b/testing/mozharness/mozharness/mozilla/testing/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/testing/__init__.py diff --git a/testing/mozharness/mozharness/mozilla/testing/android.py b/testing/mozharness/mozharness/mozilla/testing/android.py new file mode 100644 index 0000000000..7e17707552 --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/testing/android.py @@ -0,0 +1,725 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** + +import datetime +import functools +import glob +import os +import posixpath +import re +import signal +import subprocess +import tempfile +import time +from threading import Timer + +import six + +from mozharness.base.script import PostScriptAction, PreScriptAction +from mozharness.mozilla.automation import EXIT_STATUS_DICT, TBPL_RETRY + + +def ensure_dir(dir): + """Ensures the given directory exists""" + if dir and not os.path.exists(dir): + try: + os.makedirs(dir) + except OSError as error: + if error.errno != errno.EEXIST: + raise + + +class AndroidMixin(object): + """ + Mixin class used by Android test scripts. + """ + + def __init__(self, **kwargs): + self._adb_path = None + self._device = None + self.app_name = None + self.device_name = os.environ.get("DEVICE_NAME", None) + self.device_serial = os.environ.get("DEVICE_SERIAL", None) + self.device_ip = os.environ.get("DEVICE_IP", None) + self.logcat_proc = None + self.logcat_file = None + self.use_gles3 = False + self.use_root = True + self.xre_path = None + super(AndroidMixin, self).__init__(**kwargs) + + @property + def adb_path(self): + """Get the path to the adb executable.""" + self.activate_virtualenv() + if not self._adb_path: + self._adb_path = self.query_exe("adb") + return self._adb_path + + @property + def device(self): + if not self._device: + # We must access the adb_path property to activate the + # virtualenv before importing mozdevice in order to + # import the mozdevice installed into the virtualenv and + # not any system-wide installation of mozdevice. + adb = self.adb_path + import mozdevice + + self._device = mozdevice.ADBDeviceFactory( + adb=adb, device=self.device_serial, use_root=self.use_root + ) + return self._device + + @property + def is_android(self): + c = self.config + installer_url = c.get("installer_url", None) + return ( + self.device_serial is not None + or self.is_emulator + or ( + installer_url is not None + and (installer_url.endswith(".apk") or installer_url.endswith(".aab")) + ) + ) + + @property + def is_emulator(self): + c = self.config + return True if c.get("emulator_avd_name") else False + + def _get_repo_url(self, path): + """ + Return a url for a file (typically a tooltool manifest) in this hg repo + and using this revision (or mozilla-central/default if repo/rev cannot + be determined). + + :param path specifies the directory path to the file of interest. + """ + if "GECKO_HEAD_REPOSITORY" in os.environ and "GECKO_HEAD_REV" in os.environ: + # probably taskcluster + repo = os.environ["GECKO_HEAD_REPOSITORY"] + revision = os.environ["GECKO_HEAD_REV"] + else: + # something unexpected! + repo = "https://hg.mozilla.org/mozilla-central" + revision = "default" + self.warning( + "Unable to find repo/revision for manifest; " + "using mozilla-central/default" + ) + url = "%s/raw-file/%s/%s" % (repo, revision, path) + return url + + def _tooltool_fetch(self, url, dir): + c = self.config + manifest_path = self.download_file( + url, file_name="releng.manifest", parent_dir=dir + ) + if not os.path.exists(manifest_path): + self.fatal( + "Could not retrieve manifest needed to retrieve " + "artifacts from %s" % manifest_path + ) + # from TooltoolMixin, included in TestingMixin + self.tooltool_fetch( + manifest_path, output_dir=dir, cache=c.get("tooltool_cache", None) + ) + + def _launch_emulator(self): + env = self.query_env() + + # Write a default ddms.cfg to avoid unwanted prompts + avd_home_dir = self.abs_dirs["abs_avds_dir"] + DDMS_FILE = os.path.join(avd_home_dir, "ddms.cfg") + with open(DDMS_FILE, "w") as f: + f.write("pingOptIn=false\npingId=0\n") + self.info("wrote dummy %s" % DDMS_FILE) + + # Delete emulator auth file, so it doesn't prompt + AUTH_FILE = os.path.join( + os.path.expanduser("~"), ".emulator_console_auth_token" + ) + if os.path.exists(AUTH_FILE): + try: + os.remove(AUTH_FILE) + self.info("deleted %s" % AUTH_FILE) + except Exception: + self.warning("failed to remove %s" % AUTH_FILE) + + env["ANDROID_EMULATOR_HOME"] = avd_home_dir + avd_path = os.path.join(avd_home_dir, "avd") + if os.path.exists(avd_path): + env["ANDROID_AVD_HOME"] = avd_path + self.info("Found avds at %s" % avd_path) + else: + self.warning("AVDs missing? Not found at %s" % avd_path) + + if "deprecated_sdk_path" in self.config: + sdk_path = os.path.abspath(os.path.join(avd_home_dir, "..")) + else: + sdk_path = self.abs_dirs["abs_sdk_dir"] + if os.path.exists(sdk_path): + env["ANDROID_SDK_HOME"] = sdk_path + env["ANDROID_SDK_ROOT"] = sdk_path + self.info("Found sdk at %s" % sdk_path) + else: + self.warning("Android sdk missing? Not found at %s" % sdk_path) + + avd_config_path = os.path.join( + avd_path, "%s.ini" % self.config["emulator_avd_name"] + ) + avd_folder = os.path.join(avd_path, "%s.avd" % self.config["emulator_avd_name"]) + if os.path.isfile(avd_config_path): + # The ini file points to the absolute path to the emulator folder, + # which might be different, so we need to update it. + old_config = "" + with open(avd_config_path, "r") as config_file: + old_config = config_file.readlines() + self.info("Old Config: %s" % old_config) + with open(avd_config_path, "w") as config_file: + for line in old_config: + if line.startswith("path="): + config_file.write("path=%s\n" % avd_folder) + self.info("Updating path from: %s" % line) + else: + config_file.write("%s\n" % line) + else: + self.warning("Could not find config path at %s" % avd_config_path) + + # enable EGL 3.0 in advancedFeatures.ini + AF_FILE = os.path.join(avd_home_dir, "advancedFeatures.ini") + with open(AF_FILE, "w") as f: + if self.use_gles3: + f.write("GLESDynamicVersion=on\n") + else: + f.write("GLESDynamicVersion=off\n") + + # extra diagnostics for kvm acceleration + emu = self.config.get("emulator_process_name") + if os.path.exists("/dev/kvm") and emu and "x86" in emu: + try: + self.run_command(["ls", "-l", "/dev/kvm"]) + self.run_command(["kvm-ok"]) + self.run_command(["emulator", "-accel-check"], env=env) + except Exception as e: + self.warning("Extra kvm diagnostics failed: %s" % str(e)) + + self.info("emulator env: %s" % str(env)) + command = ["emulator", "-avd", self.config["emulator_avd_name"]] + if "emulator_extra_args" in self.config: + command += self.config["emulator_extra_args"] + + dir = self.query_abs_dirs()["abs_blob_upload_dir"] + tmp_file = tempfile.NamedTemporaryFile( + mode="w", prefix="emulator-", suffix=".log", dir=dir, delete=False + ) + self.info("Launching the emulator with: %s" % " ".join(command)) + self.info("Writing log to %s" % tmp_file.name) + proc = subprocess.Popen( + command, stdout=tmp_file, stderr=tmp_file, env=env, bufsize=0 + ) + return proc + + def _verify_emulator(self): + boot_ok = self._retry( + 30, + 10, + self.is_boot_completed, + "Verify Android boot completed", + max_time=330, + ) + if not boot_ok: + self.warning("Unable to verify Android boot completion") + return False + return True + + def _verify_emulator_and_restart_on_fail(self): + emulator_ok = self._verify_emulator() + if not emulator_ok: + self.device_screenshot("screenshot-emulator-start") + self.kill_processes(self.config["emulator_process_name"]) + subprocess.check_call(["ps", "-ef"]) + # remove emulator tmp files + for dir in glob.glob("/tmp/android-*"): + self.rmtree(dir) + time.sleep(5) + self.emulator_proc = self._launch_emulator() + return emulator_ok + + def _retry(self, max_attempts, interval, func, description, max_time=0): + """ + Execute func until it returns True, up to max_attempts times, waiting for + interval seconds between each attempt. description is logged on each attempt. + If max_time is specified, no further attempts will be made once max_time + seconds have elapsed; this provides some protection for the case where + the run-time for func is long or highly variable. + """ + status = False + attempts = 0 + if max_time > 0: + end_time = datetime.datetime.now() + datetime.timedelta(seconds=max_time) + else: + end_time = None + while attempts < max_attempts and not status: + if (end_time is not None) and (datetime.datetime.now() > end_time): + self.info( + "Maximum retry run-time of %d seconds exceeded; " + "remaining attempts abandoned" % max_time + ) + break + if attempts != 0: + self.info("Sleeping %d seconds" % interval) + time.sleep(interval) + attempts += 1 + self.info( + ">> %s: Attempt #%d of %d" % (description, attempts, max_attempts) + ) + status = func() + return status + + def dump_perf_info(self): + """ + Dump some host and android device performance-related information + to an artifact file, to help understand task performance. + """ + dir = self.query_abs_dirs()["abs_blob_upload_dir"] + perf_path = os.path.join(dir, "android-performance.log") + with open(perf_path, "w") as f: + + f.write("\n\nHost cpufreq/scaling_governor:\n") + cpus = glob.glob("/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor") + for cpu in cpus: + out = subprocess.check_output(["cat", cpu], universal_newlines=True) + f.write("%s: %s" % (cpu, out)) + + f.write("\n\nHost /proc/cpuinfo:\n") + out = subprocess.check_output( + ["cat", "/proc/cpuinfo"], universal_newlines=True + ) + f.write(out) + + f.write("\n\nHost /proc/meminfo:\n") + out = subprocess.check_output( + ["cat", "/proc/meminfo"], universal_newlines=True + ) + f.write(out) + + f.write("\n\nHost process list:\n") + out = subprocess.check_output(["ps", "-ef"], universal_newlines=True) + f.write(out) + + f.write("\n\nDevice /proc/cpuinfo:\n") + cmd = "cat /proc/cpuinfo" + out = self.shell_output(cmd) + f.write(out) + cpuinfo = out + + f.write("\n\nDevice /proc/meminfo:\n") + cmd = "cat /proc/meminfo" + out = self.shell_output(cmd) + f.write(out) + + f.write("\n\nDevice process list:\n") + cmd = "ps" + out = self.shell_output(cmd) + f.write(out) + + # Search android cpuinfo for "BogoMIPS"; if found and < (minimum), retry + # this task, in hopes of getting a higher-powered environment. + # (Carry on silently if BogoMIPS is not found -- this may vary by + # Android implementation -- no big deal.) + # See bug 1321605: Sometimes the emulator is really slow, and + # low bogomips can be a good predictor of that condition. + bogomips_minimum = int(self.config.get("bogomips_minimum") or 0) + for line in cpuinfo.split("\n"): + m = re.match("BogoMIPS.*: (\d*)", line, re.IGNORECASE) + if m: + bogomips = int(m.group(1)) + if bogomips_minimum > 0 and bogomips < bogomips_minimum: + self.fatal( + "INFRA-ERROR: insufficient Android bogomips (%d < %d)" + % (bogomips, bogomips_minimum), + EXIT_STATUS_DICT[TBPL_RETRY], + ) + self.info("Found Android bogomips: %d" % bogomips) + break + + def logcat_path(self): + logcat_filename = "logcat-%s.log" % self.device_serial + return os.path.join( + self.query_abs_dirs()["abs_blob_upload_dir"], logcat_filename + ) + + def logcat_start(self): + """ + Start recording logcat. Writes logcat to the upload directory. + """ + # Start logcat for the device. The adb process runs until the + # corresponding device is stopped. Output is written directly to + # the blobber upload directory so that it is uploaded automatically + # at the end of the job. + self.logcat_file = open(self.logcat_path(), "w") + logcat_cmd = [ + self.adb_path, + "-s", + self.device_serial, + "logcat", + "-v", + "threadtime", + "Trace:S", + "StrictMode:S", + "ExchangeService:S", + ] + self.info(" ".join(logcat_cmd)) + self.logcat_proc = subprocess.Popen( + logcat_cmd, stdout=self.logcat_file, stdin=subprocess.PIPE + ) + + def logcat_stop(self): + """ + Stop logcat process started by logcat_start. + """ + if self.logcat_proc: + self.info("Killing logcat pid %d." % self.logcat_proc.pid) + self.logcat_proc.kill() + self.logcat_file.close() + + def _install_android_app_retry(self, app_path, replace): + import mozdevice + + try: + if app_path.endswith(".aab"): + self.device.install_app_bundle( + self.query_abs_dirs()["abs_bundletool_path"], app_path, timeout=120 + ) + self.device.run_as_package = self.query_package_name() + else: + self.device.run_as_package = self.device.install_app( + app_path, replace=replace, timeout=120 + ) + return True + except ( + mozdevice.ADBError, + mozdevice.ADBProcessError, + mozdevice.ADBTimeoutError, + ) as e: + self.info( + "Failed to install %s on %s: %s %s" + % (app_path, self.device_name, type(e).__name__, e) + ) + return False + + def install_android_app(self, app_path, replace=False): + """ + Install the specified app. + """ + app_installed = self._retry( + 5, + 10, + functools.partial(self._install_android_app_retry, app_path, replace), + "Install app", + ) + + if not app_installed: + self.fatal( + "INFRA-ERROR: Failed to install %s" % os.path.basename(app_path), + EXIT_STATUS_DICT[TBPL_RETRY], + ) + + def uninstall_android_app(self): + """ + Uninstall the app associated with the configured app, if it is + installed. + """ + import mozdevice + + try: + package_name = self.query_package_name() + self.device.uninstall_app(package_name) + except ( + mozdevice.ADBError, + mozdevice.ADBProcessError, + mozdevice.ADBTimeoutError, + ) as e: + self.info( + "Failed to uninstall %s from %s: %s %s" + % (package_name, self.device_name, type(e).__name__, e) + ) + self.fatal( + "INFRA-ERROR: %s Failed to uninstall %s" + % (type(e).__name__, package_name), + EXIT_STATUS_DICT[TBPL_RETRY], + ) + + def is_boot_completed(self): + import mozdevice + + try: + return self.device.is_device_ready(timeout=30) + except (ValueError, mozdevice.ADBError, mozdevice.ADBTimeoutError): + pass + return False + + def shell_output(self, cmd, enable_run_as=False): + import mozdevice + + try: + return self.device.shell_output( + cmd, timeout=30, enable_run_as=enable_run_as + ) + except (mozdevice.ADBTimeoutError) as e: + self.info( + "Failed to run shell command %s from %s: %s %s" + % (cmd, self.device_name, type(e).__name__, e) + ) + self.fatal( + "INFRA-ERROR: %s Failed to run shell command %s" + % (type(e).__name__, cmd), + EXIT_STATUS_DICT[TBPL_RETRY], + ) + + def device_screenshot(self, prefix): + """ + On emulator, save a screenshot of the entire screen to the upload directory; + otherwise, save a screenshot of the device to the upload directory. + + :param prefix specifies a filename prefix for the screenshot + """ + from mozscreenshot import dump_device_screen, dump_screen + + reset_dir = False + if not os.environ.get("MOZ_UPLOAD_DIR", None): + dirs = self.query_abs_dirs() + os.environ["MOZ_UPLOAD_DIR"] = dirs["abs_blob_upload_dir"] + reset_dir = True + if self.is_emulator: + if self.xre_path: + dump_screen(self.xre_path, self, prefix=prefix) + else: + self.info("Not saving screenshot: no XRE configured") + else: + dump_device_screen(self.device, self, prefix=prefix) + if reset_dir: + del os.environ["MOZ_UPLOAD_DIR"] + + def download_hostutils(self, xre_dir): + """ + Download and install hostutils from tooltool. + """ + xre_path = None + self.rmtree(xre_dir) + self.mkdir_p(xre_dir) + if self.config["hostutils_manifest_path"]: + url = self._get_repo_url(self.config["hostutils_manifest_path"]) + self._tooltool_fetch(url, xre_dir) + for p in glob.glob(os.path.join(xre_dir, "host-utils-*")): + if os.path.isdir(p) and os.path.isfile(os.path.join(p, "xpcshell")): + xre_path = p + if not xre_path: + self.fatal("xre path not found in %s" % xre_dir) + else: + self.fatal("configure hostutils_manifest_path!") + return xre_path + + def query_package_name(self): + if self.app_name is None: + # For convenience, assume geckoview.test/geckoview_example when install + # target looks like geckoview. + if "androidTest" in self.installer_path: + self.app_name = "org.mozilla.geckoview.test" + elif "test_runner" in self.installer_path: + self.app_name = "org.mozilla.geckoview.test_runner" + elif "geckoview" in self.installer_path: + self.app_name = "org.mozilla.geckoview_example" + if self.app_name is None: + # Find appname from package-name.txt - assumes download-and-extract + # has completed successfully. + # The app/package name will typically be org.mozilla.fennec, + # but org.mozilla.firefox for release builds, and there may be + # other variations. 'aapt dump badging <apk>' could be used as an + # alternative to package-name.txt, but introduces a dependency + # on aapt, found currently in the Android SDK build-tools component. + app_dir = self.abs_dirs["abs_work_dir"] + self.app_path = os.path.join(app_dir, self.installer_path) + unzip = self.query_exe("unzip") + package_path = os.path.join(app_dir, "package-name.txt") + unzip_cmd = [unzip, "-q", "-o", self.app_path] + self.run_command(unzip_cmd, cwd=app_dir, halt_on_failure=True) + self.app_name = str( + self.read_from_file(package_path, verbose=True) + ).rstrip() + return self.app_name + + def kill_processes(self, process_name): + self.info("Killing every process called %s" % process_name) + process_name = six.ensure_binary(process_name) + out = subprocess.check_output(["ps", "-A"]) + for line in out.splitlines(): + if process_name in line: + pid = int(line.split(None, 1)[0]) + self.info("Killing pid %d." % pid) + os.kill(pid, signal.SIGKILL) + + def delete_ANRs(self): + remote_dir = self.device.stack_trace_dir + try: + if not self.device.is_dir(remote_dir): + self.device.mkdir(remote_dir) + self.info("%s created" % remote_dir) + return + self.device.chmod(remote_dir, recursive=True) + for trace_file in self.device.ls(remote_dir, recursive=True): + trace_path = posixpath.join(remote_dir, trace_file) + if self.device.is_file(trace_path): + self.device.rm(trace_path) + self.info("%s deleted" % trace_path) + except Exception as e: + self.info( + "failed to delete %s: %s %s" % (remote_dir, type(e).__name__, str(e)) + ) + + def check_for_ANRs(self): + """ + Copy ANR (stack trace) files from device to upload directory. + """ + dirs = self.query_abs_dirs() + remote_dir = self.device.stack_trace_dir + try: + if not self.device.is_dir(remote_dir): + self.info("%s not found; ANR check skipped" % remote_dir) + return + self.device.chmod(remote_dir, recursive=True) + self.device.pull(remote_dir, dirs["abs_blob_upload_dir"]) + self.delete_ANRs() + except Exception as e: + self.info( + "failed to pull %s: %s %s" % (remote_dir, type(e).__name__, str(e)) + ) + + def delete_tombstones(self): + remote_dir = "/data/tombstones" + try: + if not self.device.is_dir(remote_dir): + self.device.mkdir(remote_dir) + self.info("%s created" % remote_dir) + return + self.device.chmod(remote_dir, recursive=True) + for trace_file in self.device.ls(remote_dir, recursive=True): + trace_path = posixpath.join(remote_dir, trace_file) + if self.device.is_file(trace_path): + self.device.rm(trace_path) + self.info("%s deleted" % trace_path) + except Exception as e: + self.info( + "failed to delete %s: %s %s" % (remote_dir, type(e).__name__, str(e)) + ) + + def check_for_tombstones(self): + """ + Copy tombstone files from device to upload directory. + """ + dirs = self.query_abs_dirs() + remote_dir = "/data/tombstones" + try: + if not self.device.is_dir(remote_dir): + self.info("%s not found; tombstone check skipped" % remote_dir) + return + self.device.chmod(remote_dir, recursive=True) + self.device.pull(remote_dir, dirs["abs_blob_upload_dir"]) + self.delete_tombstones() + except Exception as e: + self.info( + "failed to pull %s: %s %s" % (remote_dir, type(e).__name__, str(e)) + ) + + # Script actions + + def start_emulator(self): + """ + Starts the emulator + """ + if not self.is_emulator: + return + + dirs = self.query_abs_dirs() + ensure_dir(dirs["abs_work_dir"]) + ensure_dir(dirs["abs_blob_upload_dir"]) + + if not os.path.isfile(self.adb_path): + self.fatal("The adb binary '%s' is not a valid file!" % self.adb_path) + self.kill_processes("xpcshell") + self.emulator_proc = self._launch_emulator() + + def verify_device(self): + """ + Check to see if the emulator can be contacted via adb. + If any communication attempt fails, kill the emulator, re-launch, and re-check. + """ + if not self.is_android: + return + + if self.is_emulator: + max_restarts = 5 + emulator_ok = self._retry( + max_restarts, + 10, + self._verify_emulator_and_restart_on_fail, + "Check emulator", + ) + if not emulator_ok: + self.fatal( + "INFRA-ERROR: Unable to start emulator after %d attempts" + % max_restarts, + EXIT_STATUS_DICT[TBPL_RETRY], + ) + + self.mkdir_p(self.query_abs_dirs()["abs_blob_upload_dir"]) + self.dump_perf_info() + self.logcat_start() + self.delete_ANRs() + self.delete_tombstones() + self.info("verify_device complete") + + @PreScriptAction("run-tests") + def timed_screenshots(self, action, success=None): + """ + If configured, start screenshot timers. + """ + if not self.is_android: + return + + def take_screenshot(seconds): + self.device_screenshot("screenshot-%ss-" % str(seconds)) + self.info("timed (%ss) screenshot complete" % str(seconds)) + + self.timers = [] + for seconds in self.config.get("screenshot_times", []): + self.info("screenshot requested %s seconds from now" % str(seconds)) + t = Timer(int(seconds), take_screenshot, [seconds]) + t.start() + self.timers.append(t) + + @PostScriptAction("run-tests") + def stop_device(self, action, success=None): + """ + Stop logcat and kill the emulator, if necessary. + """ + if not self.is_android: + return + + for t in self.timers: + t.cancel() + if self.worst_status != TBPL_RETRY: + self.check_for_ANRs() + self.check_for_tombstones() + else: + self.info("ANR and tombstone checks skipped due to TBPL_RETRY") + self.logcat_stop() + if self.is_emulator: + self.kill_processes(self.config["emulator_process_name"]) diff --git a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py new file mode 100644 index 0000000000..fd850324ed --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py @@ -0,0 +1,679 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import errno +import json +import os +import posixpath +import shutil +import sys +import tempfile +import uuid +import zipfile + +import mozinfo + +from mozharness.base.script import PostScriptAction, PreScriptAction +from mozharness.mozilla.testing.per_test_base import SingleTestMixin + +code_coverage_config_options = [ + [ + ["--code-coverage"], + { + "action": "store_true", + "dest": "code_coverage", + "default": False, + "help": "Whether gcov c++ code coverage should be run.", + }, + ], + [ + ["--per-test-coverage"], + { + "action": "store_true", + "dest": "per_test_coverage", + "default": False, + "help": "Whether per-test coverage should be collected.", + }, + ], + [ + ["--disable-ccov-upload"], + { + "action": "store_true", + "dest": "disable_ccov_upload", + "default": False, + "help": "Whether test run should package and upload code coverage data.", + }, + ], + [ + ["--java-code-coverage"], + { + "action": "store_true", + "dest": "java_code_coverage", + "default": False, + "help": "Whether Java code coverage should be run.", + }, + ], +] + + +class CodeCoverageMixin(SingleTestMixin): + """ + Mixin for setting GCOV_PREFIX during test execution, packaging up + the resulting .gcda files and uploading them to blobber. + """ + + gcov_dir = None + grcov_dir = None + grcov_bin = None + jsvm_dir = None + prefix = None + per_test_reports = {} + + def __init__(self, **kwargs): + if mozinfo.os == "linux" or mozinfo.os == "mac": + self.grcov_bin = "grcov" + elif mozinfo.os == "win": + self.grcov_bin = "grcov.exe" + else: + raise Exception("Unexpected OS: {}".format(mozinfo.os)) + + super(CodeCoverageMixin, self).__init__(**kwargs) + + @property + def code_coverage_enabled(self): + try: + return bool(self.config.get("code_coverage")) + except (AttributeError, KeyError, TypeError): + return False + + @property + def per_test_coverage(self): + try: + return bool(self.config.get("per_test_coverage")) + except (AttributeError, KeyError, TypeError): + return False + + @property + def ccov_upload_disabled(self): + try: + return bool(self.config.get("disable_ccov_upload")) + except (AttributeError, KeyError, TypeError): + return False + + @property + def jsd_code_coverage_enabled(self): + try: + return bool(self.config.get("jsd_code_coverage")) + except (AttributeError, KeyError, TypeError): + return False + + @property + def java_code_coverage_enabled(self): + try: + return bool(self.config.get("java_code_coverage")) + except (AttributeError, KeyError, TypeError): + return False + + def _setup_cpp_js_coverage_tools(self): + fetches_dir = os.environ["MOZ_FETCHES_DIR"] + with open(os.path.join(fetches_dir, "target.mozinfo.json"), "r") as f: + build_mozinfo = json.load(f) + + self.prefix = build_mozinfo["topsrcdir"] + + strip_count = len(list(filter(None, self.prefix.split("/")))) + os.environ["GCOV_PREFIX_STRIP"] = str(strip_count) + + # Download the gcno archive from the build machine. + url_to_gcno = self.query_build_dir_url("target.code-coverage-gcno.zip") + self.download_file(url_to_gcno, parent_dir=self.grcov_dir) + + # Download the chrome-map.json file from the build machine. + url_to_chrome_map = self.query_build_dir_url("chrome-map.json") + self.download_file(url_to_chrome_map, parent_dir=self.grcov_dir) + + def _setup_java_coverage_tools(self): + # Download and extract jacoco-cli from the build task. + url_to_jacoco = self.query_build_dir_url("target.jacoco-cli.jar") + self.jacoco_jar = os.path.join(tempfile.mkdtemp(), "target.jacoco-cli.jar") + self.download_file(url_to_jacoco, self.jacoco_jar) + + # Download and extract class files from the build task. + self.classfiles_dir = tempfile.mkdtemp() + for archive in ["target.geckoview_classfiles.zip", "target.app_classfiles.zip"]: + url_to_classfiles = self.query_build_dir_url(archive) + classfiles_zip_path = os.path.join(self.classfiles_dir, archive) + self.download_file(url_to_classfiles, classfiles_zip_path) + with zipfile.ZipFile(classfiles_zip_path, "r") as z: + z.extractall(self.classfiles_dir) + os.remove(classfiles_zip_path) + + # Create the directory where the emulator coverage file will be placed. + self.java_coverage_output_dir = tempfile.mkdtemp() + + @PostScriptAction("download-and-extract") + def setup_coverage_tools(self, action, success=None): + if not self.code_coverage_enabled and not self.java_code_coverage_enabled: + return + + self.grcov_dir = os.path.join(os.environ["MOZ_FETCHES_DIR"], "grcov") + if not os.path.isfile(os.path.join(self.grcov_dir, self.grcov_bin)): + raise Exception( + "File not found: {}".format( + os.path.join(self.grcov_dir, self.grcov_bin) + ) + ) + + if self.code_coverage_enabled: + self._setup_cpp_js_coverage_tools() + + if self.java_code_coverage_enabled: + self._setup_java_coverage_tools() + + @PostScriptAction("download-and-extract") + def find_tests_for_coverage(self, action, success=None): + """ + For each file modified on this push, determine if the modified file + is a test, by searching test manifests. Populate self.verify_suites + with test files, organized by suite. + + This depends on test manifests, so can only run after test zips have + been downloaded and extracted. + """ + if not self.per_test_coverage: + return + + self.find_modified_tests() + + # TODO: Add tests that haven't been run for a while (a week? N pushes?) + + # Add baseline code coverage collection tests + baseline_tests_by_ext = { + ".html": { + "test": "testing/mochitest/baselinecoverage/plain/test_baselinecoverage.html", + "suite": "mochitest-plain", + }, + ".js": { + "test": "testing/mochitest/baselinecoverage/browser_chrome/browser_baselinecoverage.js", # NOQA: E501 + "suite": "mochitest-browser-chrome", + }, + ".xhtml": { + "test": "testing/mochitest/baselinecoverage/chrome/test_baselinecoverage.xhtml", + "suite": "mochitest-chrome", + }, + } + + baseline_tests_by_suite = { + "mochitest-browser-chrome": "testing/mochitest/baselinecoverage/browser_chrome/" + "browser_baselinecoverage_browser-chrome.js" + } + + wpt_baseline_test = "tests/web-platform/mozilla/tests/baselinecoverage/wpt_baselinecoverage.html" # NOQA: E501 + if self.config.get("per_test_category") == "web-platform": + if "testharness" not in self.suites: + self.suites["testharness"] = [] + if wpt_baseline_test not in self.suites["testharness"]: + self.suites["testharness"].append(wpt_baseline_test) + return + + # Go through all the tests and find all + # the baseline tests that are needed. + tests_to_add = {} + for suite in self.suites: + if len(self.suites[suite]) == 0: + continue + if suite in baseline_tests_by_suite: + if suite not in tests_to_add: + tests_to_add[suite] = [] + tests_to_add[suite].append(baseline_tests_by_suite[suite]) + continue + + # Default to file types if the suite has no baseline + for test in self.suites[suite]: + _, test_ext = os.path.splitext(test) + + if test_ext not in baseline_tests_by_ext: + # Add the '.js' test as a default baseline + # if none other exists. + test_ext = ".js" + baseline_test_suite = baseline_tests_by_ext[test_ext]["suite"] + baseline_test_name = baseline_tests_by_ext[test_ext]["test"] + + if baseline_test_suite not in tests_to_add: + tests_to_add[baseline_test_suite] = [] + if baseline_test_name not in tests_to_add[baseline_test_suite]: + tests_to_add[baseline_test_suite].append(baseline_test_name) + + # Add all baseline tests needed + for suite in tests_to_add: + for test in tests_to_add[suite]: + if suite not in self.suites: + self.suites[suite] = [] + if test not in self.suites[suite]: + self.suites[suite].append(test) + + @property + def coverage_args(self): + return [] + + def set_coverage_env(self, env, is_baseline_test=False): + # Set the GCOV directory. + self.gcov_dir = tempfile.mkdtemp() + env["GCOV_PREFIX"] = self.gcov_dir + + # Set the GCOV/JSVM directories where counters will be dumped in per-test mode. + if self.per_test_coverage and not is_baseline_test: + env["GCOV_RESULTS_DIR"] = tempfile.mkdtemp() + env["JSVM_RESULTS_DIR"] = tempfile.mkdtemp() + + # Set JSVM directory. + self.jsvm_dir = tempfile.mkdtemp() + env["JS_CODE_COVERAGE_OUTPUT_DIR"] = self.jsvm_dir + + @PreScriptAction("run-tests") + def _set_gcov_prefix(self, action): + if not self.code_coverage_enabled: + return + + if self.per_test_coverage: + return + + self.set_coverage_env(os.environ) + + def parse_coverage_artifacts( + self, + gcov_dir, + jsvm_dir, + merge=False, + output_format="lcov", + filter_covered=False, + ): + jsvm_output_file = "jsvm_lcov_output.info" + grcov_output_file = "grcov_lcov_output.info" + + dirs = self.query_abs_dirs() + + sys.path.append(dirs["abs_test_install_dir"]) + sys.path.append(os.path.join(dirs["abs_test_install_dir"], "mozbuild")) + + from codecoverage.lcov_rewriter import LcovFileRewriter + + jsvm_files = [os.path.join(jsvm_dir, e) for e in os.listdir(jsvm_dir)] + rewriter = LcovFileRewriter(os.path.join(self.grcov_dir, "chrome-map.json")) + rewriter.rewrite_files(jsvm_files, jsvm_output_file, "") + + # Run grcov on the zipped .gcno and .gcda files. + grcov_command = [ + os.path.join(self.grcov_dir, self.grcov_bin), + "-t", + output_format, + "-p", + self.prefix, + "--ignore", + "**/fetches/*", + os.path.join(self.grcov_dir, "target.code-coverage-gcno.zip"), + gcov_dir, + ] + + if "coveralls" in output_format: + grcov_command += ["--token", "UNUSED", "--commit-sha", "UNUSED"] + + if merge: + grcov_command += [jsvm_output_file] + + if mozinfo.os == "win" or mozinfo.os == "mac": + grcov_command += ["--llvm"] + + if filter_covered: + grcov_command += ["--filter", "covered"] + + def skip_cannot_normalize(output_to_filter): + return "\n".join( + line + for line in output_to_filter.rstrip().splitlines() + if "cannot be normalized because" not in line + ) + + # 'grcov_output' will be a tuple, the first variable is the path to the lcov output, + # the other is the path to the standard error output. + tmp_output_file, _ = self.get_output_from_command( + grcov_command, + silent=True, + save_tmpfiles=True, + return_type="files", + throw_exception=True, + output_filter=skip_cannot_normalize, + ) + shutil.move(tmp_output_file, grcov_output_file) + + shutil.rmtree(gcov_dir) + shutil.rmtree(jsvm_dir) + + if merge: + os.remove(jsvm_output_file) + return grcov_output_file + else: + return grcov_output_file, jsvm_output_file + + def add_per_test_coverage_report(self, env, suite, test): + gcov_dir = ( + env["GCOV_RESULTS_DIR"] if "GCOV_RESULTS_DIR" in env else self.gcov_dir + ) + jsvm_dir = ( + env["JSVM_RESULTS_DIR"] if "JSVM_RESULTS_DIR" in env else self.jsvm_dir + ) + + grcov_file = self.parse_coverage_artifacts( + gcov_dir, + jsvm_dir, + merge=True, + output_format="coveralls", + filter_covered=True, + ) + + report_file = str(uuid.uuid4()) + ".json" + shutil.move(grcov_file, report_file) + + # Get the test path relative to topsrcdir. + # This mapping is constructed by self.find_modified_tests(). + test = self.test_src_path.get(test.replace(os.sep, posixpath.sep), test) + + # Log a warning if the test path is still an absolute path. + if os.path.isabs(test): + self.warn("Found absolute path for test: {}".format(test)) + + if suite not in self.per_test_reports: + self.per_test_reports[suite] = {} + assert test not in self.per_test_reports[suite] + self.per_test_reports[suite][test] = report_file + + if "GCOV_RESULTS_DIR" in env: + assert "JSVM_RESULTS_DIR" in env + # In this case, parse_coverage_artifacts has removed GCOV_RESULTS_DIR and + # JSVM_RESULTS_DIR so we need to remove GCOV_PREFIX and JS_CODE_COVERAGE_OUTPUT_DIR. + try: + shutil.rmtree(self.gcov_dir) + except FileNotFoundError: + pass + + try: + shutil.rmtree(self.jsvm_dir) + except FileNotFoundError: + pass + + def is_covered(self, sf): + # For C/C++ source files, we can consider a file as being uncovered + # when all its source lines are uncovered. + all_lines_uncovered = all(c is None or c == 0 for c in sf["coverage"]) + if all_lines_uncovered: + return False + + # For JavaScript files, we can't do the same, as the top-level is always + # executed, even if it just contains declarations. So, we need to check if + # all its functions, except the top-level, are uncovered. + functions = sf["functions"] if "functions" in sf else [] + all_functions_uncovered = all( + not f["exec"] or f["name"] == "top-level" for f in functions + ) + if all_functions_uncovered and len(functions) > 1: + return False + + return True + + @PostScriptAction("run-tests") + def _package_coverage_data(self, action, success=None): + dirs = self.query_abs_dirs() + + if not self.code_coverage_enabled: + return + + if self.per_test_coverage: + if not self.per_test_reports: + self.info("No tests were found...not saving coverage data.") + return + + # Get the baseline tests that were run. + baseline_tests_ext_cov = {} + baseline_tests_suite_cov = {} + for suite, data in self.per_test_reports.items(): + for test, grcov_file in data.items(): + if "baselinecoverage" not in test: + continue + + # TODO: Optimize this part which loads JSONs + # with a size of about 40Mb into memory for diffing later. + # Bug 1460064 is filed for this. + with open(grcov_file, "r") as f: + data = json.load(f) + + if suite in os.path.split(test)[-1]: + baseline_tests_suite_cov[suite] = data + else: + _, baseline_filetype = os.path.splitext(test) + baseline_tests_ext_cov[baseline_filetype] = data + + dest = os.path.join( + dirs["abs_blob_upload_dir"], "per-test-coverage-reports.zip" + ) + with zipfile.ZipFile(dest, "w", zipfile.ZIP_DEFLATED) as z: + for suite, data in self.per_test_reports.items(): + for test, grcov_file in data.items(): + if "baselinecoverage" in test: + # Don't keep the baseline coverage + continue + else: + # Get test coverage + with open(grcov_file, "r") as f: + report = json.load(f) + + # Remove uncovered files, as they are unneeded for per-test + # coverage purposes. + report["source_files"] = [ + sf + for sf in report["source_files"] + if self.is_covered(sf) + ] + + # Get baseline coverage + baseline_coverage = {} + if suite in baseline_tests_suite_cov: + baseline_coverage = baseline_tests_suite_cov[suite] + elif self.config.get("per_test_category") == "web-platform": + baseline_coverage = baseline_tests_ext_cov[".html"] + else: + for file_type in baseline_tests_ext_cov: + if not test.endswith(file_type): + continue + baseline_coverage = baseline_tests_ext_cov[ + file_type + ] + break + + if not baseline_coverage: + # Default to the '.js' baseline as it is the largest + self.info("Did not find a baseline test for: " + test) + baseline_coverage = baseline_tests_ext_cov[".js"] + + unique_coverage = rm_baseline_cov(baseline_coverage, report) + + with open(grcov_file, "w") as f: + json.dump( + { + "test": test, + "suite": suite, + "report": unique_coverage, + }, + f, + ) + + z.write(grcov_file) + return + + del os.environ["GCOV_PREFIX_STRIP"] + del os.environ["GCOV_PREFIX"] + del os.environ["JS_CODE_COVERAGE_OUTPUT_DIR"] + + if not self.ccov_upload_disabled: + grcov_output_file, jsvm_output_file = self.parse_coverage_artifacts( + self.gcov_dir, self.jsvm_dir + ) + + try: + os.makedirs(dirs["abs_blob_upload_dir"]) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + # Zip the grcov output and upload it. + grcov_zip_path = os.path.join( + dirs["abs_blob_upload_dir"], "code-coverage-grcov.zip" + ) + with zipfile.ZipFile(grcov_zip_path, "w", zipfile.ZIP_DEFLATED) as z: + z.write(grcov_output_file) + + # Zip the JSVM coverage data and upload it. + jsvm_zip_path = os.path.join( + dirs["abs_blob_upload_dir"], "code-coverage-jsvm.zip" + ) + with zipfile.ZipFile(jsvm_zip_path, "w", zipfile.ZIP_DEFLATED) as z: + z.write(jsvm_output_file) + + shutil.rmtree(self.grcov_dir) + + @PostScriptAction("run-tests") + def process_java_coverage_data(self, action, success=None): + """ + Run JaCoCo on the coverage.ec file in order to get a XML report. + After that, run grcov on the XML report to get a lcov report. + Finally, archive the lcov file and upload it, as process_coverage_data is doing. + """ + if not self.java_code_coverage_enabled: + return + + # If the emulator became unresponsive, the task has failed and we don't + # have any coverage report file, so stop running this function and + # allow the task to be retried automatically. + if not success and not os.listdir(self.java_coverage_output_dir): + return + + report_files = [ + os.path.join(self.java_coverage_output_dir, f) + for f in os.listdir(self.java_coverage_output_dir) + ] + assert len(report_files) > 0, "JaCoCo coverage data files were not found." + + dirs = self.query_abs_dirs() + xml_path = tempfile.mkdtemp() + jacoco_command = ( + ["java", "-jar", self.jacoco_jar, "report"] + + report_files + + [ + "--classfiles", + self.classfiles_dir, + "--name", + "geckoview-junit", + "--xml", + os.path.join(xml_path, "geckoview-junit.xml"), + ] + ) + self.run_command(jacoco_command, halt_on_failure=True) + + grcov_command = [ + os.path.join(self.grcov_dir, self.grcov_bin), + "-t", + "lcov", + xml_path, + ] + tmp_output_file, _ = self.get_output_from_command( + grcov_command, + silent=True, + save_tmpfiles=True, + return_type="files", + throw_exception=True, + ) + + if not self.ccov_upload_disabled: + grcov_zip_path = os.path.join( + dirs["abs_blob_upload_dir"], "code-coverage-grcov.zip" + ) + with zipfile.ZipFile(grcov_zip_path, "w", zipfile.ZIP_DEFLATED) as z: + z.write(tmp_output_file, "grcov_lcov_output.info") + + +def rm_baseline_cov(baseline_coverage, test_coverage): + """ + Returns the difference between test_coverage and + baseline_coverage, such that what is returned + is the unique coverage for the test in question. + """ + + # Get all files into a quicker search format + unique_test_coverage = test_coverage + baseline_files = {el["name"]: el for el in baseline_coverage["source_files"]} + test_files = {el["name"]: el for el in test_coverage["source_files"]} + + # Perform the difference and find everything + # unique to the test. + unique_file_coverage = {} + for test_file in test_files: + if test_file not in baseline_files: + unique_file_coverage[test_file] = test_files[test_file] + continue + + if len(test_files[test_file]["coverage"]) != len( + baseline_files[test_file]["coverage"] + ): + # File has line number differences due to gcov bug: + # https://bugzilla.mozilla.org/show_bug.cgi?id=1410217 + continue + + # TODO: Attempt to rewrite this section to remove one of the two + # iterations over a test's source file's coverage for optimization. + # Bug 1460064 was filed for this. + + # Get line numbers and the differences + file_coverage = { + i + for i, cov in enumerate(test_files[test_file]["coverage"]) + if cov is not None and cov > 0 + } + + baseline = { + i + for i, cov in enumerate(baseline_files[test_file]["coverage"]) + if cov is not None and cov > 0 + } + + unique_coverage = file_coverage - baseline + + if len(unique_coverage) > 0: + unique_file_coverage[test_file] = test_files[test_file] + + # Return the data to original format to return + # coverage within the test_coverge data object. + fmt_unique_coverage = [] + for i, cov in enumerate(unique_file_coverage[test_file]["coverage"]): + if cov is None: + fmt_unique_coverage.append(None) + continue + + # TODO: Bug 1460061, determine if hit counts + # need to be considered. + if cov > 0: + # If there is a count + if i in unique_coverage: + # Only add the count if it's unique + fmt_unique_coverage.append( + unique_file_coverage[test_file]["coverage"][i] + ) + continue + # Zero out everything that is not unique + fmt_unique_coverage.append(0) + unique_file_coverage[test_file]["coverage"] = fmt_unique_coverage + + # Reformat to original test_coverage list structure + unique_test_coverage["source_files"] = list(unique_file_coverage.values()) + + return unique_test_coverage diff --git a/testing/mozharness/mozharness/mozilla/testing/errors.py b/testing/mozharness/mozharness/mozilla/testing/errors.py new file mode 100644 index 0000000000..84c00b0a8b --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/testing/errors.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +"""Mozilla error lists for running tests. + +Error lists are used to parse output in mozharness.base.log.OutputParser. + +Each line of output is matched against each substring or regular expression +in the error list. On a match, we determine the 'level' of that line, +whether IGNORE, DEBUG, INFO, WARNING, ERROR, CRITICAL, or FATAL. + +""" + +import re + +from mozharness.base.log import ERROR, INFO, WARNING + +# ErrorLists {{{1 +_mochitest_summary = { + "regex": re.compile( + r"""(\d+ INFO (Passed|Failed|Todo):\ +(\d+)|\t(Passed|Failed|Todo): (\d+))""" + ), # NOQA: E501 + "pass_group": "Passed", + "fail_group": "Failed", + "known_fail_group": "Todo", +} + +_reftest_summary = { + "regex": re.compile( + r"""REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \(""" + ), # NOQA: E501 + "pass_group": "Successful", + "fail_group": "Unexpected", + "known_fail_group": "Known problems", +} + +TinderBoxPrintRe = { + "mochitest-chrome_summary": _mochitest_summary, + "mochitest-webgl1-core_summary": _mochitest_summary, + "mochitest-webgl1-ext_summary": _mochitest_summary, + "mochitest-webgl2-core_summary": _mochitest_summary, + "mochitest-webgl2-ext_summary": _mochitest_summary, + "mochitest-webgl2-deqp_summary": _mochitest_summary, + "mochitest-webgpu_summary": _mochitest_summary, + "mochitest-media_summary": _mochitest_summary, + "mochitest-plain_summary": _mochitest_summary, + "mochitest-plain-gpu_summary": _mochitest_summary, + "marionette_summary": { + "regex": re.compile(r"""(passed|failed|todo):\ +(\d+)"""), + "pass_group": "passed", + "fail_group": "failed", + "known_fail_group": "todo", + }, + "reftest_summary": _reftest_summary, + "reftest-qr_summary": _reftest_summary, + "crashtest_summary": _reftest_summary, + "crashtest-qr_summary": _reftest_summary, + "xpcshell_summary": { + "regex": re.compile(r"""INFO \| (Passed|Failed|Todo): (\d+)"""), + "pass_group": "Passed", + "fail_group": "Failed", + "known_fail_group": "Todo", + }, + "jsreftest_summary": _reftest_summary, + "instrumentation_summary": _mochitest_summary, + "cppunittest_summary": { + "regex": re.compile(r"""cppunittests INFO \| (Passed|Failed): (\d+)"""), + "pass_group": "Passed", + "fail_group": "Failed", + "known_fail_group": None, + }, + "gtest_summary": { + "regex": re.compile(r"""(Passed|Failed): (\d+)"""), + "pass_group": "Passed", + "fail_group": "Failed", + "known_fail_group": None, + }, + "jittest_summary": { + "regex": re.compile(r"""(Passed|Failed): (\d+)"""), + "pass_group": "Passed", + "fail_group": "Failed", + "known_fail_group": None, + }, + "mozbase_summary": { + "regex": re.compile(r"""(OK)|(FAILED) \(errors=(\d+)"""), + "pass_group": "OK", + "fail_group": "FAILED", + "known_fail_group": None, + }, + "geckoview_summary": { + "regex": re.compile(r"""(Passed|Failed): (\d+)"""), + "pass_group": "Passed", + "fail_group": "Failed", + "known_fail_group": None, + }, + "geckoview-junit_summary": { + "regex": re.compile(r"""(Passed|Failed): (\d+)"""), + "pass_group": "Passed", + "fail_group": "Failed", + "known_fail_group": None, + }, + "harness_error": { + "full_regex": re.compile( + r"(?:TEST-UNEXPECTED-FAIL|PROCESS-CRASH) \| .* \|[^\|]* (application crashed|missing output line for total leaks!|negative leaks caught!|\d+ bytes leaked)" # NOQA: E501 + ), + "minimum_regex": re.compile(r"""(TEST-UNEXPECTED|PROCESS-CRASH)"""), + "retry_regex": re.compile( + r"""(FAIL-SHOULD-RETRY|No space left on device|ADBError|ADBProcessError|ADBTimeoutError|program finished with exit code 80|INFRA-ERROR)""" # NOQA: E501 + ), + }, +} + +TestPassed = [ + { + "regex": re.compile("""(TEST-INFO|TEST-KNOWN-FAIL|TEST-PASS|INFO \| )"""), + "level": INFO, + }, +] + +BaseHarnessErrorList = [ + { + "substr": "TEST-UNEXPECTED", + "level": ERROR, + }, + { + "substr": "PROCESS-CRASH", + "level": ERROR, + }, + { + "regex": re.compile("""ERROR: (Address|Leak)Sanitizer"""), + "level": ERROR, + }, + { + "regex": re.compile("""thread '([^']+)' panicked"""), + "level": ERROR, + }, + { + "substr": "pure virtual method called", + "level": ERROR, + }, + { + "substr": "Pure virtual function called!", + "level": ERROR, + }, +] + +HarnessErrorList = BaseHarnessErrorList + [ + { + "substr": "A content process crashed", + "level": ERROR, + }, +] + +# wpt can have expected crashes so we can't always turn treeherder orange in those cases +WptHarnessErrorList = BaseHarnessErrorList + +LogcatErrorList = [ + { + "substr": "Fatal signal 11 (SIGSEGV)", + "level": ERROR, + "explanation": "This usually indicates the B2G process has crashed", + }, + { + "substr": "Fatal signal 7 (SIGBUS)", + "level": ERROR, + "explanation": "This usually indicates the B2G process has crashed", + }, + {"substr": "[JavaScript Error:", "level": WARNING}, + { + "substr": "seccomp sandbox violation", + "level": ERROR, + "explanation": "A content process has violated the system call sandbox (bug 790923)", + }, +] diff --git a/testing/mozharness/mozharness/mozilla/testing/per_test_base.py b/testing/mozharness/mozharness/mozilla/testing/per_test_base.py new file mode 100644 index 0000000000..8e83643142 --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/testing/per_test_base.py @@ -0,0 +1,540 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** + +import itertools +import json +import math +import os +import posixpath +import sys + +import mozinfo +from manifestparser import TestManifest + + +class SingleTestMixin(object): + """Utility functions for per-test testing like test verification and per-test coverage.""" + + def __init__(self, **kwargs): + super(SingleTestMixin, self).__init__(**kwargs) + + self.suites = {} + self.tests_downloaded = False + self.reftest_test_dir = None + self.jsreftest_test_dir = None + # Map from full test path on the test machine to a relative path in the source checkout. + # Use self._map_test_path_to_source(test_machine_path, source_path) to add a mapping. + self.test_src_path = {} + self.per_test_log_index = 1 + + def _map_test_path_to_source(self, test_machine_path, source_path): + test_machine_path = test_machine_path.replace(os.sep, posixpath.sep) + source_path = source_path.replace(os.sep, posixpath.sep) + self.test_src_path[test_machine_path] = source_path + + def _is_gpu_suite(self, suite): + if suite and (suite == "gpu" or suite.startswith("webgl")): + return True + return False + + def _find_misc_tests(self, dirs, changed_files, gpu=False): + manifests = [ + ( + os.path.join(dirs["abs_mochitest_dir"], "tests", "mochitest.ini"), + "mochitest-plain", + ), + ( + os.path.join(dirs["abs_mochitest_dir"], "chrome", "chrome.ini"), + "mochitest-chrome", + ), + ( + os.path.join( + dirs["abs_mochitest_dir"], "browser", "browser-chrome.ini" + ), + "mochitest-browser-chrome", + ), + ( + os.path.join(dirs["abs_mochitest_dir"], "a11y", "a11y.ini"), + "mochitest-a11y", + ), + ( + os.path.join(dirs["abs_xpcshell_dir"], "tests", "xpcshell.ini"), + "xpcshell", + ), + ] + is_fission = "fission.autostart=true" in self.config.get("extra_prefs", []) + tests_by_path = {} + all_disabled = [] + for (path, suite) in manifests: + if os.path.exists(path): + man = TestManifest([path], strict=False) + active = man.active_tests( + exists=False, disabled=True, filters=[], **mozinfo.info + ) + # Remove disabled tests. Also, remove tests with the same path as + # disabled tests, even if they are not disabled, since per-test mode + # specifies tests by path (it cannot distinguish between two or more + # tests with the same path specified in multiple manifests). + disabled = [t["relpath"] for t in active if "disabled" in t] + all_disabled += disabled + new_by_path = { + t["relpath"]: (suite, t.get("subsuite"), None) + for t in active + if "disabled" not in t and t["relpath"] not in disabled + } + tests_by_path.update(new_by_path) + self.info( + "Per-test run updated with manifest %s (%d active, %d skipped)" + % (path, len(new_by_path), len(disabled)) + ) + + ref_manifests = [ + ( + os.path.join( + dirs["abs_reftest_dir"], + "tests", + "layout", + "reftests", + "reftest.list", + ), + "reftest", + "gpu", + ), # gpu + ( + os.path.join( + dirs["abs_reftest_dir"], + "tests", + "testing", + "crashtest", + "crashtests.list", + ), + "crashtest", + None, + ), + ] + sys.path.append(dirs["abs_reftest_dir"]) + import manifest + + self.reftest_test_dir = os.path.join(dirs["abs_reftest_dir"], "tests") + for (path, suite, subsuite) in ref_manifests: + if os.path.exists(path): + man = manifest.ReftestManifest() + man.load(path) + for t in man.tests: + relpath = os.path.relpath(t["path"], self.reftest_test_dir) + referenced = ( + t["referenced-test"] if "referenced-test" in t else None + ) + tests_by_path[relpath] = (suite, subsuite, referenced) + self._map_test_path_to_source(t["path"], relpath) + self.info( + "Per-test run updated with manifest %s (%d tests)" + % (path, len(man.tests)) + ) + + suite = "jsreftest" + self.jsreftest_test_dir = os.path.join( + dirs["abs_test_install_dir"], "jsreftest", "tests" + ) + path = os.path.join(self.jsreftest_test_dir, "jstests.list") + if os.path.exists(path): + man = manifest.ReftestManifest() + man.load(path) + for t in man.files: + # expect manifest test to look like: + # ".../tests/jsreftest/tests/jsreftest.html?test=test262/.../some_test.js" + # while the test is in mercurial at: + # js/src/tests/test262/.../some_test.js + epos = t.find("=") + if epos > 0: + relpath = t[epos + 1 :] + test_path = os.path.join(self.jsreftest_test_dir, relpath) + relpath = os.path.join("js", "src", "tests", relpath) + self._map_test_path_to_source(test_path, relpath) + tests_by_path.update({relpath: (suite, None, None)}) + else: + self.warning("unexpected jsreftest test format: %s" % str(t)) + self.info( + "Per-test run updated with manifest %s (%d tests)" + % (path, len(man.files)) + ) + + # for each changed file, determine if it is a test file, and what suite it is in + for file in changed_files: + # manifest paths use os.sep (like backslash on Windows) but + # automation-relevance uses posixpath.sep + file = file.replace(posixpath.sep, os.sep) + entry = tests_by_path.get(file) + if not entry: + if file in all_disabled: + self.info("'%s' has been skipped on this platform." % file) + if os.environ.get("MOZHARNESS_TEST_PATHS", None) is not None: + self.info("Per-test run could not find requested test '%s'" % file) + continue + + if gpu and not self._is_gpu_suite(entry[1]): + self.info( + "Per-test run (gpu) discarded non-gpu test %s (%s)" + % (file, entry[1]) + ) + continue + elif not gpu and self._is_gpu_suite(entry[1]): + self.info( + "Per-test run (non-gpu) discarded gpu test %s (%s)" + % (file, entry[1]) + ) + continue + + if is_fission and ( + (entry[0] == "mochitest-a11y") or (entry[0] == "mochitest-chrome") + ): + self.info( + "Per-test run (fission) discarded non-e10s test %s (%s)" + % (file, entry[0]) + ) + continue + + if entry[2] is not None and "about:" not in entry[2]: + # Test name substitution, for reftest reference file handling: + # - if both test and reference modified, run the test file + # - if only reference modified, run the test file + test_file = os.path.join( + os.path.dirname(file), os.path.basename(entry[2]) + ) + self.info("Per-test run substituting %s for %s" % (test_file, file)) + file = test_file + + self.info("Per-test run found test %s (%s/%s)" % (file, entry[0], entry[1])) + subsuite_mapping = { + # Map (<suite>, <subsuite>): <full-suite> + # <suite> is associated with a manifest, explicitly in code above + # <subsuite> comes from "subsuite" tags in some manifest entries + # <full-suite> is a unique id for the suite, matching desktop mozharness configs + ( + "mochitest-browser-chrome", + "a11y", + None, + ): "mochitest-browser-a11y", + ( + "mochitest-browser-chrome", + "media-bc", + None, + ): "mochitest-browser-media", + ( + "mochitest-browser-chrome", + "devtools", + None, + ): "mochitest-devtools-chrome", + ("mochitest-browser-chrome", "remote", None): "mochitest-remote", + ( + "mochitest-browser-chrome", + "screenshots", + None, + ): "mochitest-browser-chrome-screenshots", # noqa + ("mochitest-plain", "media", None): "mochitest-media", + # below should be on test-verify-gpu job + ("mochitest-chrome", "gpu", None): "mochitest-chrome-gpu", + ("mochitest-plain", "gpu", None): "mochitest-plain-gpu", + ("mochitest-plain", "webgl1-core", None): "mochitest-webgl1-core", + ("mochitest-plain", "webgl1-ext", None): "mochitest-webgl1-ext", + ("mochitest-plain", "webgl2-core", None): "mochitest-webgl2-core", + ("mochitest-plain", "webgl2-ext", None): "mochitest-webgl2-ext", + ("mochitest-plain", "webgl2-deqp", None): "mochitest-webgl2-deqp", + ("mochitest-plain", "webgpu", None): "mochitest-webgpu", + } + if entry in subsuite_mapping: + suite = subsuite_mapping[entry] + else: + suite = entry[0] + suite_files = self.suites.get(suite) + if not suite_files: + suite_files = [] + if file not in suite_files: + suite_files.append(file) + self.suites[suite] = suite_files + + def _find_wpt_tests(self, dirs, changed_files): + # Setup sys.path to include all the dependencies required to import + # the web-platform-tests manifest parser. web-platform-tests provides + # the localpaths.py to do the path manipulation, which we load, + # providing the __file__ variable so it can resolve the relative + # paths correctly. + paths_file = os.path.join( + dirs["abs_wpttest_dir"], "tests", "tools", "localpaths.py" + ) + with open(paths_file, "r") as f: + exec(f.read(), {"__file__": paths_file}) + import manifest as wptmanifest + + tests_root = os.path.join(dirs["abs_wpttest_dir"], "tests") + + for extra in ("", "mozilla"): + base_path = os.path.join(dirs["abs_wpttest_dir"], extra) + man_path = os.path.join(base_path, "meta", "MANIFEST.json") + man = wptmanifest.manifest.load(tests_root, man_path) + self.info("Per-test run updated with manifest %s" % man_path) + + repo_tests_path = os.path.join("testing", "web-platform", extra, "tests") + tests_path = os.path.join("tests", "web-platform", extra, "tests") + for (type, path, test) in man: + if type not in ["testharness", "reftest", "wdspec"]: + continue + repo_path = os.path.join(repo_tests_path, path) + # manifest paths use os.sep (like backslash on Windows) but + # automation-relevance uses posixpath.sep + repo_path = repo_path.replace(os.sep, posixpath.sep) + if repo_path in changed_files: + self.info( + "Per-test run found web-platform test '%s', type %s" + % (path, type) + ) + suite_files = self.suites.get(type) + if not suite_files: + suite_files = [] + test_path = os.path.join(tests_path, path) + suite_files.append(test_path) + self.suites[type] = suite_files + self._map_test_path_to_source(test_path, repo_path) + changed_files.remove(repo_path) + + if os.environ.get("MOZHARNESS_TEST_PATHS", None) is not None: + for file in changed_files: + self.info( + "Per-test run could not find requested web-platform test '%s'" + % file + ) + + def find_modified_tests(self): + """ + For each file modified on this push, determine if the modified file + is a test, by searching test manifests. Populate self.suites + with test files, organized by suite. + + This depends on test manifests, so can only run after test zips have + been downloaded and extracted. + """ + repository = os.environ.get("GECKO_HEAD_REPOSITORY") + revision = os.environ.get("GECKO_HEAD_REV") + if not repository or not revision: + self.warning("unable to run tests in per-test mode: no repo or revision!") + self.suites = {} + self.tests_downloaded = True + return + + def get_automationrelevance(): + response = self.load_json_url(url) + return response + + dirs = self.query_abs_dirs() + mozinfo.find_and_update_from_json(dirs["abs_test_install_dir"]) + e10s = self.config.get("e10s", False) + mozinfo.update({"e10s": e10s}) + is_fission = "fission.autostart=true" in self.config.get("extra_prefs", []) + mozinfo.update({"fission": is_fission}) + headless = self.config.get("headless", False) + mozinfo.update({"headless": headless}) + if mozinfo.info["buildapp"] == "mobile/android": + # extra android mozinfo normally comes from device queries, but this + # code may run before the device is ready, so rely on configuration + mozinfo.update( + {"android_version": str(self.config.get("android_version", 24))} + ) + mozinfo.update({"is_emulator": self.config.get("is_emulator", True)}) + mozinfo.update({"verify": True}) + self.info("Per-test run using mozinfo: %s" % str(mozinfo.info)) + + # determine which files were changed on this push + changed_files = set() + url = "%s/json-automationrelevance/%s" % (repository.rstrip("/"), revision) + contents = self.retry(get_automationrelevance, attempts=2, sleeptime=10) + for c in contents["changesets"]: + self.info( + " {cset} {desc}".format( + cset=c["node"][0:12], + desc=c["desc"].splitlines()[0].encode("ascii", "ignore"), + ) + ) + changed_files |= set(c["files"]) + changed_files = list(changed_files) + + # check specified test paths, as from 'mach try ... <path>' + if os.environ.get("MOZHARNESS_TEST_PATHS", None) is not None: + suite_to_paths = json.loads(os.environ["MOZHARNESS_TEST_PATHS"]) + specified_paths = itertools.chain.from_iterable(suite_to_paths.values()) + specified_paths = list(specified_paths) + # filter the list of changed files to those found under the + # specified path(s) + changed_and_specified = set() + for changed in changed_files: + for specified in specified_paths: + if changed.startswith(specified): + changed_and_specified.add(changed) + break + if changed_and_specified: + changed_files = changed_and_specified + else: + # if specified paths do not match changed files, assume the + # specified paths are explicitly requested tests + changed_files = set() + changed_files.update(specified_paths) + self.info("Per-test run found explicit request in MOZHARNESS_TEST_PATHS:") + self.info(str(changed_files)) + + if self.config.get("per_test_category") == "web-platform": + self._find_wpt_tests(dirs, changed_files) + elif self.config.get("gpu_required", False) is not False: + self._find_misc_tests(dirs, changed_files, gpu=True) + else: + self._find_misc_tests(dirs, changed_files) + + # per test mode run specific tests from any given test suite + # _find_*_tests organizes tests to run into suites so we can + # run each suite at a time + + # chunk files + total_tests = sum([len(self.suites[x]) for x in self.suites]) + + if total_tests == 0: + self.warning("No tests to verify.") + self.suites = {} + self.tests_downloaded = True + return + + files_per_chunk = total_tests / float(self.config.get("total_chunks", 1)) + files_per_chunk = int(math.ceil(files_per_chunk)) + + chunk_number = int(self.config.get("this_chunk", 1)) + suites = {} + start = (chunk_number - 1) * files_per_chunk + end = chunk_number * files_per_chunk + current = -1 + for suite in self.suites: + for test in self.suites[suite]: + current += 1 + if current >= start and current < end: + if suite not in suites: + suites[suite] = [] + suites[suite].append(test) + if current >= end: + break + + self.suites = suites + self.tests_downloaded = True + + def query_args(self, suite): + """ + For the specified suite, return an array of command line arguments to + be passed to test harnesses when running in per-test mode. + + Each array element is an array of command line arguments for a modified + test in the suite. + """ + # not in verify or per-test coverage mode: run once, with no additional args + if not self.per_test_coverage and not self.verify_enabled: + return [[]] + + files = [] + jsreftest_extra_dir = os.path.join("js", "src", "tests") + # For some suites, the test path needs to be updated before passing to + # the test harness. + for file in self.suites.get(suite): + if self.config.get("per_test_category") != "web-platform" and suite in [ + "reftest", + "crashtest", + ]: + file = os.path.join(self.reftest_test_dir, file) + elif ( + self.config.get("per_test_category") != "web-platform" + and suite == "jsreftest" + ): + file = os.path.relpath(file, jsreftest_extra_dir) + file = os.path.join(self.jsreftest_test_dir, file) + + if file is None: + continue + + file = file.replace(os.sep, posixpath.sep) + files.append(file) + + self.info("Per-test file(s) for '%s': %s" % (suite, files)) + + args = [] + for file in files: + cur = [] + + cur.extend(self.coverage_args) + cur.extend(self.verify_args) + + cur.append(file) + args.append(cur) + + return args + + def query_per_test_category_suites(self, category, all_suites): + """ + In per-test mode, determine which suites are active, for the given + suite category. + """ + suites = None + if self.verify_enabled or self.per_test_coverage: + if self.config.get("per_test_category") == "web-platform": + suites = list(self.suites) + self.info("Per-test suites: %s" % suites) + elif all_suites and self.tests_downloaded: + suites = dict( + (key, all_suites.get(key)) + for key in self.suites + if key in all_suites.keys() + ) + self.info("Per-test suites: %s" % suites) + else: + # Until test zips are downloaded, manifests are not available, + # so it is not possible to determine which suites are active/ + # required for per-test mode; assume all suites from supported + # suite categories are required. + if category in ["mochitest", "xpcshell", "reftest"]: + suites = all_suites + return suites + + def log_per_test_status(self, test_name, tbpl_status, log_level): + """ + Log status of a single test. This will display in the + Job Details pane in treeherder - a convenient summary of per-test mode. + Special test name formatting is needed because treeherder truncates + lines that are too long, and may remove duplicates after truncation. + """ + max_test_name_len = 40 + if len(test_name) > max_test_name_len: + head = test_name + new = "" + previous = None + max_test_name_len = max_test_name_len - len(".../") + while len(new) < max_test_name_len: + head, tail = os.path.split(head) + previous = new + new = os.path.join(tail, new) + test_name = os.path.join("...", previous or new) + test_name = test_name.rstrip(os.path.sep) + self.log( + "TinderboxPrint: Per-test run of %s<br/>: %s" % (test_name, tbpl_status), + level=log_level, + ) + + def get_indexed_logs(self, dir, test_suite): + """ + Per-test tasks need distinct file names for the raw and errorsummary logs + on each run. + """ + index = "" + if self.verify_enabled or self.per_test_coverage: + index = "-test%d" % self.per_test_log_index + self.per_test_log_index += 1 + raw_log_file = os.path.join(dir, "%s%s_raw.log" % (test_suite, index)) + error_summary_file = os.path.join( + dir, "%s%s_errorsummary.log" % (test_suite, index) + ) + return raw_log_file, error_summary_file diff --git a/testing/mozharness/mozharness/mozilla/testing/raptor.py b/testing/mozharness/mozharness/mozilla/testing/raptor.py new file mode 100644 index 0000000000..ceb97da963 --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py @@ -0,0 +1,1478 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import copy +import glob +import multiprocessing +import os +import pathlib +import re +import subprocess +import sys +import tempfile +from shutil import copyfile, rmtree + +from six import string_types + +import mozharness +from mozharness.base.errors import PythonErrorList +from mozharness.base.log import CRITICAL, DEBUG, ERROR, INFO, OutputParser +from mozharness.base.python import Python3Virtualenv +from mozharness.base.vcs.vcsbase import MercurialScript +from mozharness.mozilla.automation import ( + EXIT_STATUS_DICT, + TBPL_RETRY, + TBPL_SUCCESS, + TBPL_WORST_LEVEL_TUPLE, +) +from mozharness.mozilla.testing.android import AndroidMixin +from mozharness.mozilla.testing.codecoverage import ( + CodeCoverageMixin, + code_coverage_config_options, +) +from mozharness.mozilla.testing.errors import HarnessErrorList, TinderBoxPrintRe +from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options + +scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))) +external_tools_path = os.path.join(scripts_path, "external_tools") +here = os.path.abspath(os.path.dirname(__file__)) + +RaptorErrorList = ( + PythonErrorList + + HarnessErrorList + + [ + {"regex": re.compile(r"""run-as: Package '.*' is unknown"""), "level": DEBUG}, + {"substr": r"""raptorDebug""", "level": DEBUG}, + { + "regex": re.compile(r"""^raptor[a-zA-Z-]*( - )?( )?(?i)error(:)?"""), + "level": ERROR, + }, + { + "regex": re.compile(r"""^raptor[a-zA-Z-]*( - )?( )?(?i)critical(:)?"""), + "level": CRITICAL, + }, + { + "regex": re.compile(r"""No machine_name called '.*' can be found"""), + "level": CRITICAL, + }, + { + "substr": r"""No such file or directory: 'browser_output.txt'""", + "level": CRITICAL, + "explanation": "Most likely the browser failed to launch, or the test otherwise " + "failed to start.", + }, + ] +) + +# When running raptor locally, we can attempt to make use of +# the users locally cached ffmpeg binary from from when the user +# ran `./mach browsertime --setup` +FFMPEG_LOCAL_CACHE = { + "mac": "ffmpeg-macos", + "linux": "ffmpeg-4.4.1-i686-static", + "win": "ffmpeg-4.4.1-full_build", +} + + +class Raptor( + TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin, Python3Virtualenv +): + """ + Install and run Raptor tests + """ + + # Options to Browsertime. Paths are expected to be absolute. + browsertime_options = [ + [ + ["--browsertime-node"], + {"dest": "browsertime_node", "default": None, "help": argparse.SUPPRESS}, + ], + [ + ["--browsertime-browsertimejs"], + { + "dest": "browsertime_browsertimejs", + "default": None, + "help": argparse.SUPPRESS, + }, + ], + [ + ["--browsertime-vismet-script"], + { + "dest": "browsertime_vismet_script", + "default": None, + "help": argparse.SUPPRESS, + }, + ], + [ + ["--browsertime-chromedriver"], + { + "dest": "browsertime_chromedriver", + "default": None, + "help": argparse.SUPPRESS, + }, + ], + [ + ["--browsertime-ffmpeg"], + {"dest": "browsertime_ffmpeg", "default": None, "help": argparse.SUPPRESS}, + ], + [ + ["--browsertime-geckodriver"], + { + "dest": "browsertime_geckodriver", + "default": None, + "help": argparse.SUPPRESS, + }, + ], + [ + ["--browsertime-video"], + { + "dest": "browsertime_video", + "action": "store_true", + "default": False, + "help": argparse.SUPPRESS, + }, + ], + [ + ["--browsertime-visualmetrics"], + { + "dest": "browsertime_visualmetrics", + "action": "store_true", + "default": False, + "help": argparse.SUPPRESS, + }, + ], + [ + ["--browsertime-no-ffwindowrecorder"], + { + "dest": "browsertime_no_ffwindowrecorder", + "action": "store_true", + "default": False, + "help": argparse.SUPPRESS, + }, + ], + [ + ["--browsertime-arg"], + { + "action": "append", + "metavar": "PREF=VALUE", + "dest": "browsertime_user_args", + "default": [], + "help": argparse.SUPPRESS, + }, + ], + [ + ["--browsertime"], + { + "dest": "browsertime", + "action": "store_true", + "default": True, + "help": argparse.SUPPRESS, + }, + ], + ] + + config_options = ( + [ + [ + ["--test"], + {"action": "store", "dest": "test", "help": "Raptor test to run"}, + ], + [ + ["--app"], + { + "default": "firefox", + "choices": [ + "firefox", + "chrome", + "chrome-m", + "chromium", + "fennec", + "geckoview", + "refbrow", + "fenix", + "safari", + "custom-car", + ], + "dest": "app", + "help": "Name of the application we are testing (default: firefox).", + }, + ], + [ + ["--activity"], + { + "dest": "activity", + "help": "The Android activity used to launch the Android app. " + "e.g.: org.mozilla.fenix.browser.BrowserPerformanceTestActivity", + }, + ], + [ + ["--intent"], + { + "dest": "intent", + "help": "Name of the Android intent action used to launch the Android app", + }, + ], + [ + ["--is-release-build"], + { + "action": "store_true", + "dest": "is_release_build", + "help": "Whether the build is a release build which requires work arounds " + "using MOZ_DISABLE_NONLOCAL_CONNECTIONS to support installing unsigned " + "webextensions. Defaults to False.", + }, + ], + [ + ["--add-option"], + { + "action": "extend", + "dest": "raptor_cmd_line_args", + "default": None, + "help": "Extra options to Raptor.", + }, + ], + [ + ["--device-name"], + { + "dest": "device_name", + "default": None, + "help": "Device name of mobile device.", + }, + ], + [ + ["--geckoProfile"], + { + "dest": "gecko_profile", + "action": "store_true", + "default": False, + "help": argparse.SUPPRESS, + }, + ], + [ + ["--geckoProfileInterval"], + { + "dest": "gecko_profile_interval", + "type": "int", + "help": argparse.SUPPRESS, + }, + ], + [ + ["--geckoProfileEntries"], + { + "dest": "gecko_profile_entries", + "type": "int", + "help": argparse.SUPPRESS, + }, + ], + [ + ["--geckoProfileFeatures"], + { + "dest": "gecko_profile_features", + "type": "str", + "help": argparse.SUPPRESS, + }, + ], + [ + ["--gecko-profile"], + { + "dest": "gecko_profile", + "action": "store_true", + "default": False, + "help": "Whether to profile the test run and save the profile results.", + }, + ], + [ + ["--gecko-profile-interval"], + { + "dest": "gecko_profile_interval", + "type": "int", + "help": "The interval between samples taken by the profiler (ms).", + }, + ], + [ + ["--gecko-profile-entries"], + { + "dest": "gecko_profile_entries", + "type": "int", + "help": "How many samples to take with the profiler.", + }, + ], + [ + ["--gecko-profile-threads"], + { + "dest": "gecko_profile_threads", + "type": "str", + "help": "Comma-separated list of threads to sample.", + }, + ], + [ + ["--gecko-profile-features"], + { + "dest": "gecko_profile_features", + "type": "str", + "help": "Features to enable in the profiler.", + }, + ], + [ + ["--extra-profiler-run"], + { + "dest": "extra_profiler_run", + "action": "store_true", + "default": False, + "help": "Run the tests again with profiler enabled after the main run.", + }, + ], + [ + ["--page-cycles"], + { + "dest": "page_cycles", + "type": "int", + "help": ( + "How many times to repeat loading the test page (for page load " + "tests); for benchmark tests this is how many times the benchmark test " + "will be run." + ), + }, + ], + [ + ["--page-timeout"], + { + "dest": "page_timeout", + "type": "int", + "help": "How long to wait (ms) for one page_cycle to complete, before timing out.", # NOQA: E501 + }, + ], + [ + ["--browser-cycles"], + { + "dest": "browser_cycles", + "type": "int", + "help": ( + "The number of times a cold load test is repeated (for cold load tests " + "only, where the browser is shutdown and restarted between test " + "iterations)." + ), + }, + ], + [ + ["--project"], + { + "action": "store", + "dest": "project", + "default": "mozilla-central", + "type": "str", + "help": "Name of the project (try, mozilla-central, etc.)", + }, + ], + [ + ["--test-url-params"], + { + "action": "store", + "dest": "test_url_params", + "help": "Parameters to add to the test_url query string.", + }, + ], + [ + ["--host"], + { + "dest": "host", + "type": "str", + "default": "127.0.0.1", + "help": "Hostname from which to serve urls (default: 127.0.0.1). " + "The value HOST_IP will cause the value of host to be " + "to be loaded from the environment variable HOST_IP.", + }, + ], + [ + ["--power-test"], + { + "dest": "power_test", + "action": "store_true", + "default": False, + "help": ( + "Use Raptor to measure power usage on Android browsers (Geckoview " + "Example, Fenix, Refbrow, and Fennec) as well as on Intel-based MacOS " + "machines that have Intel Power Gadget installed." + ), + }, + ], + [ + ["--memory-test"], + { + "dest": "memory_test", + "action": "store_true", + "default": False, + "help": "Use Raptor to measure memory usage.", + }, + ], + [ + ["--cpu-test"], + { + "dest": "cpu_test", + "action": "store_true", + "default": False, + "help": "Use Raptor to measure CPU usage.", + }, + ], + [ + ["--disable-perf-tuning"], + { + "action": "store_true", + "dest": "disable_perf_tuning", + "default": False, + "help": "Disable performance tuning on android.", + }, + ], + [ + ["--conditioned-profile"], + { + "dest": "conditioned_profile", + "type": "str", + "default": None, + "help": ( + "Name of conditioned profile to use. Prefix with `artifact:` " + "if we should obtain the profile from CI.", + ), + }, + ], + [ + ["--live-sites"], + { + "dest": "live_sites", + "action": "store_true", + "default": False, + "help": "Run tests using live sites instead of recorded sites.", + }, + ], + [ + ["--test-bytecode-cache"], + { + "dest": "test_bytecode_cache", + "action": "store_true", + "default": False, + "help": ( + "If set, the pageload test will set the preference " + "`dom.script_loader.bytecode_cache.strategy=-1` and wait 20 seconds " + "after the first cold pageload to populate the bytecode cache before " + "running a warm pageload test. Only available if `--chimera` " + "is also provided." + ), + }, + ], + [ + ["--chimera"], + { + "dest": "chimera", + "action": "store_true", + "default": False, + "help": "Run tests in chimera mode. Each browser cycle will run a cold and warm test.", # NOQA: E501 + }, + ], + [ + ["--debug-mode"], + { + "dest": "debug_mode", + "action": "store_true", + "default": False, + "help": "Run Raptor in debug mode (open browser console, limited page-cycles, etc.)", # NOQA: E501 + }, + ], + [ + ["--noinstall"], + { + "dest": "noinstall", + "action": "store_true", + "default": False, + "help": "Do not offer to install Android APK.", + }, + ], + [ + ["--disable-e10s"], + { + "dest": "e10s", + "action": "store_false", + "default": True, + "help": "Run without multiple processes (e10s).", + }, + ], + [ + ["--disable-fission"], + { + "action": "store_false", + "dest": "fission", + "default": True, + "help": "Disable Fission (site isolation) in Gecko.", + }, + ], + [ + ["--setpref"], + { + "action": "append", + "metavar": "PREF=VALUE", + "dest": "extra_prefs", + "default": [], + "help": "Set a browser preference. May be used multiple times.", + }, + ], + [ + ["--setenv"], + { + "action": "append", + "metavar": "NAME=VALUE", + "dest": "environment", + "default": [], + "help": "Set a variable in the test environment. May be used multiple times.", + }, + ], + [ + ["--skip-preflight"], + { + "action": "store_true", + "dest": "skip_preflight", + "default": False, + "help": "skip preflight commands to prepare machine.", + }, + ], + [ + ["--cold"], + { + "action": "store_true", + "dest": "cold", + "default": False, + "help": "Enable cold page-load for browsertime tp6", + }, + ], + [ + ["--verbose"], + { + "action": "store_true", + "dest": "verbose", + "default": False, + "help": "Verbose output", + }, + ], + [ + ["--enable-marionette-trace"], + { + "action": "store_true", + "dest": "enable_marionette_trace", + "default": False, + "help": "Enable marionette tracing", + }, + ], + [ + ["--clean"], + { + "action": "store_true", + "dest": "clean", + "default": False, + "help": ( + "Clean the python virtualenv (remove, and rebuild) for " + "Raptor before running tests." + ), + }, + ], + [ + ["--webext"], + { + "action": "store_true", + "dest": "webext", + "default": False, + "help": ( + "Whether to use webextension to execute pageload tests " + "(WebExtension is being deprecated).", + ), + }, + ], + [ + ["--collect-perfstats"], + { + "action": "store_true", + "dest": "collect_perfstats", + "default": False, + "help": ( + "If set, the test will collect perfstats in addition to " + "the regular metrics it gathers." + ), + }, + ], + [ + ["--extra-summary-methods"], + { + "action": "append", + "metavar": "OPTION", + "dest": "extra_summary_methods", + "default": [], + "help": ( + "Alternative methods for summarizing technical and visual" + "pageload metrics." + "Options: geomean, mean." + ), + }, + ], + [ + ["--benchmark-repository"], + { + "dest": "benchmark_repository", + "type": "str", + "default": None, + "help": ( + "Repository that should be used for a particular benchmark test. " + "e.g. https://github.com/mozilla-mobile/firefox-android" + ), + }, + ], + [ + ["--benchmark-revision"], + { + "dest": "benchmark_revision", + "type": "str", + "default": None, + "help": ( + "Repository revision that should be used for a particular " + "benchmark test." + ), + }, + ], + [ + ["--benchmark-branch"], + { + "dest": "benchmark_branch", + "type": "str", + "default": None, + "help": ( + "Repository branch that should be used for a particular benchmark test." + ), + }, + ], + ] + + testing_config_options + + copy.deepcopy(code_coverage_config_options) + + browsertime_options + ) + + def __init__(self, **kwargs): + kwargs.setdefault("config_options", self.config_options) + kwargs.setdefault( + "all_actions", + [ + "clobber", + "download-and-extract", + "populate-webroot", + "create-virtualenv", + "install-chrome-android", + "install-chromium-distribution", + "install", + "run-tests", + ], + ) + kwargs.setdefault( + "default_actions", + [ + "clobber", + "download-and-extract", + "populate-webroot", + "create-virtualenv", + "install-chromium-distribution", + "install", + "run-tests", + ], + ) + kwargs.setdefault("config", {}) + super(Raptor, self).__init__(**kwargs) + + # Convenience + self.workdir = self.query_abs_dirs()["abs_work_dir"] + + self.run_local = self.config.get("run_local") + + # App (browser testing on) defaults to firefox + self.app = "firefox" + + if self.run_local: + # Get app from command-line args, passed in from mach, inside 'raptor_cmd_line_args' + # Command-line args can be in two formats depending on how the user entered them + # i.e. "--app=geckoview" or separate as "--app", "geckoview" so we have to + # parse carefully. It's simplest to use `argparse` to parse partially. + self.app = "firefox" + if "raptor_cmd_line_args" in self.config: + sub_parser = argparse.ArgumentParser() + # It's not necessary to limit the allowed values: each value + # will be parsed and verifed by raptor/raptor.py. + sub_parser.add_argument("--app", default=None, dest="app") + sub_parser.add_argument("-i", "--intent", default=None, dest="intent") + sub_parser.add_argument( + "-a", "--activity", default=None, dest="activity" + ) + + # We'd prefer to use `parse_known_intermixed_args`, but that's + # new in Python 3.7. + known, unknown = sub_parser.parse_known_args( + self.config["raptor_cmd_line_args"] + ) + + if known.app: + self.app = known.app + if known.intent: + self.intent = known.intent + if known.activity: + self.activity = known.activity + else: + # Raptor initiated in production via mozharness + self.test = self.config["test"] + self.app = self.config.get("app", "firefox") + self.binary_path = self.config.get("binary_path", None) + + if self.app in ("refbrow", "fenix"): + self.app_name = self.binary_path + + self.installer_url = self.config.get("installer_url") + self.raptor_json_url = self.config.get("raptor_json_url") + self.raptor_json = self.config.get("raptor_json") + self.raptor_json_config = self.config.get("raptor_json_config") + self.repo_path = self.config.get("repo_path") + self.obj_path = self.config.get("obj_path") + self.mozbuild_path = self.config.get("mozbuild_path") + self.test = None + self.gecko_profile = self.config.get( + "gecko_profile" + ) or "--geckoProfile" in self.config.get("raptor_cmd_line_args", []) + self.gecko_profile_interval = self.config.get("gecko_profile_interval") + self.gecko_profile_entries = self.config.get("gecko_profile_entries") + self.gecko_profile_threads = self.config.get("gecko_profile_threads") + self.gecko_profile_features = self.config.get("gecko_profile_features") + self.extra_profiler_run = self.config.get("extra_profiler_run") + self.test_packages_url = self.config.get("test_packages_url") + self.test_url_params = self.config.get("test_url_params") + self.host = self.config.get("host") + if self.host == "HOST_IP": + self.host = os.environ["HOST_IP"] + self.power_test = self.config.get("power_test") + self.memory_test = self.config.get("memory_test") + self.cpu_test = self.config.get("cpu_test") + self.live_sites = self.config.get("live_sites") + self.chimera = self.config.get("chimera") + self.disable_perf_tuning = self.config.get("disable_perf_tuning") + self.conditioned_profile = self.config.get("conditioned_profile") + self.extra_prefs = self.config.get("extra_prefs") + self.environment = self.config.get("environment") + self.is_release_build = self.config.get("is_release_build") + self.debug_mode = self.config.get("debug_mode", False) + self.chromium_dist_path = None + self.firefox_android_browsers = ["fennec", "geckoview", "refbrow", "fenix"] + self.android_browsers = self.firefox_android_browsers + ["chrome-m"] + self.browsertime_visualmetrics = self.config.get("browsertime_visualmetrics") + self.browsertime_node = self.config.get("browsertime_node") + self.browsertime_user_args = self.config.get("browsertime_user_args") + self.browsertime_video = False + self.enable_marionette_trace = self.config.get("enable_marionette_trace") + self.browser_cycles = self.config.get("browser_cycles") + self.clean = self.config.get("clean") + + for (arg,), details in Raptor.browsertime_options: + # Allow overriding defaults on the `./mach raptor-test ...` command-line. + value = self.config.get(details["dest"]) + if value and arg not in self.config.get("raptor_cmd_line_args", []): + setattr(self, details["dest"], value) + + # We accept some configuration options from the try commit message in the + # format mozharness: <options>. Example try commit message: mozharness: + # --geckoProfile try: <stuff> + def query_gecko_profile_options(self): + gecko_results = [] + # If gecko_profile is set, we add that to Raptor's options + if self.gecko_profile: + gecko_results.append("--gecko-profile") + if self.gecko_profile_interval: + gecko_results.extend( + ["--gecko-profile-interval", str(self.gecko_profile_interval)] + ) + if self.gecko_profile_entries: + gecko_results.extend( + ["--gecko-profile-entries", str(self.gecko_profile_entries)] + ) + if self.gecko_profile_features: + gecko_results.extend( + ["--gecko-profile-features", self.gecko_profile_features] + ) + if self.gecko_profile_threads: + gecko_results.extend( + ["--gecko-profile-threads", self.gecko_profile_threads] + ) + else: + if self.extra_profiler_run: + gecko_results.append("--extra-profiler-run") + return gecko_results + + def query_abs_dirs(self): + if self.abs_dirs: + return self.abs_dirs + abs_dirs = super(Raptor, self).query_abs_dirs() + abs_dirs["abs_blob_upload_dir"] = os.path.join( + abs_dirs["abs_work_dir"], "blobber_upload_dir" + ) + abs_dirs["abs_test_install_dir"] = os.path.join( + abs_dirs["abs_work_dir"], "tests" + ) + + self.abs_dirs = abs_dirs + return self.abs_dirs + + def install_chrome_android(self): + """Install Google Chrome for Android in production from tooltool""" + if self.app != "chrome-m": + self.info("Google Chrome for Android not required") + return + if self.config.get("run_local"): + self.info( + "Google Chrome for Android will not be installed " + "from tooltool when running locally" + ) + return + self.info("Fetching and installing Google Chrome for Android") + self.device.shell_output("cmd package install-existing com.android.chrome") + self.info("Google Chrome for Android successfully installed") + + def download_chrome_android(self): + # Fetch the APK + tmpdir = tempfile.mkdtemp() + self.tooltool_fetch( + os.path.join( + self.raptor_path, + "raptor", + "tooltool-manifests", + "chrome-android", + "chrome87.manifest", + ), + output_dir=tmpdir, + ) + files = os.listdir(tmpdir) + if len(files) > 1: + raise Exception( + "Found more than one chrome APK file after tooltool download" + ) + chromeapk = os.path.join(tmpdir, files[0]) + + # Disable verification and install the APK + self.device.shell_output("settings put global verifier_verify_adb_installs 0") + self.install_android_app(chromeapk, replace=True) + + # Re-enable verification and delete the temporary directory + self.device.shell_output("settings put global verifier_verify_adb_installs 1") + rmtree(tmpdir) + + def install_chromium_distribution(self): + """Install Google Chromium distribution in production""" + linux, mac, win = "linux", "mac", "win" + chrome, chromium, chromium_release = "chrome", "chromium", "custom-car" + + available_chromium_dists = [chrome, chromium, chromium_release] + binary_location = { + chromium: { + linux: ["chrome-linux", "chrome"], + mac: ["chrome-mac", "Chromium.app", "Contents", "MacOS", "Chromium"], + win: ["chrome-win", "Chrome.exe"], + }, + chromium_release: { + linux: ["chromium", "Default", "chrome"], + win: ["chromium", "Default", "chrome.exe"], + }, + } + + if self.app not in available_chromium_dists: + self.info("Google Chrome or Chromium distributions are not required.") + return + + if self.app == "chrome": + self.info("Chrome should be preinstalled.") + if win in self.platform_name(): + base_path = "C:\\%s\\Google\\Chrome\\Application\\chrome.exe" + self.chromium_dist_path = base_path % "Progra~1" + if not os.path.exists(self.chromium_dist_path): + self.chromium_dist_path = base_path % "Progra~2" + elif linux in self.platform_name(): + self.chromium_dist_path = "/usr/bin/google-chrome" + elif mac in self.platform_name(): + self.chromium_dist_path = ( + "/Applications/Google Chrome.app/" "Contents/MacOS/Google Chrome" + ) + else: + self.error( + "Chrome is not installed on the platform %s yet." + % self.platform_name() + ) + + if os.path.exists(self.chromium_dist_path): + self.info( + "Google Chrome found in expected location %s" + % self.chromium_dist_path + ) + else: + self.error("Cannot find Google Chrome at %s" % self.chromium_dist_path) + + return + + chromium_dist = self.app + + if self.config.get("run_local"): + self.info("Expecting %s to be pre-installed locally" % chromium_dist) + return + + self.info("Getting fetched %s build" % chromium_dist) + self.chromium_dist_dest = os.path.normpath( + os.path.abspath(os.environ["MOZ_FETCHES_DIR"]) + ) + + if mac in self.platform_name(): + self.chromium_dist_path = os.path.join( + self.chromium_dist_dest, *binary_location[chromium_dist][mac] + ) + + elif linux in self.platform_name(): + self.chromium_dist_path = os.path.join( + self.chromium_dist_dest, *binary_location[chromium_dist][linux] + ) + + else: + self.chromium_dist_path = os.path.join( + self.chromium_dist_dest, *binary_location[chromium_dist][win] + ) + + self.info("%s dest is: %s" % (chromium_dist, self.chromium_dist_dest)) + self.info("%s path is: %s" % (chromium_dist, self.chromium_dist_path)) + + # Now ensure Chromium binary exists + if os.path.exists(self.chromium_dist_path): + self.info( + "Successfully installed %s to: %s" + % (chromium_dist, self.chromium_dist_path) + ) + else: + self.info("Abort: failed to install %s" % chromium_dist) + + def raptor_options(self, args=None, **kw): + """Return options to Raptor""" + options = [] + kw_options = {} + + # Get the APK location to be able to get the browser version + # through mozversion + if self.app in self.firefox_android_browsers and not self.run_local: + kw_options["installerpath"] = self.installer_path + + # If testing on Firefox, the binary path already came from mozharness/pro; + # otherwise the binary path is forwarded from command-line arg (raptor_cmd_line_args). + kw_options["app"] = self.app + if self.app == "firefox" or ( + self.app in self.firefox_android_browsers and not self.run_local + ): + binary_path = self.binary_path or self.config.get("binary_path") + if not binary_path: + self.fatal("Raptor requires a path to the binary.") + kw_options["binary"] = binary_path + if self.app in self.firefox_android_browsers: + # In production ensure we have correct app name, + # i.e. fennec_aurora or fennec_release etc. + kw_options["binary"] = self.query_package_name() + self.info( + "Set binary to %s instead of %s" + % (kw_options["binary"], binary_path) + ) + elif self.app == "safari" and not self.run_local: + binary_path = "/Applications/Safari.app/Contents/MacOS/Safari" + kw_options["binary"] = binary_path + else: # Running on Chromium + if not self.run_local: + # When running locally we already set the Chromium binary above, in init. + # In production, we already installed Chromium, so set the binary path + # to our install. + kw_options["binary"] = self.chromium_dist_path or "" + + # Options overwritten from **kw + if "test" in self.config: + kw_options["test"] = self.config["test"] + if "binary" in self.config: + kw_options["binary"] = self.config["binary"] + if self.symbols_path: + kw_options["symbolsPath"] = self.symbols_path + if self.config.get("obj_path", None) is not None: + kw_options["obj-path"] = self.config["obj_path"] + if self.config.get("mozbuild_path", None) is not None: + kw_options["mozbuild-path"] = self.config["mozbuild_path"] + if self.test_url_params: + kw_options["test-url-params"] = self.test_url_params + if self.config.get("device_name") is not None: + kw_options["device-name"] = self.config["device_name"] + if self.config.get("activity") is not None: + kw_options["activity"] = self.config["activity"] + if self.config.get("conditioned_profile") is not None: + kw_options["conditioned-profile"] = self.config["conditioned_profile"] + if self.config.get("benchmark_repository"): + kw_options["benchmark_repository"] = self.config["benchmark_repository"] + if self.config.get("benchmark_revision"): + kw_options["benchmark_revision"] = self.config["benchmark_revision"] + if self.config.get("benchmark_repository"): + kw_options["benchmark_branch"] = self.config["benchmark_branch"] + + kw_options.update(kw) + if self.host: + kw_options["host"] = self.host + # Configure profiling options + options.extend(self.query_gecko_profile_options()) + # Extra arguments + if args is not None: + options += args + if os.getenv("PERF_FLAGS"): + for option in os.getenv("PERF_FLAGS").split(): + if "=" in option: + kw_option, value = option.split("=") + kw_options[kw_option] = value + else: + options.extend(["--" + option]) + + if self.config.get("run_local", False): + options.extend(["--run-local"]) + if "raptor_cmd_line_args" in self.config: + options += self.config["raptor_cmd_line_args"] + if self.config.get("code_coverage", False): + options.extend(["--code-coverage"]) + if self.config.get("is_release_build", False): + options.extend(["--is-release-build"]) + if self.config.get("power_test", False): + options.extend(["--power-test"]) + if self.config.get("memory_test", False): + options.extend(["--memory-test"]) + if self.config.get("cpu_test", False): + options.extend(["--cpu-test"]) + if self.config.get("live_sites", False): + options.extend(["--live-sites"]) + if self.config.get("chimera", False): + options.extend(["--chimera"]) + if self.config.get("disable_perf_tuning", False): + options.extend(["--disable-perf-tuning"]) + if self.config.get("cold", False): + options.extend(["--cold"]) + if not self.config.get("fission", True): + options.extend(["--disable-fission"]) + if self.config.get("verbose", False): + options.extend(["--verbose"]) + if self.config.get("extra_prefs"): + options.extend( + ["--setpref={}".format(i) for i in self.config.get("extra_prefs")] + ) + if self.config.get("environment"): + options.extend( + ["--setenv={}".format(i) for i in self.config.get("environment")] + ) + if self.config.get("enable_marionette_trace", False): + options.extend(["--enable-marionette-trace"]) + if self.config.get("browser_cycles"): + options.extend( + ["--browser-cycles={}".format(self.config.get("browser_cycles"))] + ) + if self.config.get("test_bytecode_cache", False): + options.extend(["--test-bytecode-cache"]) + if self.config.get("collect_perfstats", False): + options.extend(["--collect-perfstats"]) + if self.config.get("extra_summary_methods"): + options.extend( + [ + "--extra-summary-methods={}".format(method) + for method in self.config.get("extra_summary_methods") + ] + ) + if self.config.get("webext", False): + options.extend(["--webext"]) + else: + for (arg,), details in Raptor.browsertime_options: + # Allow overriding defaults on the `./mach raptor-test ...` command-line + value = self.config.get(details["dest"]) + if value is None or value != getattr(self, details["dest"], None): + # Check for modifications done to the instance variables + value = getattr(self, details["dest"], None) + if value and arg not in self.config.get("raptor_cmd_line_args", []): + if isinstance(value, string_types): + options.extend([arg, os.path.expandvars(value)]) + elif isinstance(value, (tuple, list)): + for val in value: + options.extend([arg, val]) + else: + options.extend([arg]) + + for key, value in kw_options.items(): + options.extend(["--%s" % key, value]) + + return options + + def populate_webroot(self): + """Populate the production test machines' webroots""" + self.raptor_path = os.path.join( + self.query_abs_dirs()["abs_test_install_dir"], "raptor" + ) + if self.config.get("run_local"): + self.raptor_path = os.path.join(self.repo_path, "testing", "raptor") + + def clobber(self): + # Recreate the upload directory for storing the logcat collected + # during APK installation. + super(Raptor, self).clobber() + upload_dir = self.query_abs_dirs()["abs_blob_upload_dir"] + if not os.path.isdir(upload_dir): + self.mkdir_p(upload_dir) + + def install_android_app(self, apk, replace=False): + # Override AndroidMixin's install_android_app in order to capture + # logcat during the installation. If the installation fails, + # the logcat file will be left in the upload directory. + self.logcat_start() + try: + super(Raptor, self).install_android_app(apk, replace=replace) + finally: + self.logcat_stop() + + def download_and_extract(self, extract_dirs=None, suite_categories=None): + # Use in-tree wptserve for Python 3.10 compatibility + extract_dirs = [ + "tools/wptserve/*", + "tools/wpt_third_party/pywebsocket3/*", + ] + return super(Raptor, self).download_and_extract( + extract_dirs=extract_dirs, suite_categories=["common", "condprof", "raptor"] + ) + + def create_virtualenv(self, **kwargs): + """VirtualenvMixin.create_virtualenv() assumes we're using + self.config['virtualenv_modules']. Since we're installing + raptor from its source, we have to wrap that method here.""" + # If virtualenv already exists, just add to path and don't re-install. + # We need it in-path to import jsonschema later when validating output for perfherder. + _virtualenv_path = self.config.get("virtualenv_path") + + if self.clean: + rmtree(_virtualenv_path, ignore_errors=True) + + _python_interp = self.query_exe("python") + if "win" in self.platform_name() and os.path.exists(_python_interp): + multiprocessing.set_executable(_python_interp) + + if self.run_local and os.path.exists(_virtualenv_path): + self.info("Virtualenv already exists, skipping creation") + # ffmpeg exists outside of this virtual environment so + # we re-add it to the platform environment on repeated + # local runs of browsertime visual metric tests + self.setup_local_ffmpeg() + + if "win" in self.platform_name(): + _path = os.path.join(_virtualenv_path, "Lib", "site-packages") + else: + _path = os.path.join( + _virtualenv_path, + "lib", + os.path.basename(_python_interp), + "site-packages", + ) + + sys.path.append(_path) + return + + # virtualenv doesn't already exist so create it + # Install mozbase first, so we use in-tree versions + # Additionally, decide where to pull raptor requirements from. + if not self.run_local: + mozbase_requirements = os.path.join( + self.query_abs_dirs()["abs_test_install_dir"], + "config", + "mozbase_requirements.txt", + ) + raptor_requirements = os.path.join(self.raptor_path, "requirements.txt") + else: + mozbase_requirements = os.path.join( + os.path.dirname(self.raptor_path), + "config", + "mozbase_source_requirements.txt", + ) + raptor_requirements = os.path.join( + self.raptor_path, "source_requirements.txt" + ) + self.register_virtualenv_module( + requirements=[mozbase_requirements], + two_pass=True, + editable=True, + ) + + modules = ["pip>=1.5"] + + # Add modules required for visual metrics + py3_minor = sys.version_info.minor + if py3_minor <= 7: + modules.extend( + [ + "numpy==1.16.1", + "Pillow==6.1.0", + "scipy==1.2.3", + "pyssim==0.4", + "opencv-python==4.5.4.60", + ] + ) + else: # python version >= 3.8 + modules.extend( + [ + "numpy==1.22.0", + "Pillow==9.0.0", + "scipy==1.7.3", + "pyssim==0.4", + "opencv-python==4.5.4.60", + ] + ) + + if self.run_local: + self.setup_local_ffmpeg() + + # Require pip >= 1.5 so pip will prefer .whl files to install + super(Raptor, self).create_virtualenv(modules=modules) + + # Install Raptor dependencies + self.install_module(requirements=[raptor_requirements]) + + def setup_local_ffmpeg(self): + """Make use of the users local ffmpeg when running browsertime visual + metrics tests. + """ + + if "ffmpeg" in os.environ["PATH"]: + return + + platform = self.platform_name() + btime_cache = os.path.join(self.config["mozbuild_path"], "browsertime") + if "mac" in platform: + path_to_ffmpeg = os.path.join( + btime_cache, + FFMPEG_LOCAL_CACHE["mac"], + ) + elif "linux" in platform: + path_to_ffmpeg = os.path.join( + btime_cache, + FFMPEG_LOCAL_CACHE["linux"], + ) + elif "win" in platform: + path_to_ffmpeg = os.path.join( + btime_cache, + FFMPEG_LOCAL_CACHE["win"], + "bin", + ) + + if os.path.exists(path_to_ffmpeg): + os.environ["PATH"] += os.pathsep + path_to_ffmpeg + self.browsertime_ffmpeg = path_to_ffmpeg + self.info( + "Added local ffmpeg found at: %s to environment." % path_to_ffmpeg + ) + else: + raise Exception( + "No local ffmpeg binary found. Expected it to be here: %s" + % path_to_ffmpeg + ) + + def install(self): + if not self.config.get("noinstall", False): + if self.app in self.firefox_android_browsers: + self.device.uninstall_app(self.binary_path) + + # Check if the user supplied their own APK, and install + # that instead + installer_path = pathlib.Path( + self.raptor_path, "raptor", "user_upload.apk" + ) + if not installer_path.exists(): + installer_path = self.installer_path + + self.info(f"Installing APK from: {installer_path}") + self.install_android_app(str(installer_path)) + else: + super(Raptor, self).install() + + def _artifact_perf_data(self, src, dest): + if not os.path.isdir(os.path.dirname(dest)): + # create upload dir if it doesn't already exist + self.info("Creating dir: %s" % os.path.dirname(dest)) + os.makedirs(os.path.dirname(dest)) + self.info("Copying raptor results from %s to %s" % (src, dest)) + try: + copyfile(src, dest) + except Exception as e: + self.critical("Error copying results %s to upload dir %s" % (src, dest)) + self.info(str(e)) + + def run_tests(self, args=None, **kw): + """Run raptor tests""" + + # Get Raptor options + options = self.raptor_options(args=args, **kw) + + # Python version check + python = self.query_python_path() + self.run_command([python, "--version"]) + parser = RaptorOutputParser( + config=self.config, log_obj=self.log_obj, error_list=RaptorErrorList + ) + env = {} + env["MOZ_UPLOAD_DIR"] = self.query_abs_dirs()["abs_blob_upload_dir"] + if not self.run_local: + env["MINIDUMP_STACKWALK"] = self.query_minidump_stackwalk() + env["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs()["abs_blob_upload_dir"] + env["RUST_BACKTRACE"] = "full" + if not os.path.isdir(env["MOZ_UPLOAD_DIR"]): + self.mkdir_p(env["MOZ_UPLOAD_DIR"]) + env = self.query_env(partial_env=env, log_level=INFO) + # adjust PYTHONPATH to be able to use raptor as a python package + if "PYTHONPATH" in env: + env["PYTHONPATH"] = self.raptor_path + os.pathsep + env["PYTHONPATH"] + else: + env["PYTHONPATH"] = self.raptor_path + + # mitmproxy needs path to mozharness when installing the cert, and tooltool + env["SCRIPTSPATH"] = scripts_path + env["EXTERNALTOOLSPATH"] = external_tools_path + + # Needed to load unsigned Raptor WebExt on release builds + if self.is_release_build: + env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" + + if self.repo_path is not None: + env["MOZ_DEVELOPER_REPO_DIR"] = self.repo_path + if self.obj_path is not None: + env["MOZ_DEVELOPER_OBJ_DIR"] = self.obj_path + if self.mozbuild_path is not None: + env["MOZ_MOZBUILD_DIR"] = self.mozbuild_path + + # Sets a timeout for how long Raptor should run without output + output_timeout = self.config.get("raptor_output_timeout", 3600) + # Run Raptor tests + run_tests = os.path.join(self.raptor_path, "raptor", "raptor.py") + + # Dynamically set the log level based on the raptor config for consistency + # throughout the test + mozlog_opts = [f"--log-tbpl-level={self.config['log_level']}"] + + if not self.run_local and "suite" in self.config: + fname_pattern = "%s_%%s.log" % self.config["test"] + mozlog_opts.append( + "--log-errorsummary=%s" + % os.path.join(env["MOZ_UPLOAD_DIR"], fname_pattern % "errorsummary") + ) + + def launch_in_debug_mode(cmdline): + cmdline = set(cmdline) + debug_opts = {"--debug", "--debugger", "--debugger_args"} + + return bool(debug_opts.intersection(cmdline)) + + if self.app in self.android_browsers: + self.logcat_start() + + command = [python, run_tests] + options + mozlog_opts + if launch_in_debug_mode(command): + raptor_process = subprocess.Popen(command, cwd=self.workdir, env=env) + raptor_process.wait() + else: + self.return_code = self.run_command( + command, + cwd=self.workdir, + output_timeout=output_timeout, + output_parser=parser, + env=env, + ) + + if self.app in self.android_browsers: + self.logcat_stop() + + if parser.minidump_output: + self.info("Looking at the minidump files for debugging purposes...") + for item in parser.minidump_output: + self.run_command(["ls", "-l", item]) + + elif not self.run_local: + # Copy results to upload dir so they are included as an artifact + self.info("Copying Raptor results to upload dir:") + + src = os.path.join(self.query_abs_dirs()["abs_work_dir"], "raptor.json") + dest = os.path.join(env["MOZ_UPLOAD_DIR"], "perfherder-data.json") + self.info(str(dest)) + self._artifact_perf_data(src, dest) + + # Make individual perfherder data JSON's for each supporting data type + for file in glob.glob( + os.path.join(self.query_abs_dirs()["abs_work_dir"], "*") + ): + path, filename = os.path.split(file) + + if not filename.startswith("raptor-"): + continue + + # filename is expected to contain a unique data name + # i.e. raptor-os-baseline-power.json would result in + # the data name os-baseline-power + data_name = "-".join(filename.split("-")[1:]) + data_name = ".".join(data_name.split(".")[:-1]) + + src = file + dest = os.path.join( + env["MOZ_UPLOAD_DIR"], "perfherder-data-%s.json" % data_name + ) + self._artifact_perf_data(src, dest) + + src = os.path.join( + self.query_abs_dirs()["abs_work_dir"], "screenshots.html" + ) + if os.path.exists(src): + dest = os.path.join(env["MOZ_UPLOAD_DIR"], "screenshots.html") + self.info(str(dest)) + self._artifact_perf_data(src, dest) + + # Allow log failures to over-ride successful runs of the test harness and + # give log failures priority, so that, for instance, log failures resulting + # in TBPL_RETRY cause a retry rather than simply reporting an error. + if parser.tbpl_status != TBPL_SUCCESS: + parser_status = EXIT_STATUS_DICT[parser.tbpl_status] + self.info( + "return code %s changed to %s due to log output" + % (str(self.return_code), str(parser_status)) + ) + self.return_code = parser_status + + +class RaptorOutputParser(OutputParser): + minidump_regex = re.compile( + r'''raptorError: "error executing: '(\S+) (\S+) (\S+)'"''' + ) + RE_PERF_DATA = re.compile(r".*PERFHERDER_DATA:\s+(\{.*\})") + + def __init__(self, **kwargs): + super(RaptorOutputParser, self).__init__(**kwargs) + self.minidump_output = None + self.found_perf_data = [] + self.tbpl_status = TBPL_SUCCESS + self.worst_log_level = INFO + self.harness_retry_re = TinderBoxPrintRe["harness_error"]["retry_regex"] + + def parse_single_line(self, line): + m = self.minidump_regex.search(line) + if m: + self.minidump_output = (m.group(1), m.group(2), m.group(3)) + + m = self.RE_PERF_DATA.match(line) + if m: + self.found_perf_data.append(m.group(1)) + + if self.harness_retry_re.search(line): + self.critical(" %s" % line) + self.worst_log_level = self.worst_level(CRITICAL, self.worst_log_level) + self.tbpl_status = self.worst_level( + TBPL_RETRY, self.tbpl_status, levels=TBPL_WORST_LEVEL_TUPLE + ) + return # skip base parse_single_line + super(RaptorOutputParser, self).parse_single_line(line) diff --git a/testing/mozharness/mozharness/mozilla/testing/talos.py b/testing/mozharness/mozharness/mozilla/testing/talos.py new file mode 100755 index 0000000000..b6827cd3d2 --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/testing/talos.py @@ -0,0 +1,893 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +""" +run talos tests in a virtualenv +""" + +import copy +import io +import json +import multiprocessing +import os +import pprint +import re +import shutil +import subprocess +import sys + +import six + +import mozharness +from mozharness.base.config import parse_config_file +from mozharness.base.errors import PythonErrorList +from mozharness.base.log import CRITICAL, DEBUG, ERROR, INFO, WARNING, OutputParser +from mozharness.base.python import Python3Virtualenv +from mozharness.base.vcs.vcsbase import MercurialScript +from mozharness.mozilla.automation import ( + TBPL_FAILURE, + TBPL_RETRY, + TBPL_SUCCESS, + TBPL_WARNING, + TBPL_WORST_LEVEL_TUPLE, +) +from mozharness.mozilla.testing.codecoverage import ( + CodeCoverageMixin, + code_coverage_config_options, +) +from mozharness.mozilla.testing.errors import TinderBoxPrintRe +from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options +from mozharness.mozilla.tooltool import TooltoolMixin + +scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))) +external_tools_path = os.path.join(scripts_path, "external_tools") + +TalosErrorList = PythonErrorList + [ + {"regex": re.compile(r"""run-as: Package '.*' is unknown"""), "level": DEBUG}, + {"substr": r"""FAIL: Graph server unreachable""", "level": CRITICAL}, + {"substr": r"""FAIL: Busted:""", "level": CRITICAL}, + {"substr": r"""FAIL: failed to cleanup""", "level": ERROR}, + {"substr": r"""erfConfigurator.py: Unknown error""", "level": CRITICAL}, + {"substr": r"""talosError""", "level": CRITICAL}, + { + "regex": re.compile(r"""No machine_name called '.*' can be found"""), + "level": CRITICAL, + }, + { + "substr": r"""No such file or directory: 'browser_output.txt'""", + "level": CRITICAL, + "explanation": "Most likely the browser failed to launch, or the test was otherwise " + "unsuccessful in even starting.", + }, +] + +GeckoProfilerSettings = ( + "gecko_profile_interval", + "gecko_profile_entries", + "gecko_profile_features", + "gecko_profile_threads", +) + +# TODO: check for running processes on script invocation + + +class TalosOutputParser(OutputParser): + minidump_regex = re.compile( + r'''talosError: "error executing: '(\S+) (\S+) (\S+)'"''' + ) + RE_PERF_DATA = re.compile(r".*PERFHERDER_DATA:\s+(\{.*\})") + worst_tbpl_status = TBPL_SUCCESS + + def __init__(self, **kwargs): + super(TalosOutputParser, self).__init__(**kwargs) + self.minidump_output = None + self.found_perf_data = [] + + def update_worst_log_and_tbpl_levels(self, log_level, tbpl_level): + self.worst_log_level = self.worst_level(log_level, self.worst_log_level) + self.worst_tbpl_status = self.worst_level( + tbpl_level, self.worst_tbpl_status, levels=TBPL_WORST_LEVEL_TUPLE + ) + + def parse_single_line(self, line): + """In Talos land, every line that starts with RETURN: needs to be + printed with a TinderboxPrint:""" + if line.startswith("RETURN:"): + line.replace("RETURN:", "TinderboxPrint:") + m = self.minidump_regex.search(line) + if m: + self.minidump_output = (m.group(1), m.group(2), m.group(3)) + + m = self.RE_PERF_DATA.match(line) + if m: + self.found_perf_data.append(m.group(1)) + + # now let's check if we should retry + harness_retry_re = TinderBoxPrintRe["harness_error"]["retry_regex"] + if harness_retry_re.search(line): + self.critical(" %s" % line) + self.update_worst_log_and_tbpl_levels(CRITICAL, TBPL_RETRY) + return # skip base parse_single_line + super(TalosOutputParser, self).parse_single_line(line) + + +class Talos( + TestingMixin, MercurialScript, TooltoolMixin, Python3Virtualenv, CodeCoverageMixin +): + """ + install and run Talos tests + """ + + config_options = ( + [ + [ + ["--use-talos-json"], + { + "action": "store_true", + "dest": "use_talos_json", + "default": False, + "help": "Use talos config from talos.json", + }, + ], + [ + ["--suite"], + { + "action": "store", + "dest": "suite", + "help": "Talos suite to run (from talos json)", + }, + ], + [ + ["--system-bits"], + { + "action": "store", + "dest": "system_bits", + "type": "choice", + "default": "32", + "choices": ["32", "64"], + "help": "Testing 32 or 64 (for talos json plugins)", + }, + ], + [ + ["--add-option"], + { + "action": "extend", + "dest": "talos_extra_options", + "default": None, + "help": "extra options to talos", + }, + ], + [ + ["--gecko-profile"], + { + "dest": "gecko_profile", + "action": "store_true", + "default": False, + "help": "Whether or not to profile the test run and save the profile results", + }, + ], + [ + ["--gecko-profile-interval"], + { + "dest": "gecko_profile_interval", + "type": "int", + "help": "The interval between samples taken by the profiler (milliseconds)", + }, + ], + [ + ["--gecko-profile-entries"], + { + "dest": "gecko_profile_entries", + "type": "int", + "help": "How many samples to take with the profiler", + }, + ], + [ + ["--gecko-profile-features"], + { + "dest": "gecko_profile_features", + "type": "str", + "default": None, + "help": "The features to enable in the profiler (comma-separated)", + }, + ], + [ + ["--gecko-profile-threads"], + { + "dest": "gecko_profile_threads", + "type": "str", + "help": "Comma-separated list of threads to sample.", + }, + ], + [ + ["--disable-e10s"], + { + "dest": "e10s", + "action": "store_false", + "default": True, + "help": "Run without multiple processes (e10s).", + }, + ], + [ + ["--disable-fission"], + { + "action": "store_false", + "dest": "fission", + "default": True, + "help": "Disable Fission (site isolation) in Gecko.", + }, + ], + [ + ["--project"], + { + "dest": "project", + "type": "str", + "help": "The project branch we're running tests on. Used for " + "disabling/skipping tests.", + }, + ], + [ + ["--setpref"], + { + "action": "append", + "metavar": "PREF=VALUE", + "dest": "extra_prefs", + "default": [], + "help": "Set a browser preference. May be used multiple times.", + }, + ], + [ + ["--skip-preflight"], + { + "action": "store_true", + "dest": "skip_preflight", + "default": False, + "help": "skip preflight commands to prepare machine.", + }, + ], + ] + + testing_config_options + + copy.deepcopy(code_coverage_config_options) + ) + + def __init__(self, **kwargs): + kwargs.setdefault("config_options", self.config_options) + kwargs.setdefault( + "all_actions", + [ + "clobber", + "download-and-extract", + "populate-webroot", + "create-virtualenv", + "install", + "run-tests", + ], + ) + kwargs.setdefault( + "default_actions", + [ + "clobber", + "download-and-extract", + "populate-webroot", + "create-virtualenv", + "install", + "run-tests", + ], + ) + kwargs.setdefault("config", {}) + super(Talos, self).__init__(**kwargs) + + self.workdir = self.query_abs_dirs()["abs_work_dir"] # convenience + + self.run_local = self.config.get("run_local") + self.installer_url = self.config.get("installer_url") + self.test_packages_url = self.config.get("test_packages_url") + self.talos_json_url = self.config.get("talos_json_url") + self.talos_json = self.config.get("talos_json") + self.talos_json_config = self.config.get("talos_json_config") + self.repo_path = self.config.get("repo_path") + self.obj_path = self.config.get("obj_path") + self.tests = None + extra_opts = self.config.get("talos_extra_options", []) + self.gecko_profile = ( + self.config.get("gecko_profile") or "--gecko-profile" in extra_opts + ) + for setting in GeckoProfilerSettings: + value = self.config.get(setting) + arg = "--" + setting.replace("_", "-") + if value is None: + try: + value = extra_opts[extra_opts.index(arg) + 1] + except ValueError: + pass # Not found + if value is not None: + setattr(self, setting, value) + if not self.gecko_profile: + self.warning("enabling Gecko profiler for %s setting!" % setting) + self.gecko_profile = True + self.pagesets_name = None + self.benchmark_zip = None + self.webextensions_zip = None + + # We accept some configuration options from the try commit message in the format + # mozharness: <options> + # Example try commit message: + # mozharness: --gecko-profile try: <stuff> + def query_gecko_profile_options(self): + gecko_results = [] + # finally, if gecko_profile is set, we add that to the talos options + if self.gecko_profile: + gecko_results.append("--gecko-profile") + for setting in GeckoProfilerSettings: + value = getattr(self, setting, None) + if value: + arg = "--" + setting.replace("_", "-") + gecko_results.extend([arg, str(value)]) + return gecko_results + + def query_abs_dirs(self): + if self.abs_dirs: + return self.abs_dirs + abs_dirs = super(Talos, self).query_abs_dirs() + abs_dirs["abs_blob_upload_dir"] = os.path.join( + abs_dirs["abs_work_dir"], "blobber_upload_dir" + ) + abs_dirs["abs_test_install_dir"] = os.path.join( + abs_dirs["abs_work_dir"], "tests" + ) + self.abs_dirs = abs_dirs + return self.abs_dirs + + def query_talos_json_config(self): + """Return the talos json config.""" + if self.talos_json_config: + return self.talos_json_config + if not self.talos_json: + self.talos_json = os.path.join(self.talos_path, "talos.json") + self.talos_json_config = parse_config_file(self.talos_json) + self.info(pprint.pformat(self.talos_json_config)) + return self.talos_json_config + + def make_talos_domain(self, host): + return host + "-talos" + + def split_path(self, path): + result = [] + while True: + path, folder = os.path.split(path) + if folder: + result.append(folder) + continue + elif path: + result.append(path) + break + + result.reverse() + return result + + def merge_paths(self, lhs, rhs): + backtracks = 0 + for subdir in rhs: + if subdir == "..": + backtracks += 1 + else: + break + return lhs[:-backtracks] + rhs[backtracks:] + + def replace_relative_iframe_paths(self, directory, filename): + """This will find iframes with relative paths and replace them with + absolute paths containing domains derived from the original source's + domain. This helps us better simulate real-world cases for fission + """ + if not filename.endswith(".html"): + return + + directory_pieces = self.split_path(directory) + while directory_pieces and directory_pieces[0] != "fis": + directory_pieces = directory_pieces[1:] + path = os.path.join(directory, filename) + + # XXX: ugh, is there a better way to account for multiple encodings than just + # trying each of them? + encodings = ["utf-8", "latin-1"] + iframe_pattern = re.compile(r'(iframe.*")(\.\./.*\.html)"') + for encoding in encodings: + try: + with io.open(path, "r", encoding=encoding) as f: + content = f.read() + + def replace_iframe_src(match): + src = match.group(2) + split = self.split_path(src) + merged = self.merge_paths(directory_pieces, split) + host = merged[3] + site_origin_hash = self.make_talos_domain(host) + new_url = 'http://%s/%s"' % ( + site_origin_hash, + "/".join(merged), # pylint --py3k: W1649 + ) + self.info( + "Replacing %s with %s in iframe inside %s" + % (match.group(2), new_url, path) + ) + return match.group(1) + new_url + + content = re.sub(iframe_pattern, replace_iframe_src, content) + with io.open(path, "w", encoding=encoding) as f: + f.write(content) + break + except UnicodeDecodeError: + pass + + def query_pagesets_name(self): + """Certain suites require external pagesets to be downloaded and + extracted. + """ + if self.pagesets_name: + return self.pagesets_name + if self.query_talos_json_config() and self.suite is not None: + self.pagesets_name = self.talos_json_config["suites"][self.suite].get( + "pagesets_name" + ) + self.pagesets_name_manifest = "tp5n-pageset.manifest" + return self.pagesets_name + + def query_benchmark_zip(self): + """Certain suites require external benchmarks to be downloaded and + extracted. + """ + if self.benchmark_zip: + return self.benchmark_zip + if self.query_talos_json_config() and self.suite is not None: + self.benchmark_zip = self.talos_json_config["suites"][self.suite].get( + "benchmark_zip" + ) + self.benchmark_zip_manifest = "jetstream-benchmark.manifest" + return self.benchmark_zip + + def query_webextensions_zip(self): + """Certain suites require external WebExtension sets to be downloaded and + extracted. + """ + if self.webextensions_zip: + return self.webextensions_zip + if self.query_talos_json_config() and self.suite is not None: + self.webextensions_zip = self.talos_json_config["suites"][self.suite].get( + "webextensions_zip" + ) + self.webextensions_zip_manifest = "webextensions.manifest" + return self.webextensions_zip + + def get_suite_from_test(self): + """Retrieve the talos suite name from a given talos test name.""" + # running locally, single test name provided instead of suite; go through tests and + # find suite name + suite_name = None + if self.query_talos_json_config(): + if "-a" in self.config["talos_extra_options"]: + test_name_index = self.config["talos_extra_options"].index("-a") + 1 + if "--activeTests" in self.config["talos_extra_options"]: + test_name_index = ( + self.config["talos_extra_options"].index("--activeTests") + 1 + ) + if test_name_index < len(self.config["talos_extra_options"]): + test_name = self.config["talos_extra_options"][test_name_index] + for talos_suite in self.talos_json_config["suites"]: + if test_name in self.talos_json_config["suites"][talos_suite].get( + "tests" + ): + suite_name = talos_suite + if not suite_name: + # no suite found to contain the specified test, error out + self.fatal("Test name is missing or invalid") + else: + self.fatal("Talos json config not found, cannot verify suite") + return suite_name + + def query_suite_extra_prefs(self): + if self.query_talos_json_config() and self.suite is not None: + return self.talos_json_config["suites"][self.suite].get("extra_prefs", []) + + return [] + + def validate_suite(self): + """Ensure suite name is a valid talos suite.""" + if self.query_talos_json_config() and self.suite is not None: + if self.suite not in self.talos_json_config.get("suites"): + self.fatal( + "Suite '%s' is not valid (not found in talos json config)" + % self.suite + ) + + def talos_options(self, args=None, **kw): + """return options to talos""" + # binary path + binary_path = self.binary_path or self.config.get("binary_path") + if not binary_path: + msg = """Talos requires a path to the binary. You can specify binary_path or add + download-and-extract to your action list.""" + self.fatal(msg) + + # talos options + options = [] + # talos can't gather data if the process name ends with '.exe' + if binary_path.endswith(".exe"): + binary_path = binary_path[:-4] + # options overwritten from **kw + kw_options = {"executablePath": binary_path} + if "suite" in self.config: + kw_options["suite"] = self.config["suite"] + if self.config.get("title"): + kw_options["title"] = self.config["title"] + if self.symbols_path: + kw_options["symbolsPath"] = self.symbols_path + if self.config.get("project", None): + kw_options["project"] = self.config["project"] + + kw_options.update(kw) + # talos expects tests to be in the format (e.g.) 'ts:tp5:tsvg' + tests = kw_options.get("activeTests") + if tests and not isinstance(tests, six.string_types): + tests = ":".join(tests) # Talos expects this format + kw_options["activeTests"] = tests + for key, value in kw_options.items(): + options.extend(["--%s" % key, value]) + # configure profiling options + options.extend(self.query_gecko_profile_options()) + # extra arguments + if args is not None: + options += args + if "talos_extra_options" in self.config: + options += self.config["talos_extra_options"] + if self.config.get("code_coverage", False): + options.extend(["--code-coverage"]) + + # Add extra_prefs defined by individual test suites in talos.json + extra_prefs = self.query_suite_extra_prefs() + # Add extra_prefs from the configuration + if self.config["extra_prefs"]: + extra_prefs.extend(self.config["extra_prefs"]) + + options.extend(["--setpref={}".format(p) for p in extra_prefs]) + + # disabling fission can come from the --disable-fission cmd line argument; or in CI + # it comes from a taskcluster transform which adds a --setpref for fission.autostart + if (not self.config["fission"]) or "fission.autostart=false" in self.config[ + "extra_prefs" + ]: + options.extend(["--disable-fission"]) + + return options + + def populate_webroot(self): + """Populate the production test machines' webroots""" + self.talos_path = os.path.join( + self.query_abs_dirs()["abs_test_install_dir"], "talos" + ) + + # need to determine if talos pageset is required to be downloaded + if self.config.get("run_local") and "talos_extra_options" in self.config: + # talos initiated locally, get and verify test/suite from cmd line + self.talos_path = os.path.dirname(self.talos_json) + if ( + "-a" in self.config["talos_extra_options"] + or "--activeTests" in self.config["talos_extra_options"] + ): + # test name (-a or --activeTests) specified, find out what suite it is a part of + self.suite = self.get_suite_from_test() + elif "--suite" in self.config["talos_extra_options"]: + # --suite specified, get suite from cmd line and ensure is valid + suite_name_index = ( + self.config["talos_extra_options"].index("--suite") + 1 + ) + if suite_name_index < len(self.config["talos_extra_options"]): + self.suite = self.config["talos_extra_options"][suite_name_index] + self.validate_suite() + else: + self.fatal("Suite name not provided") + else: + # talos initiated in production via mozharness + self.suite = self.config["suite"] + + tooltool_artifacts = [] + src_talos_pageset_dest = os.path.join(self.talos_path, "talos", "tests") + # unfortunately this path has to be short and can't be descriptive, because + # on Windows we tend to already push the boundaries of the max path length + # constraint. This will contain the tp5 pageset, but adjusted to have + # absolute URLs on iframes for the purposes of better modeling things for + # fission. + src_talos_pageset_multidomain_dest = os.path.join( + self.talos_path, "talos", "fis" + ) + webextension_dest = os.path.join(self.talos_path, "talos", "webextensions") + + if self.query_pagesets_name(): + tooltool_artifacts.append( + { + "name": self.pagesets_name, + "manifest": self.pagesets_name_manifest, + "dest": src_talos_pageset_dest, + } + ) + tooltool_artifacts.append( + { + "name": self.pagesets_name, + "manifest": self.pagesets_name_manifest, + "dest": src_talos_pageset_multidomain_dest, + "postprocess": self.replace_relative_iframe_paths, + } + ) + + if self.query_benchmark_zip(): + tooltool_artifacts.append( + { + "name": self.benchmark_zip, + "manifest": self.benchmark_zip_manifest, + "dest": src_talos_pageset_dest, + } + ) + + if self.query_webextensions_zip(): + tooltool_artifacts.append( + { + "name": self.webextensions_zip, + "manifest": self.webextensions_zip_manifest, + "dest": webextension_dest, + } + ) + + # now that have the suite name, check if artifact is required, if so download it + # the --no-download option will override this + for artifact in tooltool_artifacts: + if "--no-download" not in self.config.get("talos_extra_options", []): + self.info("Downloading %s with tooltool..." % artifact) + + archive = os.path.join(artifact["dest"], artifact["name"]) + output_dir_path = re.sub(r"\.zip$", "", archive) + if not os.path.exists(archive): + manifest_file = os.path.join(self.talos_path, artifact["manifest"]) + self.tooltool_fetch( + manifest_file, + output_dir=artifact["dest"], + cache=self.config.get("tooltool_cache"), + ) + unzip = self.query_exe("unzip") + unzip_cmd = [unzip, "-q", "-o", archive, "-d", artifact["dest"]] + self.run_command(unzip_cmd, halt_on_failure=True) + + if "postprocess" in artifact: + for subdir, dirs, files in os.walk(output_dir_path): + for file in files: + artifact["postprocess"](subdir, file) + else: + self.info("%s already available" % artifact) + + else: + self.info( + "Not downloading %s because the no-download option was specified" + % artifact + ) + + # if running webkit tests locally, need to copy webkit source into talos/tests + if self.config.get("run_local") and ( + "stylebench" in self.suite or "motionmark" in self.suite + ): + self.get_webkit_source() + + def get_webkit_source(self): + # in production the build system auto copies webkit source into place; + # but when run locally we need to do this manually, so that talos can find it + src = os.path.join(self.repo_path, "third_party", "webkit", "PerformanceTests") + dest = os.path.join( + self.talos_path, "talos", "tests", "webkit", "PerformanceTests" + ) + + if os.path.exists(dest): + shutil.rmtree(dest) + + self.info("Copying webkit benchmarks from %s to %s" % (src, dest)) + try: + shutil.copytree(src, dest) + except Exception: + self.critical("Error copying webkit benchmarks from %s to %s" % (src, dest)) + + # Action methods. {{{1 + # clobber defined in BaseScript + + def download_and_extract(self, extract_dirs=None, suite_categories=None): + # Use in-tree wptserve for Python 3.10 compatibility + extract_dirs = [ + "tools/wptserve/*", + "tools/wpt_third_party/pywebsocket3/*", + ] + return super(Talos, self).download_and_extract( + extract_dirs=extract_dirs, suite_categories=["common", "talos"] + ) + + def create_virtualenv(self, **kwargs): + """VirtualenvMixin.create_virtualenv() assuemes we're using + self.config['virtualenv_modules']. Since we are installing + talos from its source, we have to wrap that method here.""" + # if virtualenv already exists, just add to path and don't re-install, need it + # in path so can import jsonschema later when validating output for perfherder + _virtualenv_path = self.config.get("virtualenv_path") + + _python_interp = self.query_exe("python") + if "win" in self.platform_name() and os.path.exists(_python_interp): + multiprocessing.set_executable(_python_interp) + + if self.run_local and os.path.exists(_virtualenv_path): + self.info("Virtualenv already exists, skipping creation") + + if "win" in self.platform_name(): + _path = os.path.join(_virtualenv_path, "Lib", "site-packages") + else: + _path = os.path.join( + _virtualenv_path, + "lib", + os.path.basename(_python_interp), + "site-packages", + ) + + sys.path.append(_path) + return + + # virtualenv doesn't already exist so create it + # install mozbase first, so we use in-tree versions + # Additionally, decide where to pull talos requirements from. + if not self.run_local: + mozbase_requirements = os.path.join( + self.query_abs_dirs()["abs_test_install_dir"], + "config", + "mozbase_requirements.txt", + ) + talos_requirements = os.path.join(self.talos_path, "requirements.txt") + else: + mozbase_requirements = os.path.join( + os.path.dirname(self.talos_path), + "config", + "mozbase_source_requirements.txt", + ) + talos_requirements = os.path.join( + self.talos_path, "source_requirements.txt" + ) + self.register_virtualenv_module( + requirements=[mozbase_requirements], + two_pass=True, + editable=True, + ) + super(Talos, self).create_virtualenv() + # talos in harness requires what else is + # listed in talos requirements.txt file. + self.install_module(requirements=[talos_requirements]) + + def _validate_treeherder_data(self, parser): + # late import is required, because install is done in create_virtualenv + import jsonschema + + if len(parser.found_perf_data) != 1: + self.critical( + "PERFHERDER_DATA was seen %d times, expected 1." + % len(parser.found_perf_data) + ) + parser.update_worst_log_and_tbpl_levels(WARNING, TBPL_WARNING) + return + + schema_path = os.path.join( + external_tools_path, "performance-artifact-schema.json" + ) + self.info("Validating PERFHERDER_DATA against %s" % schema_path) + try: + with open(schema_path) as f: + schema = json.load(f) + data = json.loads(parser.found_perf_data[0]) + jsonschema.validate(data, schema) + except Exception: + self.exception("Error while validating PERFHERDER_DATA") + parser.update_worst_log_and_tbpl_levels(WARNING, TBPL_WARNING) + + def _artifact_perf_data(self, parser, dest): + src = os.path.join(self.query_abs_dirs()["abs_work_dir"], "local.json") + try: + shutil.copyfile(src, dest) + except Exception: + self.critical("Error copying results %s to upload dir %s" % (src, dest)) + parser.update_worst_log_and_tbpl_levels(CRITICAL, TBPL_FAILURE) + + def run_tests(self, args=None, **kw): + """run Talos tests""" + + # get talos options + options = self.talos_options(args=args, **kw) + + # XXX temporary python version check + python = self.query_python_path() + self.run_command([python, "--version"]) + parser = TalosOutputParser( + config=self.config, log_obj=self.log_obj, error_list=TalosErrorList + ) + env = {} + env["MOZ_UPLOAD_DIR"] = self.query_abs_dirs()["abs_blob_upload_dir"] + if not self.run_local: + env["MINIDUMP_STACKWALK"] = self.query_minidump_stackwalk() + env["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs()["abs_blob_upload_dir"] + env["RUST_BACKTRACE"] = "full" + if not os.path.isdir(env["MOZ_UPLOAD_DIR"]): + self.mkdir_p(env["MOZ_UPLOAD_DIR"]) + env = self.query_env(partial_env=env, log_level=INFO) + # adjust PYTHONPATH to be able to use talos as a python package + if "PYTHONPATH" in env: + env["PYTHONPATH"] = self.talos_path + os.pathsep + env["PYTHONPATH"] + else: + env["PYTHONPATH"] = self.talos_path + + if self.repo_path is not None: + env["MOZ_DEVELOPER_REPO_DIR"] = self.repo_path + if self.obj_path is not None: + env["MOZ_DEVELOPER_OBJ_DIR"] = self.obj_path + + # sets a timeout for how long talos should run without output + output_timeout = self.config.get("talos_output_timeout", 3600) + # run talos tests + run_tests = os.path.join(self.talos_path, "talos", "run_tests.py") + + # Dynamically set the log level based on the talos config for consistency + # throughout the test + mozlog_opts = [f"--log-tbpl-level={self.config['log_level']}"] + + if not self.run_local and "suite" in self.config: + fname_pattern = "%s_%%s.log" % self.config["suite"] + mozlog_opts.append( + "--log-errorsummary=%s" + % os.path.join(env["MOZ_UPLOAD_DIR"], fname_pattern % "errorsummary") + ) + + def launch_in_debug_mode(cmdline): + cmdline = set(cmdline) + debug_opts = {"--debug", "--debugger", "--debugger_args"} + + return bool(debug_opts.intersection(cmdline)) + + command = [python, run_tests] + options + mozlog_opts + if launch_in_debug_mode(command): + talos_process = subprocess.Popen( + command, cwd=self.workdir, env=env, bufsize=0 + ) + talos_process.wait() + else: + self.return_code = self.run_command( + command, + cwd=self.workdir, + output_timeout=output_timeout, + output_parser=parser, + env=env, + ) + if parser.minidump_output: + self.info("Looking at the minidump files for debugging purposes...") + for item in parser.minidump_output: + self.run_command(["ls", "-l", item]) + + if self.return_code not in [0]: + # update the worst log level and tbpl status + log_level = ERROR + tbpl_level = TBPL_FAILURE + if self.return_code == 1: + log_level = WARNING + tbpl_level = TBPL_WARNING + if self.return_code == 4: + log_level = WARNING + tbpl_level = TBPL_RETRY + + parser.update_worst_log_and_tbpl_levels(log_level, tbpl_level) + elif "--no-upload-results" not in options: + if not self.gecko_profile: + self._validate_treeherder_data(parser) + if not self.run_local: + # copy results to upload dir so they are included as an artifact + dest = os.path.join(env["MOZ_UPLOAD_DIR"], "perfherder-data.json") + self._artifact_perf_data(parser, dest) + + self.record_status(parser.worst_tbpl_status, level=parser.worst_log_level) diff --git a/testing/mozharness/mozharness/mozilla/testing/testbase.py b/testing/mozharness/mozharness/mozilla/testing/testbase.py new file mode 100755 index 0000000000..e8f37ceb8b --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/testing/testbase.py @@ -0,0 +1,767 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** + +import copy +import json +import os +import platform +import ssl + +from six.moves import urllib +from six.moves.urllib.parse import ParseResult, urlparse + +from mozharness.base.errors import BaseErrorList +from mozharness.base.log import FATAL, WARNING +from mozharness.base.python import ( + ResourceMonitoringMixin, + VirtualenvMixin, + virtualenv_config_options, +) +from mozharness.lib.python.authentication import get_credentials +from mozharness.mozilla.automation import TBPL_WARNING, AutomationMixin +from mozharness.mozilla.structuredlog import StructuredOutputParser +from mozharness.mozilla.testing.try_tools import TryToolsMixin, try_config_options +from mozharness.mozilla.testing.unittest import DesktopUnittestOutputParser +from mozharness.mozilla.testing.verify_tools import ( + VerifyToolsMixin, + verify_config_options, +) +from mozharness.mozilla.tooltool import TooltoolMixin + +INSTALLER_SUFFIXES = ( + ".apk", # Android + ".tar.bz2", + ".tar.gz", # Linux + ".dmg", # Mac + ".installer-stub.exe", + ".installer.exe", + ".exe", + ".zip", # Windows +) + +# https://searchfox.org/mozilla-central/source/testing/config/tooltool-manifests +TOOLTOOL_PLATFORM_DIR = { + "linux": "linux32", + "linux64": "linux64", + "win32": "win32", + "win64": "win32", + "macosx": "macosx64", +} + + +testing_config_options = ( + [ + [ + ["--installer-url"], + { + "action": "store", + "dest": "installer_url", + "default": None, + "help": "URL to the installer to install", + }, + ], + [ + ["--installer-path"], + { + "action": "store", + "dest": "installer_path", + "default": None, + "help": "Path to the installer to install. " + "This is set automatically if run with --download-and-extract.", + }, + ], + [ + ["--binary-path"], + { + "action": "store", + "dest": "binary_path", + "default": None, + "help": "Path to installed binary. This is set automatically if run with --install.", # NOQA: E501 + }, + ], + [ + ["--exe-suffix"], + { + "action": "store", + "dest": "exe_suffix", + "default": None, + "help": "Executable suffix for binaries on this platform", + }, + ], + [ + ["--test-url"], + { + "action": "store", + "dest": "test_url", + "default": None, + "help": "URL to the zip file containing the actual tests", + }, + ], + [ + ["--test-packages-url"], + { + "action": "store", + "dest": "test_packages_url", + "default": None, + "help": "URL to a json file describing which tests archives to download", + }, + ], + [ + ["--jsshell-url"], + { + "action": "store", + "dest": "jsshell_url", + "default": None, + "help": "URL to the jsshell to install", + }, + ], + [ + ["--download-symbols"], + { + "action": "store", + "dest": "download_symbols", + "type": "choice", + "choices": ["ondemand", "true"], + "help": "Download and extract crash reporter symbols.", + }, + ], + ] + + copy.deepcopy(virtualenv_config_options) + + copy.deepcopy(try_config_options) + + copy.deepcopy(verify_config_options) +) + + +# TestingMixin {{{1 +class TestingMixin( + VirtualenvMixin, + AutomationMixin, + ResourceMonitoringMixin, + TooltoolMixin, + TryToolsMixin, + VerifyToolsMixin, +): + """ + The steps to identify + download the proper bits for [browser] unit + tests and Talos. + """ + + installer_url = None + installer_path = None + binary_path = None + test_url = None + test_packages_url = None + symbols_url = None + symbols_path = None + jsshell_url = None + minidump_stackwalk_path = None + ssl_context = None + + def query_build_dir_url(self, file_name): + """ + Resolve a file name to a potential url in the build upload directory where + that file can be found. + """ + if self.test_packages_url: + reference_url = self.test_packages_url + elif self.installer_url: + reference_url = self.installer_url + else: + self.fatal( + "Can't figure out build directory urls without an installer_url " + "or test_packages_url!" + ) + + reference_url = urllib.parse.unquote(reference_url) + parts = list(urlparse(reference_url)) + + last_slash = parts[2].rfind("/") + parts[2] = "/".join([parts[2][:last_slash], file_name]) + + url = ParseResult(*parts).geturl() + + return url + + def query_prefixed_build_dir_url(self, suffix): + """Resolve a file name prefixed with platform and build details to a potential url + in the build upload directory where that file can be found. + """ + if self.test_packages_url: + reference_suffixes = [".test_packages.json"] + reference_url = self.test_packages_url + elif self.installer_url: + reference_suffixes = INSTALLER_SUFFIXES + reference_url = self.installer_url + else: + self.fatal( + "Can't figure out build directory urls without an installer_url " + "or test_packages_url!" + ) + + url = None + for reference_suffix in reference_suffixes: + if reference_url.endswith(reference_suffix): + url = reference_url[: -len(reference_suffix)] + suffix + break + + return url + + def query_symbols_url(self, raise_on_failure=False): + if self.symbols_url: + return self.symbols_url + + elif self.installer_url: + symbols_url = self.query_prefixed_build_dir_url( + ".crashreporter-symbols.zip" + ) + + # Check if the URL exists. If not, use none to allow mozcrash to auto-check for symbols + try: + if symbols_url: + self._urlopen(symbols_url, timeout=120) + self.symbols_url = symbols_url + except Exception as ex: + self.warning( + "Cannot open symbols url %s (installer url: %s): %s" + % (symbols_url, self.installer_url, ex) + ) + if raise_on_failure: + raise + + # If no symbols URL can be determined let minidump-stackwalk query the symbols. + # As of now this only works for Nightly and release builds. + if not self.symbols_url: + self.warning( + "No symbols_url found. Let minidump-stackwalk query for symbols." + ) + + return self.symbols_url + + def _pre_config_lock(self, rw_config): + for i, (target_file, target_dict) in enumerate( + rw_config.all_cfg_files_and_dicts + ): + if "developer_config" in target_file: + self._developer_mode_changes(rw_config) + + def _developer_mode_changes(self, rw_config): + """This function is called when you append the config called + developer_config.py. This allows you to run a job + outside of the Release Engineering infrastructure. + + What this functions accomplishes is: + * --installer-url is set + * --test-url is set if needed + * every url is substituted by another external to the + Release Engineering network + """ + c = self.config + orig_config = copy.deepcopy(c) + self.actions = tuple(rw_config.actions) + + def _replace_url(url, changes): + for from_, to_ in changes: + if url.startswith(from_): + new_url = url.replace(from_, to_) + self.info("Replacing url %s -> %s" % (url, new_url)) + return new_url + return url + + if c.get("installer_url") is None: + self.exception("You must use --installer-url with developer_config.py") + if c.get("require_test_zip"): + if not c.get("test_url") and not c.get("test_packages_url"): + self.exception( + "You must use --test-url or --test-packages-url with " + "developer_config.py" + ) + + c["installer_url"] = _replace_url(c["installer_url"], c["replace_urls"]) + if c.get("test_url"): + c["test_url"] = _replace_url(c["test_url"], c["replace_urls"]) + if c.get("test_packages_url"): + c["test_packages_url"] = _replace_url( + c["test_packages_url"], c["replace_urls"] + ) + + for key, value in self.config.items(): + if type(value) == str and value.startswith("http"): + self.config[key] = _replace_url(value, c["replace_urls"]) + + # Any changes to c means that we need credentials + if not c == orig_config: + get_credentials() + + def _urlopen(self, url, **kwargs): + """ + This function helps dealing with downloading files while outside + of the releng network. + """ + # Code based on http://code.activestate.com/recipes/305288-http-basic-authentication + def _urlopen_basic_auth(url, **kwargs): + self.info("We want to download this file %s" % url) + if not hasattr(self, "https_username"): + self.info( + "NOTICE: Files downloaded from outside of " + "Release Engineering network require LDAP " + "credentials." + ) + + self.https_username, self.https_password = get_credentials() + # This creates a password manager + passman = urllib.request.HTTPPasswordMgrWithDefaultRealm() + # Because we have put None at the start it will use this username/password + # combination from here on + passman.add_password(None, url, self.https_username, self.https_password) + authhandler = urllib.request.HTTPBasicAuthHandler(passman) + + return urllib.request.build_opener(authhandler).open(url, **kwargs) + + # If we have the developer_run flag enabled then we will switch + # URLs to the right place and enable http authentication + if "developer_config.py" in self.config["config_files"]: + return _urlopen_basic_auth(url, **kwargs) + else: + # windows certificates need to be refreshed (https://bugs.python.org/issue36011) + if self.platform_name() in ("win64",) and platform.architecture()[0] in ( + "x64", + ): + if self.ssl_context is None: + self.ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS) + self.ssl_context.load_default_certs() + return urllib.request.urlopen(url, context=self.ssl_context, **kwargs) + else: + return urllib.request.urlopen(url, **kwargs) + + def _query_binary_version(self, regex, cmd): + output = self.get_output_from_command(cmd, silent=False) + return regex.search(output).group(0) + + def preflight_download_and_extract(self): + message = "" + if not self.installer_url: + message += """installer_url isn't set! + +You can set this by specifying --installer-url URL +""" + if ( + self.config.get("require_test_zip") + and not self.test_url + and not self.test_packages_url + ): + message += """test_url isn't set! + +You can set this by specifying --test-url URL +""" + if message: + self.fatal(message + "Can't run download-and-extract... exiting") + + def _read_packages_manifest(self): + dirs = self.query_abs_dirs() + source = self.download_file( + self.test_packages_url, parent_dir=dirs["abs_work_dir"], error_level=FATAL + ) + + with self.opened(os.path.realpath(source)) as (fh, err): + package_requirements = json.load(fh) + if not package_requirements or err: + self.fatal( + "There was an error reading test package requirements from %s " + "requirements: `%s` - error: `%s`" + % (source, package_requirements or "None", err or "No error") + ) + return package_requirements + + def _download_test_packages(self, suite_categories, extract_dirs): + # Some platforms define more suite categories/names than others. + # This is a difference in the convention of the configs more than + # to how these tests are run, so we pave over these differences here. + aliases = { + "mochitest-chrome": "mochitest", + "mochitest-media": "mochitest", + "mochitest-plain": "mochitest", + "mochitest-plain-gpu": "mochitest", + "mochitest-webgl1-core": "mochitest", + "mochitest-webgl1-ext": "mochitest", + "mochitest-webgl2-core": "mochitest", + "mochitest-webgl2-ext": "mochitest", + "mochitest-webgl2-deqp": "mochitest", + "mochitest-webgpu": "mochitest", + "geckoview": "mochitest", + "geckoview-junit": "mochitest", + "reftest-qr": "reftest", + "crashtest": "reftest", + "crashtest-qr": "reftest", + "reftest-debug": "reftest", + "crashtest-debug": "reftest", + } + suite_categories = [aliases.get(name, name) for name in suite_categories] + + dirs = self.query_abs_dirs() + test_install_dir = dirs.get( + "abs_test_install_dir", os.path.join(dirs["abs_work_dir"], "tests") + ) + self.mkdir_p(test_install_dir) + package_requirements = self._read_packages_manifest() + target_packages = [] + c = self.config + for category in suite_categories: + specified_suites = c.get("specified_{}_suites".format(category)) + if specified_suites: + found = False + for specified_suite in specified_suites: + if specified_suite in package_requirements: + target_packages.extend(package_requirements[specified_suite]) + found = True + if found: + continue + + if category in package_requirements: + target_packages.extend(package_requirements[category]) + else: + # If we don't harness specific requirements, assume the common zip + # has everything we need to run tests for this suite. + target_packages.extend(package_requirements["common"]) + + # eliminate duplicates -- no need to download anything twice + target_packages = list(set(target_packages)) + self.info( + "Downloading packages: %s for test suite categories: %s" + % (target_packages, suite_categories) + ) + for file_name in target_packages: + target_dir = test_install_dir + unpack_dirs = extract_dirs + + if "common.tests" in file_name and isinstance(unpack_dirs, list): + # Ensure that the following files are always getting extracted + required_files = [ + "mach", + "mozinfo.json", + ] + for req_file in required_files: + if req_file not in unpack_dirs: + self.info( + "Adding '{}' for extraction from common.tests archive".format( + req_file + ) + ) + unpack_dirs.append(req_file) + + if "jsshell-" in file_name or file_name == "target.jsshell.zip": + self.info("Special-casing the jsshell zip file") + unpack_dirs = None + target_dir = dirs["abs_test_bin_dir"] + + if "web-platform" in file_name: + self.info("Extracting everything from web-platform archive") + unpack_dirs = None + + url = self.query_build_dir_url(file_name) + self.download_unpack(url, target_dir, extract_dirs=unpack_dirs) + + def _download_test_zip(self, extract_dirs=None): + dirs = self.query_abs_dirs() + test_install_dir = dirs.get( + "abs_test_install_dir", os.path.join(dirs["abs_work_dir"], "tests") + ) + self.download_unpack(self.test_url, test_install_dir, extract_dirs=extract_dirs) + + def structured_output(self, suite_category): + """Defines whether structured logging is in use in this configuration. This + may need to be replaced with data from a different config at the resolution + of bug 1070041 and related bugs. + """ + return ( + "structured_suites" in self.config + and suite_category in self.config["structured_suites"] + ) + + def get_test_output_parser( + self, + suite_category, + strict=False, + fallback_parser_class=DesktopUnittestOutputParser, + **kwargs + ): + """Derive and return an appropriate output parser, either the structured + output parser or a fallback based on the type of logging in use as determined by + configuration. + """ + if not self.structured_output(suite_category): + if fallback_parser_class is DesktopUnittestOutputParser: + return DesktopUnittestOutputParser( + suite_category=suite_category, **kwargs + ) + return fallback_parser_class(**kwargs) + self.info("Structured output parser in use for %s." % suite_category) + return StructuredOutputParser( + suite_category=suite_category, strict=strict, **kwargs + ) + + def _download_installer(self): + file_name = None + if self.installer_path: + file_name = self.installer_path + dirs = self.query_abs_dirs() + source = self.download_file( + self.installer_url, + file_name=file_name, + parent_dir=dirs["abs_work_dir"], + error_level=FATAL, + ) + self.installer_path = os.path.realpath(source) + + def _download_and_extract_symbols(self): + dirs = self.query_abs_dirs() + if self.config.get("download_symbols") == "ondemand": + self.symbols_url = self.retry( + action=self.query_symbols_url, + kwargs={"raise_on_failure": True}, + sleeptime=10, + failure_status=None, + ) + self.symbols_path = self.symbols_url + return + + else: + # In the case for 'ondemand', we're OK to proceed without getting a hold of the + # symbols right this moment, however, in other cases we need to at least retry + # before being unable to proceed (e.g. debug tests need symbols) + self.symbols_url = self.retry( + action=self.query_symbols_url, + kwargs={"raise_on_failure": True}, + sleeptime=20, + error_level=FATAL, + error_message="We can't proceed without downloading symbols.", + ) + if not self.symbols_path: + self.symbols_path = os.path.join(dirs["abs_work_dir"], "symbols") + + if self.symbols_url: + self.download_unpack(self.symbols_url, self.symbols_path) + + def download_and_extract(self, extract_dirs=None, suite_categories=None): + """ + download and extract test zip / download installer + """ + # Swap plain http for https when we're downloading from ftp + # See bug 957502 and friends + from_ = "http://ftp.mozilla.org" + to_ = "https://ftp-ssl.mozilla.org" + for attr in "symbols_url", "installer_url", "test_packages_url", "test_url": + url = getattr(self, attr) + if url and url.startswith(from_): + new_url = url.replace(from_, to_) + self.info("Replacing url %s -> %s" % (url, new_url)) + setattr(self, attr, new_url) + + if "test_url" in self.config: + # A user has specified a test_url directly, any test_packages_url will + # be ignored. + if self.test_packages_url: + self.error( + 'Test data will be downloaded from "%s", the specified test ' + ' package data at "%s" will be ignored.' + % (self.config.get("test_url"), self.test_packages_url) + ) + + self._download_test_zip(extract_dirs) + else: + if not self.test_packages_url: + # The caller intends to download harness specific packages, but doesn't know + # where the packages manifest is located. This is the case when the + # test package manifest isn't set as a property, which is true + # for some self-serve jobs and platforms using parse_make_upload. + self.test_packages_url = self.query_prefixed_build_dir_url( + ".test_packages.json" + ) + + suite_categories = suite_categories or ["common"] + self._download_test_packages(suite_categories, extract_dirs) + + self._download_installer() + if self.config.get("download_symbols"): + self._download_and_extract_symbols() + + # create_virtualenv is in VirtualenvMixin. + + def preflight_install(self): + if not self.installer_path: + if self.config.get("installer_path"): + self.installer_path = self.config["installer_path"] + else: + self.fatal( + """installer_path isn't set! + +You can set this by: + +1. specifying --installer-path PATH, or +2. running the download-and-extract action +""" + ) + if not self.is_python_package_installed("mozInstall"): + self.fatal( + """Can't call install() without mozinstall! +Did you run with --create-virtualenv? Is mozinstall in virtualenv_modules?""" + ) + + def install_app(self, app=None, target_dir=None, installer_path=None): + """Dependent on mozinstall""" + # install the application + cmd = [self.query_python_path("mozinstall")] + if app: + cmd.extend(["--app", app]) + dirs = self.query_abs_dirs() + if not target_dir: + target_dir = dirs.get( + "abs_app_install_dir", os.path.join(dirs["abs_work_dir"], "application") + ) + self.mkdir_p(target_dir) + if not installer_path: + installer_path = self.installer_path + cmd.extend([installer_path, "--destination", target_dir]) + # TODO we'll need some error checking here + return self.get_output_from_command( + cmd, halt_on_failure=True, fatal_exit_code=3 + ) + + def install(self): + self.binary_path = self.install_app(app=self.config.get("application")) + self.install_dir = os.path.dirname(self.binary_path) + + def uninstall_app(self, install_dir=None): + """Dependent on mozinstall""" + # uninstall the application + cmd = self.query_exe( + "mozuninstall", + default=self.query_python_path("mozuninstall"), + return_type="list", + ) + dirs = self.query_abs_dirs() + if not install_dir: + install_dir = dirs.get( + "abs_app_install_dir", os.path.join(dirs["abs_work_dir"], "application") + ) + cmd.append(install_dir) + # TODO we'll need some error checking here + self.get_output_from_command(cmd, halt_on_failure=True, fatal_exit_code=3) + + def uninstall(self): + self.uninstall_app() + + def query_minidump_stackwalk(self, manifest=None): + if self.minidump_stackwalk_path: + return self.minidump_stackwalk_path + + minidump_stackwalk_path = None + + if "MOZ_FETCHES_DIR" in os.environ: + minidump_stackwalk_path = os.path.join( + os.environ["MOZ_FETCHES_DIR"], + "minidump-stackwalk", + "minidump-stackwalk", + ) + + if self.platform_name() in ("win32", "win64"): + minidump_stackwalk_path += ".exe" + + if not minidump_stackwalk_path or not os.path.isfile(minidump_stackwalk_path): + self.error("minidump-stackwalk path was not fetched?") + # don't burn the job but we should at least turn them orange so it is caught + self.record_status(TBPL_WARNING, WARNING) + return None + + self.minidump_stackwalk_path = minidump_stackwalk_path + return self.minidump_stackwalk_path + + def query_options(self, *args, **kwargs): + if "str_format_values" in kwargs: + str_format_values = kwargs.pop("str_format_values") + else: + str_format_values = {} + + arguments = [] + + for arg in args: + if arg is not None: + arguments.extend(argument % str_format_values for argument in arg) + + return arguments + + def query_tests_args(self, *args, **kwargs): + if "str_format_values" in kwargs: + str_format_values = kwargs.pop("str_format_values") + else: + str_format_values = {} + + arguments = [] + + for arg in reversed(args): + if arg: + arguments.append("--") + arguments.extend(argument % str_format_values for argument in arg) + break + + return arguments + + def _run_cmd_checks(self, suites): + if not suites: + return + dirs = self.query_abs_dirs() + for suite in suites: + # XXX platform.architecture() may give incorrect values for some + # platforms like mac as excutable files may be universal + # files containing multiple architectures + # NOTE 'enabled' is only here while we have unconsolidated configs + if not suite["enabled"]: + continue + if suite.get("architectures"): + arch = platform.architecture()[0] + if arch not in suite["architectures"]: + continue + cmd = suite["cmd"] + name = suite["name"] + self.info( + "Running pre test command %(name)s with '%(cmd)s'" + % {"name": name, "cmd": " ".join(cmd)} + ) + self.run_command( + cmd, + cwd=dirs["abs_work_dir"], + error_list=BaseErrorList, + halt_on_failure=suite["halt_on_failure"], + fatal_exit_code=suite.get("fatal_exit_code", 3), + ) + + def preflight_run_tests(self): + """preflight commands for all tests""" + c = self.config + if c.get("skip_preflight"): + self.info("skipping preflight") + return + + if c.get("run_cmd_checks_enabled"): + self._run_cmd_checks(c.get("preflight_run_cmd_suites", [])) + elif c.get("preflight_run_cmd_suites"): + self.warning( + "Proceeding without running prerun test commands." + " These are often OS specific and disabling them may" + " result in spurious test results!" + ) + + def postflight_run_tests(self): + """preflight commands for all tests""" + c = self.config + if c.get("run_cmd_checks_enabled"): + self._run_cmd_checks(c.get("postflight_run_cmd_suites", [])) + + def query_abs_dirs(self): + abs_dirs = super(TestingMixin, self).query_abs_dirs() + if "MOZ_FETCHES_DIR" in os.environ: + abs_dirs["abs_fetches_dir"] = os.environ["MOZ_FETCHES_DIR"] + return abs_dirs diff --git a/testing/mozharness/mozharness/mozilla/testing/try_tools.py b/testing/mozharness/mozharness/mozilla/testing/try_tools.py new file mode 100644 index 0000000000..ac92ef534c --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/testing/try_tools.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** + +import argparse +import os +import re +from collections import defaultdict + +import six + +from mozharness.base.script import PostScriptAction +from mozharness.base.transfer import TransferMixin + +try_config_options = [ + [ + ["--try-message"], + { + "action": "store", + "dest": "try_message", + "default": None, + "help": "try syntax string to select tests to run", + }, + ], +] + +test_flavors = { + "browser-chrome": {}, + "browser-a11y": {}, + "browser-media": {}, + "chrome": {}, + "devtools-chrome": {}, + "mochitest": {}, + "xpcshell": {}, + "reftest": {"path": lambda x: os.path.join("tests", "reftest", "tests", x)}, + "crashtest": {"path": lambda x: os.path.join("tests", "reftest", "tests", x)}, + "remote": {"path": lambda x: os.path.join("remote", "test", "browser", x)}, + "web-platform-tests": { + "path": lambda x: os.path.join("tests", x.split("testing" + os.path.sep)[1]) + }, + "web-platform-tests-reftests": { + "path": lambda x: os.path.join("tests", x.split("testing" + os.path.sep)[1]) + }, + "web-platform-tests-wdspec": { + "path": lambda x: os.path.join("tests", x.split("testing" + os.path.sep)[1]) + }, +} + + +class TryToolsMixin(TransferMixin): + """Utility functions for an interface between try syntax and out test harnesses. + Requires log and script mixins.""" + + harness_extra_args = None + try_test_paths = {} + known_try_arguments = { + "--tag": ( + { + "action": "append", + "dest": "tags", + "default": None, + }, + ( + "browser-chrome", + "browser-a11y", + "browser-media", + "chrome", + "devtools-chrome", + "marionette", + "mochitest", + "web-plaftform-tests", + "xpcshell", + ), + ), + } + + def _extract_try_message(self): + msg = None + if "try_message" in self.config and self.config["try_message"]: + msg = self.config["try_message"] + elif "TRY_COMMIT_MSG" in os.environ: + msg = os.environ["TRY_COMMIT_MSG"] + + if not msg: + self.warning("Try message not found.") + return msg + + def _extract_try_args(self, msg): + """Returns a list of args from a try message, for parsing""" + if not msg: + return None + all_try_args = None + for line in msg.splitlines(): + if "try: " in line: + # Autoland adds quotes to try strings that will confuse our + # args later on. + if line.startswith('"') and line.endswith('"'): + line = line[1:-1] + # Allow spaces inside of [filter expressions] + try_message = line.strip().split("try: ", 1) + all_try_args = re.findall(r"(?:\[.*?\]|\S)+", try_message[1]) + break + if not all_try_args: + self.warning("Try syntax not found in: %s." % msg) + return all_try_args + + def try_message_has_flag(self, flag, message=None): + """ + Returns True if --`flag` is present in message. + """ + parser = argparse.ArgumentParser() + parser.add_argument("--" + flag, action="store_true") + message = message or self._extract_try_message() + if not message: + return False + msg_list = self._extract_try_args(message) + args, _ = parser.parse_known_args(msg_list) + return getattr(args, flag, False) + + def _is_try(self): + repo_path = None + get_branch = self.config.get("branch", repo_path) + if get_branch is not None: + on_try = "try" in get_branch or "Try" in get_branch + elif os.environ is not None: + on_try = "TRY_COMMIT_MSG" in os.environ + else: + on_try = False + return on_try + + @PostScriptAction("download-and-extract") + def set_extra_try_arguments(self, action, success=None): + """Finds a commit message and parses it for extra arguments to pass to the test + harness command line and test paths used to filter manifests. + + Extracting arguments from a commit message taken directly from the try_parser. + """ + if not self._is_try(): + return + + msg = self._extract_try_message() + if not msg: + return + + all_try_args = self._extract_try_args(msg) + if not all_try_args: + return + + parser = argparse.ArgumentParser( + description=( + "Parse an additional subset of arguments passed to try syntax" + " and forward them to the underlying test harness command." + ) + ) + + label_dict = {} + + def label_from_val(val): + if val in label_dict: + return label_dict[val] + return "--%s" % val.replace("_", "-") + + for label, (opts, _) in six.iteritems(self.known_try_arguments): + if "action" in opts and opts["action"] not in ( + "append", + "store", + "store_true", + "store_false", + ): + self.fatal( + "Try syntax does not support passing custom or store_const " + "arguments to the harness process." + ) + if "dest" in opts: + label_dict[opts["dest"]] = label + + parser.add_argument(label, **opts) + + parser.add_argument("--try-test-paths", nargs="*") + (args, _) = parser.parse_known_args(all_try_args) + self.try_test_paths = self._group_test_paths(args.try_test_paths) + del args.try_test_paths + + out_args = defaultdict(list) + # This is a pretty hacky way to echo arguments down to the harness. + # Hopefully this can be improved once we have a configuration system + # in tree for harnesses that relies less on a command line. + for arg, value in six.iteritems(vars(args)): + if value: + label = label_from_val(arg) + _, flavors = self.known_try_arguments[label] + + for f in flavors: + if isinstance(value, bool): + # A store_true or store_false argument. + out_args[f].append(label) + elif isinstance(value, list): + out_args[f].extend(["%s=%s" % (label, el) for el in value]) + else: + out_args[f].append("%s=%s" % (label, value)) + + self.harness_extra_args = dict(out_args) + + def _group_test_paths(self, args): + rv = defaultdict(list) + + if args is None: + return rv + + for item in args: + suite, path = item.split(":", 1) + rv[suite].append(path) + return rv + + def try_args(self, flavor): + """Get arguments, test_list derived from try syntax to apply to a command""" + args = [] + if self.harness_extra_args: + args = self.harness_extra_args.get(flavor, [])[:] + + if self.try_test_paths.get(flavor): + self.info( + "TinderboxPrint: Tests will be run from the following " + "files: %s." % ",".join(self.try_test_paths[flavor]) + ) + args.extend(["--this-chunk=1", "--total-chunks=1"]) + + path_func = test_flavors[flavor].get("path", lambda x: x) + tests = [ + path_func(os.path.normpath(item)) + for item in self.try_test_paths[flavor] + ] + else: + tests = [] + + if args or tests: + self.info( + "TinderboxPrint: The following arguments were forwarded from mozharness " + "to the test command:\nTinderboxPrint: \t%s -- %s" + % (" ".join(args), " ".join(tests)) + ) + + return args, tests diff --git a/testing/mozharness/mozharness/mozilla/testing/unittest.py b/testing/mozharness/mozharness/mozilla/testing/unittest.py new file mode 100755 index 0000000000..be144bbe1f --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/testing/unittest.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** + +import os +import re + +from mozharness.base.log import CRITICAL, ERROR, INFO, WARNING, OutputParser +from mozharness.mozilla.automation import ( + TBPL_FAILURE, + TBPL_RETRY, + TBPL_SUCCESS, + TBPL_WARNING, + TBPL_WORST_LEVEL_TUPLE, +) +from mozharness.mozilla.testing.errors import TinderBoxPrintRe + +SUITE_CATEGORIES = ["mochitest", "reftest", "xpcshell"] + + +def tbox_print_summary( + pass_count, fail_count, known_fail_count=None, crashed=False, leaked=False +): + emphasize_fail_text = '<em class="testfail">%s</em>' + + if ( + pass_count < 0 + or fail_count < 0 + or (known_fail_count is not None and known_fail_count < 0) + ): + summary = emphasize_fail_text % "T-FAIL" + elif ( + pass_count == 0 + and fail_count == 0 + and (known_fail_count == 0 or known_fail_count is None) + ): + summary = emphasize_fail_text % "T-FAIL" + else: + str_fail_count = str(fail_count) + if fail_count > 0: + str_fail_count = emphasize_fail_text % str_fail_count + summary = "%d/%s" % (pass_count, str_fail_count) + if known_fail_count is not None: + summary += "/%d" % known_fail_count + # Format the crash status. + if crashed: + summary += " %s" % emphasize_fail_text % "CRASH" + # Format the leak status. + if leaked is not False: + summary += " %s" % emphasize_fail_text % ((leaked and "LEAK") or "L-FAIL") + return summary + + +class TestSummaryOutputParserHelper(OutputParser): + def __init__(self, regex=re.compile(r"(passed|failed|todo): (\d+)"), **kwargs): + self.regex = regex + self.failed = 0 + self.passed = 0 + self.todo = 0 + self.last_line = None + self.tbpl_status = TBPL_SUCCESS + self.worst_log_level = INFO + super(TestSummaryOutputParserHelper, self).__init__(**kwargs) + + def parse_single_line(self, line): + super(TestSummaryOutputParserHelper, self).parse_single_line(line) + self.last_line = line + m = self.regex.search(line) + if m: + try: + setattr(self, m.group(1), int(m.group(2))) + except ValueError: + # ignore bad values + pass + + def evaluate_parser(self, return_code, success_codes=None, previous_summary=None): + # TestSummaryOutputParserHelper is for Marionette, which doesn't support test-verify + # When it does we can reset the internal state variables as needed + joined_summary = previous_summary + + if return_code == 0 and self.passed > 0 and self.failed == 0: + self.tbpl_status = TBPL_SUCCESS + elif return_code == 10 and self.failed > 0: + self.tbpl_status = TBPL_WARNING + else: + self.tbpl_status = TBPL_FAILURE + self.worst_log_level = ERROR + + return (self.tbpl_status, self.worst_log_level, joined_summary) + + def print_summary(self, suite_name): + # generate the TinderboxPrint line for TBPL + emphasize_fail_text = '<em class="testfail">%s</em>' + failed = "0" + if self.passed == 0 and self.failed == 0: + self.tsummary = emphasize_fail_text % "T-FAIL" + else: + if self.failed > 0: + failed = emphasize_fail_text % str(self.failed) + self.tsummary = "%d/%s/%d" % (self.passed, failed, self.todo) + + self.info("TinderboxPrint: %s<br/>%s\n" % (suite_name, self.tsummary)) + + def append_tinderboxprint_line(self, suite_name): + self.print_summary(suite_name) + + +class DesktopUnittestOutputParser(OutputParser): + """ + A class that extends OutputParser such that it can parse the number of + passed/failed/todo tests from the output. + """ + + def __init__(self, suite_category, **kwargs): + # worst_log_level defined already in DesktopUnittestOutputParser + # but is here to make pylint happy + self.worst_log_level = INFO + super(DesktopUnittestOutputParser, self).__init__(**kwargs) + self.summary_suite_re = TinderBoxPrintRe.get("%s_summary" % suite_category, {}) + self.harness_error_re = TinderBoxPrintRe["harness_error"]["minimum_regex"] + self.full_harness_error_re = TinderBoxPrintRe["harness_error"]["full_regex"] + self.harness_retry_re = TinderBoxPrintRe["harness_error"]["retry_regex"] + self.fail_count = -1 + self.pass_count = -1 + # known_fail_count does not exist for some suites + self.known_fail_count = self.summary_suite_re.get("known_fail_group") and -1 + self.crashed, self.leaked = False, False + self.tbpl_status = TBPL_SUCCESS + + def parse_single_line(self, line): + if self.summary_suite_re: + summary_m = self.summary_suite_re["regex"].match(line) # pass/fail/todo + if summary_m: + message = " %s" % line + log_level = INFO + # remove all the none values in groups() so this will work + # with all suites including mochitest browser-chrome + summary_match_list = [ + group for group in summary_m.groups() if group is not None + ] + r = summary_match_list[0] + if self.summary_suite_re["pass_group"] in r: + if len(summary_match_list) > 1: + self.pass_count = int(summary_match_list[-1]) + else: + # This handles suites that either pass or report + # number of failures. We need to set both + # pass and fail count in the pass case. + self.pass_count = 1 + self.fail_count = 0 + elif self.summary_suite_re["fail_group"] in r: + self.fail_count = int(summary_match_list[-1]) + if self.fail_count > 0: + message += "\n One or more unittests failed." + log_level = WARNING + # If self.summary_suite_re['known_fail_group'] == None, + # then r should not match it, # so this test is fine as is. + elif self.summary_suite_re["known_fail_group"] in r: + self.known_fail_count = int(summary_match_list[-1]) + self.log(message, log_level) + return # skip harness check and base parse_single_line + harness_match = self.harness_error_re.search(line) + if harness_match: + self.warning(" %s" % line) + self.worst_log_level = self.worst_level(WARNING, self.worst_log_level) + self.tbpl_status = self.worst_level( + TBPL_WARNING, self.tbpl_status, levels=TBPL_WORST_LEVEL_TUPLE + ) + full_harness_match = self.full_harness_error_re.search(line) + if full_harness_match: + r = full_harness_match.group(1) + if r == "application crashed": + self.crashed = True + elif r == "missing output line for total leaks!": + self.leaked = None + else: + self.leaked = True + return # skip base parse_single_line + if self.harness_retry_re.search(line): + self.critical(" %s" % line) + self.worst_log_level = self.worst_level(CRITICAL, self.worst_log_level) + self.tbpl_status = self.worst_level( + TBPL_RETRY, self.tbpl_status, levels=TBPL_WORST_LEVEL_TUPLE + ) + return # skip base parse_single_line + super(DesktopUnittestOutputParser, self).parse_single_line(line) + + def evaluate_parser(self, return_code, success_codes=None, previous_summary=None): + success_codes = success_codes or [0] + + if self.num_errors: # mozharness ran into a script error + self.tbpl_status = self.worst_level( + TBPL_FAILURE, self.tbpl_status, levels=TBPL_WORST_LEVEL_TUPLE + ) + + """ + We can run evaluate_parser multiple times, it will duplicate failures + and status which can mean that future tests will fail if a previous test fails. + When we have a previous summary, we want to do: + 1) reset state so we only evaluate the current results + """ + joined_summary = {"pass_count": self.pass_count} + if previous_summary: + self.tbpl_status = TBPL_SUCCESS + self.worst_log_level = INFO + self.crashed = False + self.leaked = False + + # I have to put this outside of parse_single_line because this checks not + # only if fail_count was more then 0 but also if fail_count is still -1 + # (no fail summary line was found) + if self.fail_count != 0: + self.worst_log_level = self.worst_level(WARNING, self.worst_log_level) + self.tbpl_status = self.worst_level( + TBPL_WARNING, self.tbpl_status, levels=TBPL_WORST_LEVEL_TUPLE + ) + + # Account for the possibility that no test summary was output. + if ( + self.pass_count <= 0 + and self.fail_count <= 0 + and (self.known_fail_count is None or self.known_fail_count <= 0) + and os.environ.get("TRY_SELECTOR") != "coverage" + ): + self.error("No tests run or test summary not found") + self.worst_log_level = self.worst_level(WARNING, self.worst_log_level) + self.tbpl_status = self.worst_level( + TBPL_WARNING, self.tbpl_status, levels=TBPL_WORST_LEVEL_TUPLE + ) + + if return_code not in success_codes: + self.tbpl_status = self.worst_level( + TBPL_FAILURE, self.tbpl_status, levels=TBPL_WORST_LEVEL_TUPLE + ) + + # we can trust in parser.worst_log_level in either case + return (self.tbpl_status, self.worst_log_level, joined_summary) + + def append_tinderboxprint_line(self, suite_name): + # We are duplicating a condition (fail_count) from evaluate_parser and + # parse parse_single_line but at little cost since we are not parsing + # the log more then once. I figured this method should stay isolated as + # it is only here for tbpl highlighted summaries and is not part of + # result status IIUC. + summary = tbox_print_summary( + self.pass_count, + self.fail_count, + self.known_fail_count, + self.crashed, + self.leaked, + ) + self.info("TinderboxPrint: %s<br/>%s\n" % (suite_name, summary)) diff --git a/testing/mozharness/mozharness/mozilla/testing/verify_tools.py b/testing/mozharness/mozharness/mozilla/testing/verify_tools.py new file mode 100644 index 0000000000..3cf19351c5 --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/testing/verify_tools.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** + +from mozharness.base.script import PostScriptAction +from mozharness.mozilla.testing.per_test_base import SingleTestMixin + +verify_config_options = [ + [ + ["--verify"], + { + "action": "store_true", + "dest": "verify", + "default": False, + "help": "Run additional verification on modified tests.", + }, + ], +] + + +class VerifyToolsMixin(SingleTestMixin): + """Utility functions for test verification.""" + + def __init__(self): + super(VerifyToolsMixin, self).__init__() + + @property + def verify_enabled(self): + try: + return bool(self.config.get("verify")) + except (AttributeError, KeyError, TypeError): + return False + + @PostScriptAction("download-and-extract") + def find_tests_for_verification(self, action, success=None): + """ + For each file modified on this push, determine if the modified file + is a test, by searching test manifests. Populate self.verify_suites + with test files, organized by suite. + + This depends on test manifests, so can only run after test zips have + been downloaded and extracted. + """ + + if not self.verify_enabled: + return + + self.find_modified_tests() + + @property + def verify_args(self): + if not self.verify_enabled: + return [] + + # Limit each test harness run to 15 minutes, to avoid task timeouts + # when executing long-running tests. + MAX_TIME_PER_TEST = 900 + + if self.config.get("per_test_category") == "web-platform": + args = ["--verify-log-full"] + else: + args = ["--verify-max-time=%d" % MAX_TIME_PER_TEST] + + args.append("--verify") + + return args diff --git a/testing/mozharness/mozharness/mozilla/tooltool.py b/testing/mozharness/mozharness/mozilla/tooltool.py new file mode 100644 index 0000000000..db43071e50 --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/tooltool.py @@ -0,0 +1,86 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +"""module for tooltool operations""" +import os +import sys + +from mozharness.base.errors import PythonErrorList +from mozharness.base.log import ERROR, FATAL + +TooltoolErrorList = PythonErrorList + [{"substr": "ERROR - ", "level": ERROR}] + + +_here = os.path.abspath(os.path.dirname(__file__)) +_external_tools_path = os.path.normpath( + os.path.join(_here, "..", "..", "external_tools") +) + + +class TooltoolMixin(object): + """Mixin class for handling tooltool manifests. + To use a tooltool server other than the Mozilla server, set + TOOLTOOL_HOST in the environment. + """ + + def tooltool_fetch(self, manifest, output_dir=None, privileged=False, cache=None): + """docstring for tooltool_fetch""" + if cache is None: + cache = os.environ.get("TOOLTOOL_CACHE") + + for d in (output_dir, cache): + if d is not None and not os.path.exists(d): + self.mkdir_p(d) + if self.topsrcdir: + cmd = [ + sys.executable, + "-u", + os.path.join(self.topsrcdir, "mach"), + "artifact", + "toolchain", + "-v", + ] + else: + cmd = [ + sys.executable, + "-u", + os.path.join(_external_tools_path, "tooltool.py"), + ] + + if self.topsrcdir: + cmd.extend(["--tooltool-manifest", manifest]) + cmd.extend( + ["--artifact-manifest", os.path.join(self.topsrcdir, "toolchains.json")] + ) + else: + cmd.extend(["fetch", "-m", manifest, "-o"]) + + if cache: + cmd.extend(["--cache-dir" if self.topsrcdir else "-c", cache]) + + timeout = self.config.get("tooltool_timeout", 10 * 60) + + self.retry( + self.run_command, + args=(cmd,), + kwargs={ + "cwd": output_dir, + "error_list": TooltoolErrorList, + "privileged": privileged, + "output_timeout": timeout, + }, + good_statuses=(0,), + error_message="Tooltool %s fetch failed!" % manifest, + error_level=FATAL, + ) + + def create_tooltool_manifest(self, contents, path=None): + """Currently just creates a manifest, given the contents. + We may want a template and individual values in the future? + """ + if path is None: + dirs = self.query_abs_dirs() + path = os.path.join(dirs["abs_work_dir"], "tooltool.tt") + self.write_to_file(path, contents, error_level=FATAL) + return path diff --git a/testing/mozharness/mozharness/mozilla/vcstools.py b/testing/mozharness/mozharness/mozilla/vcstools.py new file mode 100644 index 0000000000..974923b6ec --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/vcstools.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +"""vcstools.py + +Author: Armen Zambrano G. +""" +import os + +from mozharness.base.script import PreScriptAction +from mozharness.base.vcs.vcsbase import VCSScript + +VCS_TOOLS = ("gittool.py",) + + +class VCSToolsScript(VCSScript): + """This script allows us to fetch gittool.py if + we're running the script on developer mode. + """ + + @PreScriptAction("checkout") + def _pre_checkout(self, action): + if self.config.get("developer_mode"): + # We put them on base_work_dir to prevent the clobber action + # to delete them before we use them + for vcs_tool in VCS_TOOLS: + file_path = self.query_exe(vcs_tool) + if not os.path.exists(file_path): + self.download_file( + url=self.config[vcs_tool], + file_name=file_path, + parent_dir=os.path.dirname(file_path), + create_parent_dir=True, + ) + self.chmod(file_path, 0o755) + else: + # We simply verify that everything is in order + # or if the user forgot to specify developer mode + for vcs_tool in VCS_TOOLS: + file_path = self.which(vcs_tool) + + if not file_path: + file_path = self.query_exe(vcs_tool) + + # If the tool is specified and it is a list is + # because we're running on Windows and we won't check + if type(self.query_exe(vcs_tool)) is list: + continue + + if file_path is None: + self.fatal( + "This machine is missing %s, if this is your " + "local machine you can use --cfg " + "developer_config.py" % vcs_tool + ) + elif not self.is_exe(file_path): + self.critical("%s is not executable." % file_path) |