diff options
Diffstat (limited to 'testing/mozharness/mozharness/base/config.py')
| -rw-r--r-- | testing/mozharness/mozharness/base/config.py | 693 |
1 files changed, 693 insertions, 0 deletions
diff --git a/testing/mozharness/mozharness/base/config.py b/testing/mozharness/mozharness/base/config.py new file mode 100644 index 0000000000..d12b3aecad --- /dev/null +++ b/testing/mozharness/mozharness/base/config.py @@ -0,0 +1,693 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +"""Generic config parsing and dumping, the way I remember it from scripts +gone by. + +The config should be built from script-level defaults, overlaid by +config-file defaults, overlaid by command line options. + + (For buildbot-analogues that would be factory-level defaults, + builder-level defaults, and build request/scheduler settings.) + +The config should then be locked (set to read-only, to prevent runtime +alterations). Afterwards we should dump the config to a file that is +uploaded with the build, and can be used to debug or replicate the build +at a later time. + +TODO: + +* check_required_settings or something -- run at init, assert that + these settings are set. +""" + +import os +import socket +import sys +import time +from copy import deepcopy +from optparse import Option, OptionGroup, OptionParser + +from mozharness.base.log import CRITICAL, DEBUG, ERROR, FATAL, INFO, WARNING + +try: + from urllib2 import URLError, urlopen +except ImportError: + from urllib.error import URLError + from urllib.request import urlopen + + +try: + import simplejson as json +except ImportError: + import json + + +# optparse {{{1 +class ExtendedOptionParser(OptionParser): + """OptionParser, but with ExtendOption as the option_class.""" + + def __init__(self, **kwargs): + kwargs["option_class"] = ExtendOption + OptionParser.__init__(self, **kwargs) + + +class ExtendOption(Option): + """from http://docs.python.org/library/optparse.html?highlight=optparse#adding-new-actions""" + + ACTIONS = Option.ACTIONS + ("extend",) + STORE_ACTIONS = Option.STORE_ACTIONS + ("extend",) + TYPED_ACTIONS = Option.TYPED_ACTIONS + ("extend",) + ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + ("extend",) + + def take_action(self, action, dest, opt, value, values, parser): + if action == "extend": + lvalue = value.split(",") + values.ensure_value(dest, []).extend(lvalue) + else: + Option.take_action(self, action, dest, opt, value, values, parser) + + +def make_immutable(item): + if isinstance(item, list) or isinstance(item, tuple): + result = LockedTuple(item) + elif isinstance(item, dict): + result = ReadOnlyDict(item) + result.lock() + else: + result = item + return result + + +class LockedTuple(tuple): + def __new__(cls, items): + return tuple.__new__(cls, (make_immutable(x) for x in items)) + + def __deepcopy__(self, memo): + return [deepcopy(elem, memo) for elem in self] + + +# ReadOnlyDict {{{1 +class ReadOnlyDict(dict): + def __init__(self, dictionary): + self._lock = False + self.update(dictionary.copy()) + + def _check_lock(self): + assert not self._lock, "ReadOnlyDict is locked!" + + def lock(self): + for (k, v) in list(self.items()): + self[k] = make_immutable(v) + self._lock = True + + def __setitem__(self, *args): + self._check_lock() + return dict.__setitem__(self, *args) + + def __delitem__(self, *args): + self._check_lock() + return dict.__delitem__(self, *args) + + def clear(self, *args): + self._check_lock() + return dict.clear(self, *args) + + def pop(self, *args): + self._check_lock() + return dict.pop(self, *args) + + def popitem(self, *args): + self._check_lock() + return dict.popitem(self, *args) + + def setdefault(self, *args): + self._check_lock() + return dict.setdefault(self, *args) + + def update(self, *args): + self._check_lock() + dict.update(self, *args) + + def __deepcopy__(self, memo): + cls = self.__class__ + result = cls.__new__(cls) + memo[id(self)] = result + for k, v in list(self.__dict__.items()): + setattr(result, k, deepcopy(v, memo)) + result._lock = False + for k, v in list(self.items()): + result[k] = deepcopy(v, memo) + return result + + +DEFAULT_CONFIG_PATH = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))), + "configs", +) + + +# parse_config_file {{{1 +def parse_config_file( + file_name, quiet=False, search_path=None, config_dict_name="config" +): + """Read a config file and return a dictionary.""" + file_path = None + if os.path.exists(file_name): + file_path = file_name + else: + if not search_path: + search_path = [".", DEFAULT_CONFIG_PATH] + for path in search_path: + if os.path.exists(os.path.join(path, file_name)): + file_path = os.path.join(path, file_name) + break + else: + raise IOError("Can't find %s in %s!" % (file_name, search_path)) + if file_name.endswith(".py"): + global_dict = {} + local_dict = {} + exec( + compile(open(file_path, "rb").read(), file_path, "exec"), + global_dict, + local_dict, + ) + config = local_dict[config_dict_name] + elif file_name.endswith(".json"): + fh = open(file_path) + config = {} + json_config = json.load(fh) + config = dict(json_config) + fh.close() + else: + raise RuntimeError( + "Unknown config file type %s! (config files must end in .json or .py)" + % file_name + ) + # TODO return file_path + return config + + +def download_config_file(url, file_name): + n = 0 + attempts = 5 + sleeptime = 60 + max_sleeptime = 5 * 60 + while True: + if n >= attempts: + print( + "Failed to download from url %s after %d attempts, quiting..." + % (url, attempts) + ) + raise SystemError(-1) + try: + contents = urlopen(url, timeout=30).read() + break + except URLError as e: + print("Error downloading from url %s: %s" % (url, str(e))) + except socket.timeout as e: + print("Time out accessing %s: %s" % (url, str(e))) + except socket.error as e: + print("Socket error when accessing %s: %s" % (url, str(e))) + print("Sleeping %d seconds before retrying" % sleeptime) + time.sleep(sleeptime) + sleeptime = sleeptime * 2 + if sleeptime > max_sleeptime: + sleeptime = max_sleeptime + n += 1 + + try: + f = open(file_name, "w") + f.write(contents) + f.close() + except IOError as e: + print("Error writing downloaded contents to file %s: %s" % (file_name, str(e))) + raise SystemError(-1) + + +# BaseConfig {{{1 +class BaseConfig(object): + """Basic config setting/getting.""" + + def __init__( + self, + config=None, + initial_config_file=None, + config_options=None, + all_actions=None, + default_actions=None, + volatile_config=None, + option_args=None, + require_config_file=False, + append_env_variables_from_configs=False, + usage="usage: %prog [options]", + ): + self._config = {} + self.all_cfg_files_and_dicts = [] + self.actions = [] + self.config_lock = False + self.require_config_file = require_config_file + # It allows to append env variables from multiple config files + self.append_env_variables_from_configs = append_env_variables_from_configs + + if all_actions: + self.all_actions = all_actions[:] + else: + self.all_actions = ["clobber", "build"] + if default_actions: + self.default_actions = default_actions[:] + else: + self.default_actions = self.all_actions[:] + if volatile_config is None: + self.volatile_config = { + "actions": None, + "add_actions": None, + "no_actions": None, + } + else: + self.volatile_config = deepcopy(volatile_config) + + if config: + self.set_config(config) + if initial_config_file: + initial_config = parse_config_file(initial_config_file) + self.all_cfg_files_and_dicts.append((initial_config_file, initial_config)) + self.set_config(initial_config) + # Since initial_config_file is only set when running unit tests, + # if no option_args have been specified, then the parser will + # parse sys.argv which in this case would be the command line + # options specified to run the tests, e.g. nosetests -v. Clearly, + # the options passed to nosetests (such as -v) should not be + # interpreted by mozharness as mozharness options, so we specify + # a dummy command line with no options, so that the parser does + # not add anything from the test invocation command line + # arguments to the mozharness options. + if option_args is None: + option_args = [ + "dummy_mozharness_script_with_no_command_line_options.py" + ] + if config_options is None: + config_options = [] + self._create_config_parser(config_options, usage) + # we allow manually passing of option args for things like nosetests + self.parse_args(args=option_args) + + def get_read_only_config(self): + return ReadOnlyDict(self._config) + + def _create_config_parser(self, config_options, usage): + self.config_parser = ExtendedOptionParser(usage=usage) + self.config_parser.add_option( + "--work-dir", + action="store", + dest="work_dir", + type="string", + default="build", + help="Specify the work_dir (subdir of base_work_dir)", + ) + self.config_parser.add_option( + "--base-work-dir", + action="store", + dest="base_work_dir", + type="string", + default=os.getcwd(), + help="Specify the absolute path of the parent of the working directory", + ) + self.config_parser.add_option( + "--extra-config-path", + action="extend", + dest="config_paths", + type="string", + help="Specify additional paths to search for config files.", + ) + self.config_parser.add_option( + "-c", + "--config-file", + "--cfg", + action="extend", + dest="config_files", + default=[], + type="string", + help="Specify a config file; can be repeated", + ) + self.config_parser.add_option( + "-C", + "--opt-config-file", + "--opt-cfg", + action="extend", + dest="opt_config_files", + type="string", + default=[], + help="Specify an optional config file, like --config-file but with no " + "error if the file is missing; can be repeated", + ) + self.config_parser.add_option( + "--dump-config", + action="store_true", + dest="dump_config", + help="List and dump the config generated from this run to " "a JSON file.", + ) + self.config_parser.add_option( + "--dump-config-hierarchy", + action="store_true", + dest="dump_config_hierarchy", + help="Like --dump-config but will list and dump which config " + "files were used making up the config and specify their own " + "keys/values that were not overwritten by another cfg -- " + "held the highest hierarchy.", + ) + self.config_parser.add_option( + "--append-env-variables-from-configs", + action="store_true", + dest="append_env_variables_from_configs", + help="Merge environment variables from config files.", + ) + + # Logging + log_option_group = OptionGroup(self.config_parser, "Logging") + log_option_group.add_option( + "--log-level", + action="store", + type="choice", + dest="log_level", + default=INFO, + choices=[DEBUG, INFO, WARNING, ERROR, CRITICAL, FATAL], + help="Set log level (debug|info|warning|error|critical|fatal)", + ) + log_option_group.add_option( + "-q", + "--quiet", + action="store_false", + dest="log_to_console", + default=True, + help="Don't log to the console", + ) + log_option_group.add_option( + "--append-to-log", + action="store_true", + dest="append_to_log", + default=False, + help="Append to the log", + ) + log_option_group.add_option( + "--multi-log", + action="store_const", + const="multi", + dest="log_type", + help="Log using MultiFileLogger", + ) + log_option_group.add_option( + "--simple-log", + action="store_const", + const="simple", + dest="log_type", + help="Log using SimpleFileLogger", + ) + self.config_parser.add_option_group(log_option_group) + + # Actions + action_option_group = OptionGroup( + self.config_parser, + "Actions", + "Use these options to list or enable/disable actions.", + ) + action_option_group.add_option( + "--list-actions", + action="store_true", + dest="list_actions", + help="List all available actions, then exit", + ) + action_option_group.add_option( + "--add-action", + action="extend", + dest="add_actions", + metavar="ACTIONS", + help="Add action %s to the list of actions" % self.all_actions, + ) + action_option_group.add_option( + "--no-action", + action="extend", + dest="no_actions", + metavar="ACTIONS", + help="Don't perform action", + ) + action_option_group.add_option( + "--requires-gpu", + action="store_true", + dest="requires_gpu", + default=False, + help="Indicates if the task requires gpu. ", + ) + for action in self.all_actions: + action_option_group.add_option( + "--%s" % action, + action="append_const", + dest="actions", + const=action, + help="Add %s to the limited list of actions" % action, + ) + action_option_group.add_option( + "--no-%s" % action, + action="append_const", + dest="no_actions", + const=action, + help="Remove %s from the list of actions to perform" % action, + ) + self.config_parser.add_option_group(action_option_group) + # Child-specified options + # TODO error checking for overlapping options + if config_options: + for option in config_options: + self.config_parser.add_option(*option[0], **option[1]) + + # Initial-config-specified options + config_options = self._config.get("config_options", None) + if config_options: + for option in config_options: + self.config_parser.add_option(*option[0], **option[1]) + + def set_config(self, config, overwrite=False): + """This is probably doable some other way.""" + if self._config and not overwrite: + self._config.update(config) + else: + self._config = config + return self._config + + def get_actions(self): + return self.actions + + def verify_actions(self, action_list, quiet=False): + for action in action_list: + if action not in self.all_actions: + if not quiet: + print("Invalid action %s not in %s!" % (action, self.all_actions)) + raise SystemExit(-1) + return action_list + + def verify_actions_order(self, action_list): + try: + indexes = [self.all_actions.index(elt) for elt in action_list] + sorted_indexes = sorted(indexes) + for i in range(len(indexes)): + if indexes[i] != sorted_indexes[i]: + print( + ("Action %s comes in different order in %s\n" + "than in %s") + % (action_list[i], action_list, self.all_actions) + ) + raise SystemExit(-1) + except ValueError as e: + print("Invalid action found: " + str(e)) + raise SystemExit(-1) + + def list_actions(self): + print("Actions available:") + for a in self.all_actions: + print(" " + ("*" if a in self.default_actions else " "), a) + raise SystemExit(0) + + def get_cfgs_from_files(self, all_config_files, options): + """Returns the configuration derived from the list of configuration + files. The result is represented as a list of `(filename, + config_dict)` tuples; they will be combined with keys in later + dictionaries taking precedence over earlier. + + `all_config_files` is all files specified with `--config-file` and + `--opt-config-file`; `options` is the argparse options object giving + access to any other command-line options. + + This function is also responsible for downloading any configuration + files specified by URL. It uses ``parse_config_file`` in this module + to parse individual files. + + This method can be overridden in a subclass to add extra logic to the + way that self.config is made up. See + `mozharness.mozilla.building.buildbase.BuildingConfig` for an example. + """ + config_paths = options.config_paths or ["."] + all_cfg_files_and_dicts = [] + for cf in all_config_files: + try: + if "://" in cf: # config file is an url + file_name = os.path.basename(cf) + file_path = os.path.join(os.getcwd(), file_name) + download_config_file(cf, file_path) + all_cfg_files_and_dicts.append( + ( + file_path, + parse_config_file( + file_path, + search_path=["."], + ), + ) + ) + else: + all_cfg_files_and_dicts.append( + ( + cf, + parse_config_file( + cf, + search_path=config_paths + [DEFAULT_CONFIG_PATH], + ), + ) + ) + except Exception: + if cf in options.opt_config_files: + print("WARNING: optional config file not found %s" % cf) + else: + raise + + if "EXTRA_MOZHARNESS_CONFIG" in os.environ: + env_config = json.loads(os.environ["EXTRA_MOZHARNESS_CONFIG"]) + all_cfg_files_and_dicts.append(("[EXTRA_MOZHARENSS_CONFIG]", env_config)) + + return all_cfg_files_and_dicts + + def parse_args(self, args=None): + """Parse command line arguments in a generic way. + Return the parser object after adding the basic options, so + child objects can manipulate it. + """ + self.command_line = " ".join(sys.argv) + if args is None: + args = sys.argv[1:] + (options, args) = self.config_parser.parse_args(args) + + defaults = self.config_parser.defaults.copy() + + if not options.config_files: + if self.require_config_file: + if options.list_actions: + self.list_actions() + print("Required config file not set! (use --config-file option)") + raise SystemExit(-1) + + os.environ["REQUIRE_GPU"] = "0" + if options.requires_gpu: + os.environ["REQUIRE_GPU"] = "1" + + # this is what get_cfgs_from_files returns. It will represent each + # config file name and its assoctiated dict + # eg ('builds/branch_specifics.py', {'foo': 'bar'}) + # let's store this to self for things like --interpret-config-files + self.all_cfg_files_and_dicts.extend( + self.get_cfgs_from_files( + # append opt_config to allow them to overwrite previous configs + options.config_files + options.opt_config_files, + options=options, + ) + ) + config = {} + if ( + self.append_env_variables_from_configs + or options.append_env_variables_from_configs + ): + # We only append values from various configs for the 'env' entry + # For everything else we follow the standard behaviour + for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts): + for v in list(c_dict.keys()): + if v == "env" and v in config: + config[v].update(c_dict[v]) + else: + config[v] = c_dict[v] + else: + for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts): + config.update(c_dict) + # assign or update self._config depending on if it exists or not + # NOTE self._config will be passed to ReadOnlyConfig's init -- a + # dict subclass with immutable locking capabilities -- and serve + # as the keys/values that make up that instance. Ultimately, + # this becomes self.config during BaseScript's init + self.set_config(config) + + for key in list(defaults.keys()): + value = getattr(options, key) + if value is None: + continue + # Don't override config_file defaults with config_parser defaults + if key in defaults and value == defaults[key] and key in self._config: + continue + self._config[key] = value + + # The idea behind the volatile_config is we don't want to save this + # info over multiple runs. This defaults to the action-specific + # config options, but can be anything. + for key in list(self.volatile_config.keys()): + if self._config.get(key) is not None: + self.volatile_config[key] = self._config[key] + del self._config[key] + + self.update_actions() + if options.list_actions: + self.list_actions() + + # Keep? This is for saving the volatile config in the dump_config + self._config["volatile_config"] = self.volatile_config + + self.options = options + self.args = args + return (self.options, self.args) + + def update_actions(self): + """Update actions after reading in config. + + Seems a little complex, but the logic goes: + + First, if default_actions is specified in the config, set our + default actions even if the script specifies other default actions. + + Without any other action-specific options, run with default actions. + + If we specify --ACTION or --only-ACTION once or multiple times, + we want to override the default_actions list with the one(s) we list. + + Otherwise, if we specify --add-action ACTION, we want to add an + action to the list. + + Finally, if we specify --no-ACTION, remove that from the list of + actions to perform. + """ + if self._config.get("default_actions"): + default_actions = self.verify_actions(self._config["default_actions"]) + self.default_actions = default_actions + self.verify_actions_order(self.default_actions) + self.actions = self.default_actions[:] + if self.volatile_config["actions"]: + actions = self.verify_actions(self.volatile_config["actions"]) + self.actions = actions + elif self.volatile_config["add_actions"]: + actions = self.verify_actions(self.volatile_config["add_actions"]) + self.actions.extend(actions) + if self.volatile_config["no_actions"]: + actions = self.verify_actions(self.volatile_config["no_actions"]) + for action in actions: + if action in self.actions: + self.actions.remove(action) + + +# __main__ {{{1 +if __name__ == "__main__": + pass |