diff options
Diffstat (limited to '')
58 files changed, 4637 insertions, 0 deletions
diff --git a/testing/mozbase/manifestparser/manifestparser/__init__.py b/testing/mozbase/manifestparser/manifestparser/__init__.py new file mode 100644 index 0000000000..c8d19d9712 --- /dev/null +++ b/testing/mozbase/manifestparser/manifestparser/__init__.py @@ -0,0 +1,8 @@ +# flake8: noqa +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from .expression import * +from .ini import * +from .manifestparser import * diff --git a/testing/mozbase/manifestparser/manifestparser/cli.py b/testing/mozbase/manifestparser/manifestparser/cli.py new file mode 100644 index 0000000000..2fefca33e7 --- /dev/null +++ b/testing/mozbase/manifestparser/manifestparser/cli.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Mozilla universal manifest parser +""" +import os +import sys +from optparse import OptionParser + +from .manifestparser import ManifestParser, convert + + +class ParserError(Exception): + """error for exceptions while parsing the command line""" + + +def parse_args(_args): + """ + parse and return: + --keys=value (or --key value) + -tags + args + """ + + # return values + _dict = {} + tags = [] + args = [] + + # parse the arguments + key = None + for arg in _args: + if arg.startswith("---"): + raise ParserError("arguments should start with '-' or '--' only") + elif arg.startswith("--"): + if key: + raise ParserError("Key %s still open" % key) + key = arg[2:] + if "=" in key: + key, value = key.split("=", 1) + _dict[key] = value + key = None + continue + elif arg.startswith("-"): + if key: + raise ParserError("Key %s still open" % key) + tags.append(arg[1:]) + continue + else: + if key: + _dict[key] = arg + continue + args.append(arg) + + # return values + return (_dict, tags, args) + + +class CLICommand(object): + usage = "%prog [options] command" + + def __init__(self, parser): + self._parser = parser # master parser + + def parser(self): + return OptionParser( + usage=self.usage, description=self.__doc__, add_help_option=False + ) + + +class Copy(CLICommand): + usage = "%prog [options] copy manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ..." + + def __call__(self, options, args): + # parse the arguments + try: + kwargs, tags, args = parse_args(args) + except ParserError as e: + self._parser.error(str(e)) + + # make sure we have some manifests, otherwise it will + # be quite boring + if not len(args) == 2: + HelpCLI(self._parser)(options, ["copy"]) + return + + # read the manifests + # TODO: should probably ensure these exist here + manifests = ManifestParser() + manifests.read(args[0]) + + # print the resultant query + manifests.copy(args[1], None, *tags, **kwargs) + + +class CreateCLI(CLICommand): + """ + create a manifest from a list of directories + """ + + usage = "%prog [options] create directory <directory> <...>" + + def parser(self): + parser = CLICommand.parser(self) + parser.add_option( + "-p", "--pattern", dest="pattern", help="glob pattern for files" + ) + parser.add_option( + "-i", + "--ignore", + dest="ignore", + default=[], + action="append", + help="directories to ignore", + ) + parser.add_option( + "-w", + "--in-place", + dest="in_place", + help="Write .ini files in place; filename to write to", + ) + return parser + + def __call__(self, _options, args): + parser = self.parser() + options, args = parser.parse_args(args) + + # need some directories + if not len(args): + parser.print_usage() + return + + # add the directories to the manifest + for arg in args: + assert os.path.exists(arg) + assert os.path.isdir(arg) + manifest = convert( + args, + pattern=options.pattern, + ignore=options.ignore, + write=options.in_place, + ) + if manifest: + print(manifest) + + +class WriteCLI(CLICommand): + """ + write a manifest based on a query + """ + + usage = "%prog [options] write manifest <manifest> -tag1 -tag2 --key1=value1 --key2=value2 ..." + + def __call__(self, options, args): + + # parse the arguments + try: + kwargs, tags, args = parse_args(args) + except ParserError as e: + self._parser.error(str(e)) + + # make sure we have some manifests, otherwise it will + # be quite boring + if not args: + HelpCLI(self._parser)(options, ["write"]) + return + + # read the manifests + # TODO: should probably ensure these exist here + manifests = ManifestParser() + manifests.read(*args) + + # print the resultant query + manifests.write(global_tags=tags, global_kwargs=kwargs) + + +class HelpCLI(CLICommand): + """ + get help on a command + """ + + usage = "%prog [options] help [command]" + + def __call__(self, options, args): + if len(args) == 1 and args[0] in commands: + commands[args[0]](self._parser).parser().print_help() + else: + self._parser.print_help() + print("\nCommands:") + for command in sorted(commands): + print(" %s : %s" % (command, commands[command].__doc__.strip())) + + +class UpdateCLI(CLICommand): + """ + update the tests as listed in a manifest from a directory + """ + + usage = "%prog [options] update manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ..." + + def __call__(self, options, args): + # parse the arguments + try: + kwargs, tags, args = parse_args(args) + except ParserError as e: + self._parser.error(str(e)) + + # make sure we have some manifests, otherwise it will + # be quite boring + if not len(args) == 2: + HelpCLI(self._parser)(options, ["update"]) + return + + # read the manifests + # TODO: should probably ensure these exist here + manifests = ManifestParser() + manifests.read(args[0]) + + # print the resultant query + manifests.update(args[1], None, *tags, **kwargs) + + +# command -> class mapping +commands = { + "create": CreateCLI, + "help": HelpCLI, + "update": UpdateCLI, + "write": WriteCLI, +} + + +def main(args=sys.argv[1:]): + """console_script entry point""" + + # set up an option parser + usage = "%prog [options] [command] ..." + description = "%s. Use `help` to display commands" % __doc__.strip() + parser = OptionParser(usage=usage, description=description) + parser.add_option( + "-s", + "--strict", + dest="strict", + action="store_true", + default=False, + help="adhere strictly to errors", + ) + parser.disable_interspersed_args() + + options, args = parser.parse_args(args) + + if not args: + HelpCLI(parser)(options, args) + parser.exit() + + # get the command + command = args[0] + if command not in commands: + parser.error( + "Command must be one of %s (you gave '%s')" + % (", ".join(sorted(commands.keys())), command) + ) + + handler = commands[command](parser) + handler(options, args[1:]) + + +if __name__ == "__main__": + main() diff --git a/testing/mozbase/manifestparser/manifestparser/expression.py b/testing/mozbase/manifestparser/manifestparser/expression.py new file mode 100644 index 0000000000..6ac02bb22a --- /dev/null +++ b/testing/mozbase/manifestparser/manifestparser/expression.py @@ -0,0 +1,329 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import re +import sys +import traceback + +import six + +__all__ = ["parse", "ParseError", "ExpressionParser"] + +# expr.py +# from: +# http://k0s.org/mozilla/hg/expressionparser +# http://hg.mozilla.org/users/tmielczarek_mozilla.com/expressionparser + +# Implements a top-down parser/evaluator for simple boolean expressions. +# ideas taken from http://effbot.org/zone/simple-top-down-parsing.htm +# +# Rough grammar: +# expr := literal +# | '(' expr ')' +# | expr '&&' expr +# | expr '||' expr +# | expr '==' expr +# | expr '!=' expr +# | expr '<' expr +# | expr '>' expr +# | expr '<=' expr +# | expr '>=' expr +# literal := BOOL +# | INT +# | STRING +# | IDENT +# BOOL := true|false +# INT := [0-9]+ +# STRING := "[^"]*" +# IDENT := [A-Za-z_]\w* + +# Identifiers take their values from a mapping dictionary passed as the second +# argument. + +# Glossary (see above URL for details): +# - nud: null denotation +# - led: left detonation +# - lbp: left binding power +# - rbp: right binding power + + +class ident_token(object): + def __init__(self, scanner, value): + self.value = value + + def nud(self, parser): + # identifiers take their value from the value mappings passed + # to the parser + return parser.value(self.value) + + +class literal_token(object): + def __init__(self, scanner, value): + self.value = value + + def nud(self, parser): + return self.value + + +class eq_op_token(object): + "==" + + def led(self, parser, left): + return left == parser.expression(self.lbp) + + +class neq_op_token(object): + "!=" + + def led(self, parser, left): + return left != parser.expression(self.lbp) + + +class lt_op_token(object): + "<" + + def led(self, parser, left): + return left < parser.expression(self.lbp) + + +class gt_op_token(object): + ">" + + def led(self, parser, left): + return left > parser.expression(self.lbp) + + +class le_op_token(object): + "<=" + + def led(self, parser, left): + return left <= parser.expression(self.lbp) + + +class ge_op_token(object): + ">=" + + def led(self, parser, left): + return left >= parser.expression(self.lbp) + + +class not_op_token(object): + "!" + + def nud(self, parser): + return not parser.expression(100) + + +class and_op_token(object): + "&&" + + def led(self, parser, left): + right = parser.expression(self.lbp) + return left and right + + +class or_op_token(object): + "||" + + def led(self, parser, left): + right = parser.expression(self.lbp) + return left or right + + +class lparen_token(object): + "(" + + def nud(self, parser): + expr = parser.expression() + parser.advance(rparen_token) + return expr + + +class rparen_token(object): + ")" + + +class end_token(object): + """always ends parsing""" + + +# derived literal tokens + + +class bool_token(literal_token): + def __init__(self, scanner, value): + value = {"true": True, "false": False}[value] + literal_token.__init__(self, scanner, value) + + +class int_token(literal_token): + def __init__(self, scanner, value): + literal_token.__init__(self, scanner, int(value)) + + +class string_token(literal_token): + def __init__(self, scanner, value): + literal_token.__init__(self, scanner, value[1:-1]) + + +precedence = [ + (end_token, rparen_token), + (or_op_token,), + (and_op_token,), + (lt_op_token, gt_op_token, le_op_token, ge_op_token, eq_op_token, neq_op_token), + (lparen_token,), +] +for index, rank in enumerate(precedence): + for token in rank: + token.lbp = index # lbp = lowest left binding power + + +class ParseError(Exception): + """error parsing conditional expression""" + + +class ExpressionParser(object): + """ + A parser for a simple expression language. + + The expression language can be described as follows:: + + EXPRESSION ::= LITERAL | '(' EXPRESSION ')' | '!' EXPRESSION | EXPRESSION OP EXPRESSION + OP ::= '==' | '!=' | '<' | '>' | '<=' | '>=' | '&&' | '||' + LITERAL ::= BOOL | INT | IDENT | STRING + BOOL ::= 'true' | 'false' + INT ::= [0-9]+ + IDENT ::= [a-zA-Z_]\w* + STRING ::= '"' [^\"] '"' | ''' [^\'] ''' + + At its core, expressions consist of booleans, integers, identifiers and. + strings. Booleans are one of *true* or *false*. Integers are a series + of digits. Identifiers are a series of English letters and underscores. + Strings are a pair of matching quote characters (single or double) with + zero or more characters inside. + + Expressions can be combined with operators: the equals (==) and not + equals (!=) operators compare two expressions and produce a boolean. The + and (&&) and or (||) operators take two expressions and produce the logical + AND or OR value of them, respectively. An expression can also be prefixed + with the not (!) operator, which produces its logical negation. + + Finally, any expression may be contained within parentheses for grouping. + + Identifiers take their values from the mapping provided. + """ + + scanner = None + + def __init__(self, text, valuemapping, strict=False): + """ + Initialize the parser + :param text: The expression to parse as a string. + :param valuemapping: A dict mapping identifier names to values. + :param strict: If true, referencing an identifier that was not + provided in :valuemapping: will raise an error. + """ + self.text = text + self.valuemapping = valuemapping + self.strict = strict + + def _tokenize(self): + """ + Lex the input text into tokens and yield them in sequence. + """ + if not ExpressionParser.scanner: + ExpressionParser.scanner = re.Scanner( + [ + # Note: keep these in sync with the class docstring above. + (r"true|false", bool_token), + (r"[a-zA-Z_]\w*", ident_token), + (r"[0-9]+", int_token), + (r'("[^"]*")|(\'[^\']*\')', string_token), + (r"==", eq_op_token()), + (r"!=", neq_op_token()), + (r"<=", le_op_token()), + (r">=", ge_op_token()), + (r"<", lt_op_token()), + (r">", gt_op_token()), + (r"\|\|", or_op_token()), + (r"!", not_op_token()), + (r"&&", and_op_token()), + (r"\(", lparen_token()), + (r"\)", rparen_token()), + (r"\s+", None), # skip whitespace + ] + ) + tokens, remainder = ExpressionParser.scanner.scan(self.text) + for t in tokens: + yield t + yield end_token() + + def value(self, ident): + """ + Look up the value of |ident| in the value mapping passed in the + constructor. + """ + if self.strict: + return self.valuemapping[ident] + else: + return self.valuemapping.get(ident, "") + + def advance(self, expected): + """ + Assert that the next token is an instance of |expected|, and advance + to the next token. + """ + if not isinstance(self.token, expected): + raise Exception("Unexpected token!") + self.token = six.next(self.iter) + + def expression(self, rbp=0): + """ + Parse and return the value of an expression until a token with + right binding power greater than rbp is encountered. + """ + t = self.token + self.token = six.next(self.iter) + left = t.nud(self) + while rbp < self.token.lbp: + t = self.token + self.token = six.next(self.iter) + left = t.led(self, left) + return left + + def parse(self): + """ + Parse and return the value of the expression in the text + passed to the constructor. Raises a ParseError if the expression + could not be parsed. + """ + try: + self.iter = self._tokenize() + self.token = six.next(self.iter) + return self.expression() + except Exception: + extype, ex, tb = sys.exc_info() + formatted = "".join(traceback.format_exception_only(extype, ex)) + six.reraise( + ParseError, + ParseError( + "could not parse: %s\nexception: %svariables: %s" + % (self.text, formatted, self.valuemapping) + ), + tb, + ) + + __call__ = parse + + +def parse(text, **values): + """ + Parse and evaluate a boolean expression. + :param text: The expression to parse, as a string. + :param values: A dict containing a name to value mapping for identifiers + referenced in *text*. + :rtype: the final value of the expression. + :raises: :py:exc::ParseError: will be raised if parsing fails. + """ + return ExpressionParser(text, values).parse() diff --git a/testing/mozbase/manifestparser/manifestparser/filters.py b/testing/mozbase/manifestparser/manifestparser/filters.py new file mode 100644 index 0000000000..36a1fd2edc --- /dev/null +++ b/testing/mozbase/manifestparser/manifestparser/filters.py @@ -0,0 +1,567 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +A filter is a callable that accepts an iterable of test objects and a +dictionary of values, and returns a new iterable of test objects. It is +possible to define custom filters if the built-in ones are not enough. +""" + +import itertools +import os +from collections import defaultdict +from collections.abc import MutableSequence + +import six +from six import string_types + +from .expression import ParseError, parse +from .util import normsep + +logger = None + + +def log(msg, level="info"): + from mozlog import get_default_logger + + global logger + if not logger: + logger = get_default_logger(component="manifestparser") + if logger: + getattr(logger, level)(msg) + + +# built-in filters + + +def _match(exprs, **values): + if any(parse(e, **values) for e in exprs.splitlines() if e): + return True + return False + + +def skip_if(tests, values): + """ + Sets disabled on all tests containing the `skip-if` tag and whose condition + is True. This filter is added by default. + """ + tag = "skip-if" + for test in tests: + if tag in test and _match(test[tag], **values): + test.setdefault("disabled", "{}: {}".format(tag, test[tag])) + yield test + + +def run_if(tests, values): + """ + Sets disabled on all tests containing the `run-if` tag and whose condition + is False. This filter is added by default. + """ + tag = "run-if" + for test in tests: + if tag in test and not _match(test[tag], **values): + test.setdefault("disabled", "{}: {}".format(tag, test[tag])) + yield test + + +def fail_if(tests, values): + """ + Sets expected to 'fail' on all tests containing the `fail-if` tag and whose + condition is True. This filter is added by default. + """ + tag = "fail-if" + for test in tests: + if tag in test and _match(test[tag], **values): + test["expected"] = "fail" + yield test + + +def enabled(tests, values): + """ + Removes all tests containing the `disabled` key. This filter can be + added by passing `disabled=False` into `active_tests`. + """ + for test in tests: + if "disabled" not in test: + yield test + + +def exists(tests, values): + """ + Removes all tests that do not exist on the file system. This filter is + added by default, but can be removed by passing `exists=False` into + `active_tests`. + """ + for test in tests: + if os.path.exists(test["path"]): + yield test + + +# built-in instance filters + + +class InstanceFilter(object): + """ + Generally only one instance of a class filter should be applied at a time. + Two instances of `InstanceFilter` are considered equal if they have the + same class name. This ensures only a single instance is ever added to + `filterlist`. This class also formats filters' __str__ method for easier + debugging. + """ + + unique = True + + __hash__ = super.__hash__ + + def __init__(self, *args, **kwargs): + self.fmt_args = ", ".join( + itertools.chain( + [str(a) for a in args], + ["{}={}".format(k, v) for k, v in six.iteritems(kwargs)], + ) + ) + + def __eq__(self, other): + if self.unique: + return self.__class__ == other.__class__ + return self.__hash__() == other.__hash__() + + def __str__(self): + return "{}({})".format(self.__class__.__name__, self.fmt_args) + + +class subsuite(InstanceFilter): + """ + If `name` is None, removes all tests that have a `subsuite` key. + Otherwise removes all tests that do not have a subsuite matching `name`. + + It is possible to specify conditional subsuite keys using: + subsuite = foo,condition + + where 'foo' is the subsuite name, and 'condition' is the same type of + condition used for skip-if. If the condition doesn't evaluate to true, + the subsuite designation will be removed from the test. + + :param name: The name of the subsuite to run (default None) + """ + + def __init__(self, name=None): + InstanceFilter.__init__(self, name=name) + self.name = name + + def __call__(self, tests, values): + # Look for conditional subsuites, and replace them with the subsuite + # itself (if the condition is true), or nothing. + for test in tests: + subsuite = test.get("subsuite", "") + if "," in subsuite: + try: + subsuite, cond = subsuite.split(",") + except ValueError: + raise ParseError("subsuite condition can't contain commas") + matched = parse(cond, **values) + if matched: + test["subsuite"] = subsuite + else: + test["subsuite"] = "" + + # Filter on current subsuite + if self.name is None: + if not test.get("subsuite"): + yield test + else: + if test.get("subsuite", "") == self.name: + yield test + + +class chunk_by_slice(InstanceFilter): + """ + Basic chunking algorithm that splits tests evenly across total chunks. + + :param this_chunk: the current chunk, 1 <= this_chunk <= total_chunks + :param total_chunks: the total number of chunks + :param disabled: Whether to include disabled tests in the chunking + algorithm. If False, each chunk contains an equal number + of non-disabled tests. If True, each chunk contains an + equal number of tests (default False) + """ + + def __init__(self, this_chunk, total_chunks, disabled=False): + assert 1 <= this_chunk <= total_chunks + InstanceFilter.__init__(self, this_chunk, total_chunks, disabled=disabled) + self.this_chunk = this_chunk + self.total_chunks = total_chunks + self.disabled = disabled + + def __call__(self, tests, values): + tests = list(tests) + if self.disabled: + chunk_tests = tests[:] + else: + chunk_tests = [t for t in tests if "disabled" not in t] + + tests_per_chunk = float(len(chunk_tests)) / self.total_chunks + # pylint: disable=W1633 + start = int(round((self.this_chunk - 1) * tests_per_chunk)) + end = int(round(self.this_chunk * tests_per_chunk)) + + if not self.disabled: + # map start and end back onto original list of tests. Disabled + # tests will still be included in the returned list, but each + # chunk will contain an equal number of enabled tests. + if self.this_chunk == 1: + start = 0 + elif start < len(chunk_tests): + start = tests.index(chunk_tests[start]) + + if self.this_chunk == self.total_chunks: + end = len(tests) + elif end < len(chunk_tests): + end = tests.index(chunk_tests[end]) + return (t for t in tests[start:end]) + + +class chunk_by_dir(InstanceFilter): + """ + Basic chunking algorithm that splits directories of tests evenly at a + given depth. + + For example, a depth of 2 means all test directories two path nodes away + from the base are gathered, then split evenly across the total number of + chunks. The number of tests in each of the directories is not taken into + account (so chunks will not contain an even number of tests). All test + paths must be relative to the same root (typically the root of the source + repository). + + :param this_chunk: the current chunk, 1 <= this_chunk <= total_chunks + :param total_chunks: the total number of chunks + :param depth: the minimum depth of a subdirectory before it will be + considered unique + """ + + def __init__(self, this_chunk, total_chunks, depth): + InstanceFilter.__init__(self, this_chunk, total_chunks, depth) + self.this_chunk = this_chunk + self.total_chunks = total_chunks + self.depth = depth + + def __call__(self, tests, values): + tests_by_dir = defaultdict(list) + ordered_dirs = [] + for test in tests: + path = test["relpath"] + + if path.startswith(os.sep): + path = path[1:] + + dirs = path.split(os.sep) + dirs = dirs[: min(self.depth, len(dirs) - 1)] + path = os.sep.join(dirs) + + # don't count directories that only have disabled tests in them, + # but still yield disabled tests that are alongside enabled tests + if path not in ordered_dirs and "disabled" not in test: + ordered_dirs.append(path) + tests_by_dir[path].append(test) + + # pylint: disable=W1633 + tests_per_chunk = float(len(ordered_dirs)) / self.total_chunks + start = int(round((self.this_chunk - 1) * tests_per_chunk)) + end = int(round(self.this_chunk * tests_per_chunk)) + + for i in range(start, end): + for test in tests_by_dir.pop(ordered_dirs[i]): + yield test + + # find directories that only contain disabled tests. They still need to + # be yielded for reporting purposes. Put them all in chunk 1 for + # simplicity. + if self.this_chunk == 1: + disabled_dirs = [ + v for k, v in six.iteritems(tests_by_dir) if k not in ordered_dirs + ] + for disabled_test in itertools.chain(*disabled_dirs): + yield disabled_test + + +class chunk_by_manifest(InstanceFilter): + """ + Chunking algorithm that tries to evenly distribute tests while ensuring + tests in the same manifest stay together. + + :param this_chunk: the current chunk, 1 <= this_chunk <= total_chunks + :param total_chunks: the total number of chunks + """ + + def __init__(self, this_chunk, total_chunks, *args, **kwargs): + InstanceFilter.__init__(self, this_chunk, total_chunks, *args, **kwargs) + self.this_chunk = this_chunk + self.total_chunks = total_chunks + + def __call__(self, tests, values): + tests = list(tests) + manifests = set(t["manifest"] for t in tests) + + tests_by_manifest = [] + for manifest in manifests: + mtests = [t for t in tests if t["manifest"] == manifest] + tests_by_manifest.append(mtests) + # Sort tests_by_manifest from largest manifest to shortest; include + # manifest name as secondary key to ensure consistent order across + # multiple runs. + tests_by_manifest.sort(reverse=True, key=lambda x: (len(x), x[0]["manifest"])) + + tests_by_chunk = [[] for i in range(self.total_chunks)] + for batch in tests_by_manifest: + # Sort to guarantee the chunk with the lowest score will always + # get the next batch of tests. + tests_by_chunk.sort( + key=lambda x: (len(x), x[0]["manifest"] if len(x) else "") + ) + tests_by_chunk[0].extend(batch) + + return (t for t in tests_by_chunk[self.this_chunk - 1]) + + +class chunk_by_runtime(InstanceFilter): + """ + Chunking algorithm that attempts to group tests into chunks based on their + average runtimes. It keeps manifests of tests together and pairs slow + running manifests with fast ones. + + :param this_chunk: the current chunk, 1 <= this_chunk <= total_chunks + :param total_chunks: the total number of chunks + :param runtimes: dictionary of manifest runtime data, of the form + {<manifest path>: <average runtime>} + """ + + def __init__(self, this_chunk, total_chunks, runtimes): + InstanceFilter.__init__(self, this_chunk, total_chunks, runtimes) + self.this_chunk = this_chunk + self.total_chunks = total_chunks + self.runtimes = {normsep(m): r for m, r in runtimes.items()} + + @classmethod + def get_manifest(cls, test): + manifest = normsep(test.get("ancestor_manifest", "")) + + # Ignore ancestor_manifests that live at the root (e.g, don't have a + # path separator). The only time this should happen is when they are + # generated by the build system and we shouldn't count generated + # manifests for chunking purposes. + if not manifest or "/" not in manifest: + manifest = normsep(test["manifest_relpath"]) + return manifest + + def get_chunked_manifests(self, manifests): + # Find runtimes for all relevant manifests. + runtimes = [(self.runtimes[m], m) for m in manifests if m in self.runtimes] + + # Compute the average to use as a default for manifests that don't exist. + times = [r[0] for r in runtimes] + # pylint --py3k W1619 + # pylint: disable=W1633 + avg = round(sum(times) / len(times), 2) if times else 0 + missing = sorted([m for m in manifests if m not in self.runtimes]) + log( + "Applying average runtime of {}s to the following missing manifests:\n{}".format( + avg, " " + "\n ".join(missing) + ) + ) + runtimes.extend([(avg, m) for m in missing]) + + # Each chunk is of the form [<runtime>, <manifests>]. + chunks = [[0, []] for i in range(self.total_chunks)] + + # Sort runtimes from slowest -> fastest. + for runtime, manifest in sorted(runtimes, reverse=True): + # Sort chunks from fastest -> slowest. This guarantees the fastest + # chunk will be assigned the slowest remaining manifest. + chunks.sort(key=lambda x: (x[0], len(x[1]), x[1])) + chunks[0][0] += runtime + chunks[0][1].append(manifest) + + # Sort one last time so we typically get chunks ordered from fastest to + # slowest. + chunks.sort(key=lambda x: (x[0], len(x[1]))) + return chunks + + def __call__(self, tests, values): + tests = list(tests) + manifests = set(self.get_manifest(t) for t in tests) + chunks = self.get_chunked_manifests(manifests) + runtime, this_manifests = chunks[self.this_chunk - 1] + # pylint --py3k W1619 + # pylint: disable=W1633 + log( + "Cumulative test runtime is around {} minutes (average is {} minutes)".format( + round(runtime / 60), + round(sum([c[0] for c in chunks]) / (60 * len(chunks))), + ) + ) + return (t for t in tests if self.get_manifest(t) in this_manifests) + + +class tags(InstanceFilter): + """ + Removes tests that don't contain any of the given tags. This overrides + InstanceFilter's __eq__ method, so multiple instances can be added. + Multiple tag filters is equivalent to joining tags with the AND operator. + + To define a tag in a manifest, add a `tags` attribute to a test or DEFAULT + section. Tests can have multiple tags, in which case they should be + whitespace delimited. For example: + + [test_foobar.html] + tags = foo bar + + :param tags: A tag or list of tags to filter tests on + """ + + unique = False + + def __init__(self, tags): + InstanceFilter.__init__(self, tags) + if isinstance(tags, string_types): + tags = [tags] + self.tags = tags + + def __call__(self, tests, values): + for test in tests: + if "tags" not in test: + continue + + test_tags = [t.strip() for t in test["tags"].split()] + if any(t in self.tags for t in test_tags): + yield test + + +class failures(InstanceFilter): + """ + [test_foobar.html] + fail-if = + keyword # <comment> + + :param keywords: A keyword to filter tests on + """ + + def __init__(self, keyword): + InstanceFilter.__init__(self, keyword) + self.keyword = keyword + + def __call__(self, tests, values): + for test in tests: + for key in ["skip-if", "fail-if"]: + if key not in test: + continue + + matched = [ + self.keyword in e and parse(e, **values) + for e in test[key].splitlines() + if e + ] + if any(matched): + test["expected"] = "fail" + yield test + + +class pathprefix(InstanceFilter): + """ + Removes tests that don't start with any of the given test paths. + + :param paths: A list of test paths (or manifests) to filter on + """ + + def __init__(self, paths): + InstanceFilter.__init__(self, paths) + if isinstance(paths, string_types): + paths = [paths] + self.paths = paths + self.missing = set() + + def __call__(self, tests, values): + seen = set() + for test in tests: + for tp in self.paths: + tp = os.path.normpath(tp) + + if tp.endswith(".ini"): + mpaths = [test["manifest_relpath"]] + if "ancestor_manifest" in test: + mpaths.append(test["ancestor_manifest"]) + + if os.path.isabs(tp): + root = test["manifest"][: -len(test["manifest_relpath"]) - 1] + mpaths = [os.path.join(root, m) for m in mpaths] + + # only return tests that are in this manifest + if not any(os.path.normpath(m) == tp for m in mpaths): + continue + else: + # only return tests that start with this path + path = test["relpath"] + if os.path.isabs(tp): + path = test["path"] + + if not os.path.normpath(path).startswith(tp): + continue + + # any test path that points to a single file will be run no + # matter what, even if it's disabled + if "disabled" in test and os.path.normpath(test["relpath"]) == tp: + del test["disabled"] + + seen.add(tp) + yield test + break + + self.missing = set(self.paths) - seen + + +# filter container + +DEFAULT_FILTERS = ( + skip_if, + run_if, + fail_if, +) +""" +By default :func:`~.active_tests` will run the :func:`~.skip_if`, +:func:`~.run_if` and :func:`~.fail_if` filters. +""" + + +class filterlist(MutableSequence): + """ + A MutableSequence that raises TypeError when adding a non-callable and + ValueError if the item is already added. + """ + + def __init__(self, items=None): + self.items = [] + if items: + self.items = list(items) + + def _validate(self, item): + if not callable(item): + raise TypeError("Filters must be callable!") + if item in self: + raise ValueError("Filter {} is already applied!".format(item)) + + def __getitem__(self, key): + return self.items[key] + + def __setitem__(self, key, value): + self._validate(value) + self.items[key] = value + + def __delitem__(self, key): + del self.items[key] + + def __len__(self): + return len(self.items) + + def insert(self, index, value): + self._validate(value) + self.items.insert(index, value) diff --git a/testing/mozbase/manifestparser/manifestparser/ini.py b/testing/mozbase/manifestparser/manifestparser/ini.py new file mode 100644 index 0000000000..338b04d58f --- /dev/null +++ b/testing/mozbase/manifestparser/manifestparser/ini.py @@ -0,0 +1,184 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import io +import os +import sys + +from six import string_types + +__all__ = ["read_ini", "combine_fields"] + + +class IniParseError(Exception): + def __init__(self, fp, linenum, msg): + if isinstance(fp, string_types): + path = fp + elif hasattr(fp, "name"): + path = fp.name + else: + path = getattr(fp, "path", "unknown") + msg = "Error parsing manifest file '{}', line {}: {}".format(path, linenum, msg) + super(IniParseError, self).__init__(msg) + + +def read_ini( + fp, + defaults=None, + default="DEFAULT", + comments=None, + separators=None, + strict=True, + handle_defaults=True, +): + """ + read an .ini file and return a list of [(section, values)] + - fp : file pointer or path to read + - defaults : default set of variables + - default : name of the section for the default section + - comments : characters that if they start a line denote a comment + - separators : strings that denote key, value separation in order + - strict : whether to be strict about parsing + - handle_defaults : whether to incorporate defaults into each section + """ + + # variables + defaults = defaults or {} + default_section = {} + comments = comments or ("#",) + separators = separators or ("=", ":") + sections = [] + key = value = None + section_names = set() + if isinstance(fp, string_types): + fp = io.open(fp, encoding="utf-8") + + # read the lines + for (linenum, line) in enumerate(fp.read().splitlines(), start=1): + + stripped = line.strip() + + # ignore blank lines + if not stripped: + # reset key and value to avoid continuation lines + key = value = None + continue + + # ignore comment lines + if any(stripped.startswith(c) for c in comments): + continue + + # strip inline comments (borrowed from configparser) + comment_start = sys.maxsize + inline_prefixes = {p: -1 for p in comments} + while comment_start == sys.maxsize and inline_prefixes: + next_prefixes = {} + for prefix, index in inline_prefixes.items(): + index = stripped.find(prefix, index + 1) + if index == -1: + continue + next_prefixes[prefix] = index + if index == 0 or (index > 0 and stripped[index - 1].isspace()): + comment_start = min(comment_start, index) + inline_prefixes = next_prefixes + + if comment_start != sys.maxsize: + stripped = stripped[:comment_start].rstrip() + + # check for a new section + if len(stripped) > 2 and stripped[0] == "[" and stripped[-1] == "]": + section = stripped[1:-1].strip() + key = value = key_indent = None + + # deal with DEFAULT section + if section.lower() == default.lower(): + if strict: + assert default not in section_names + section_names.add(default) + current_section = default_section + continue + + if strict: + # make sure this section doesn't already exist + assert ( + section not in section_names + ), "Section '%s' already found in '%s'" % (section, section_names) + + section_names.add(section) + current_section = {} + sections.append((section, current_section)) + continue + + # if there aren't any sections yet, something bad happen + if not section_names: + raise IniParseError( + fp, + linenum, + "Expected a comment or section, " "instead found '{}'".format(stripped), + ) + + # continuation line ? + line_indent = len(line) - len(line.lstrip(" ")) + if key and line_indent > key_indent: + value = "%s%s%s" % (value, os.linesep, stripped) + current_section[key] = value + continue + + # (key, value) pair + for separator in separators: + if separator in stripped: + key, value = stripped.split(separator, 1) + key = key.strip() + value = value.strip() + key_indent = line_indent + + # make sure this key isn't already in the section + if key: + assert ( + key not in current_section + ), f"Found duplicate key {key} in section {section}" + + if strict: + # make sure this key isn't empty + assert key + + current_section[key] = value + break + else: + # something bad happened! + raise IniParseError(fp, linenum, "Unexpected line '{}'".format(stripped)) + + # merge global defaults with the DEFAULT section + defaults = combine_fields(defaults, default_section) + if handle_defaults: + # merge combined defaults into each section + sections = [(i, combine_fields(defaults, j)) for i, j in sections] + return sections, defaults + + +def combine_fields(global_vars, local_vars): + """ + Combine the given manifest entries according to the semantics of specific fields. + This is used to combine manifest level defaults with a per-test definition. + """ + if not global_vars: + return local_vars + if not local_vars: + return global_vars.copy() + field_patterns = { + "args": "%s %s", + "prefs": "%s %s", + "skip-if": "%s\n%s", + "support-files": "%s %s", + } + final_mapping = global_vars.copy() + for field_name, value in local_vars.items(): + if field_name not in field_patterns or field_name not in global_vars: + final_mapping[field_name] = value + continue + global_value = global_vars[field_name] + pattern = field_patterns[field_name] + final_mapping[field_name] = pattern % (global_value, value) + + return final_mapping diff --git a/testing/mozbase/manifestparser/manifestparser/manifestparser.py b/testing/mozbase/manifestparser/manifestparser/manifestparser.py new file mode 100644 index 0000000000..c9a5df3872 --- /dev/null +++ b/testing/mozbase/manifestparser/manifestparser/manifestparser.py @@ -0,0 +1,875 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import codecs +import fnmatch +import io +import json +import os +import shutil +import sys +import types + +from six import StringIO, string_types + +from .filters import DEFAULT_FILTERS, enabled +from .filters import exists as _exists +from .filters import filterlist +from .ini import read_ini + +__all__ = ["ManifestParser", "TestManifest", "convert"] + +relpath = os.path.relpath + + +# path normalization + + +def normalize_path(path): + """normalize a relative path""" + if sys.platform.startswith("win"): + return path.replace("/", os.path.sep) + return path + + +def denormalize_path(path): + """denormalize a relative path""" + if sys.platform.startswith("win"): + return path.replace(os.path.sep, "/") + return path + + +# objects for parsing manifests + + +class ManifestParser(object): + """read .ini manifests""" + + def __init__( + self, + manifests=(), + defaults=None, + strict=True, + rootdir=None, + finder=None, + handle_defaults=True, + ): + """Creates a ManifestParser from the given manifest files. + + :param manifests: An iterable of file paths or file objects corresponding + to manifests. If a file path refers to a manifest file that + does not exist, an IOError is raised. + :param defaults: Variables to pre-define in the environment for evaluating + expressions in manifests. + :param strict: If False, the provided manifests may contain references to + listed (test) files that do not exist without raising an + IOError during reading, and certain errors in manifests + are not considered fatal. Those errors include duplicate + section names, redefining variables, and defining empty + variables. + :param rootdir: The directory used as the basis for conversion to and from + relative paths during manifest reading. + :param finder: If provided, this finder object will be used for filesystem + interactions. Finder objects are part of the mozpack package, + documented at + http://firefox-source-docs.mozilla.org/python/mozpack.html#module-mozpack.files + :param handle_defaults: If not set, do not propagate manifest defaults to individual + test objects. Callers are expected to manage per-manifest + defaults themselves via the manifest_defaults member + variable in this case. + """ + self._defaults = defaults or {} + self.tests = [] + self.manifest_defaults = {} + self.source_files = set() + self.strict = strict + self.rootdir = rootdir + self._root = None + self.finder = finder + self._handle_defaults = handle_defaults + if manifests: + self.read(*manifests) + + def path_exists(self, path): + if self.finder: + return self.finder.get(path) is not None + return os.path.exists(path) + + @property + def root(self): + if not self._root: + if self.rootdir is None: + self._root = "" + else: + assert os.path.isabs(self.rootdir) + self._root = self.rootdir + os.path.sep + return self._root + + def relative_to_root(self, path): + # Microoptimization, because relpath is quite expensive. + # We know that rootdir is an absolute path or empty. If path + # starts with rootdir, then path is also absolute and the tail + # of the path is the relative path (possibly non-normalized, + # when here is unknown). + # For this to work rootdir needs to be terminated with a path + # separator, so that references to sibling directories with + # a common prefix don't get misscomputed (e.g. /root and + # /rootbeer/file). + # When the rootdir is unknown, the relpath needs to be left + # unchanged. We use an empty string as rootdir in that case, + # which leaves relpath unchanged after slicing. + if path.startswith(self.root): + return path[len(self.root) :] + else: + return relpath(path, self.root) + + # methods for reading manifests + + def _read(self, root, filename, defaults, parentmanifest=None): + """ + Internal recursive method for reading and parsing manifests. + Stores all found tests in self.tests + :param root: The base path + :param filename: File object or string path for the base manifest file + :param defaults: Options that apply to all items + :param parentmanifest: Filename of the parent manifest, relative to rootdir (default None) + """ + + def read_file(type): + include_file = section.split(type, 1)[-1] + include_file = normalize_path(include_file) + if not os.path.isabs(include_file): + include_file = os.path.join(here, include_file) + if not self.path_exists(include_file): + message = "Included file '%s' does not exist" % include_file + if self.strict: + raise IOError(message) + else: + sys.stderr.write("%s\n" % message) + return + return include_file + + # get directory of this file if not file-like object + if isinstance(filename, string_types): + # If we're using mercurial as our filesystem via a finder + # during manifest reading, the getcwd() calls that happen + # with abspath calls will not be meaningful, so absolute + # paths are required. + if self.finder: + assert os.path.isabs(filename) + filename = os.path.abspath(filename) + filename_rel = self.relative_to_root(filename) + self.source_files.add(filename) + if self.finder: + fp = codecs.getreader("utf-8")(self.finder.get(filename).open()) + else: + fp = io.open(filename, encoding="utf-8") + here = os.path.dirname(filename) + else: + fp = filename + filename = here = None + filename_rel = None + defaults["here"] = here + + # read the configuration + sections, defaults = read_ini( + fp=fp, + defaults=defaults, + strict=self.strict, + handle_defaults=self._handle_defaults, + ) + if parentmanifest and filename: + # A manifest can be read multiple times, via "include:", optionally + # with section-specific variables. These variables only apply to + # the included manifest when included via the same parent manifest, + # so they must be associated with (parentmanifest, filename). + # + # |defaults| is a combination of variables, in the following order: + # - The defaults of the ancestor manifests if self._handle_defaults + # is True. + # - Any variables from the "[include:...]" section. + # - The defaults of the included manifest. + self.manifest_defaults[(parentmanifest, filename)] = defaults + else: + self.manifest_defaults[filename] = defaults + + # get the tests + for section, data in sections: + # a file to include + # TODO: keep track of included file structure: + # self.manifests = {'manifest.ini': 'relative/path.ini'} + if section.startswith("include:"): + include_file = read_file("include:") + if include_file: + include_defaults = data.copy() + self._read( + root, + include_file, + include_defaults, + parentmanifest=filename_rel, + ) + continue + + # otherwise an item + test = data.copy() + test["name"] = section + + # Will be None if the manifest being read is a file-like object. + test["manifest"] = filename + test["manifest_relpath"] = None + if filename: + test["manifest_relpath"] = filename_rel + + # determine the path + path = test.get("path", section) + _relpath = path + if "://" not in path: # don't futz with URLs + path = normalize_path(path) + if here and not os.path.isabs(path): + # Profiling indicates 25% of manifest parsing is spent + # in this call to normpath, but almost all calls return + # their argument unmodified, so we avoid the call if + # '..' if not present in the path. + path = os.path.join(here, path) + if ".." in path: + path = os.path.normpath(path) + _relpath = self.relative_to_root(path) + + test["path"] = path + test["relpath"] = _relpath + + if parentmanifest is not None: + # If a test was included by a parent manifest we may need to + # indicate that in the test object for the sake of identifying + # a test, particularly in the case a test file is included by + # multiple manifests. + test["ancestor_manifest"] = parentmanifest + + # append the item + self.tests.append(test) + + def read(self, *filenames, **defaults): + """ + read and add manifests from file paths or file-like objects + + filenames -- file paths or file-like objects to read as manifests + defaults -- default variables + """ + + # ensure all files exist + missing = [ + filename + for filename in filenames + if isinstance(filename, string_types) and not self.path_exists(filename) + ] + if missing: + raise IOError("Missing files: %s" % ", ".join(missing)) + + # default variables + _defaults = defaults.copy() or self._defaults.copy() + _defaults.setdefault("here", None) + + # process each file + for filename in filenames: + # set the per file defaults + defaults = _defaults.copy() + here = None + if isinstance(filename, string_types): + here = os.path.dirname(os.path.abspath(filename)) + defaults["here"] = here # directory of master .ini file + + if self.rootdir is None: + # set the root directory + # == the directory of the first manifest given + self.rootdir = here + + self._read(here, filename, defaults) + + # methods for querying manifests + + def query(self, *checks, **kw): + """ + general query function for tests + - checks : callable conditions to test if the test fulfills the query + """ + tests = kw.get("tests", None) + if tests is None: + tests = self.tests + retval = [] + for test in tests: + for check in checks: + if not check(test): + break + else: + retval.append(test) + return retval + + def get(self, _key=None, inverse=False, tags=None, tests=None, **kwargs): + # TODO: pass a dict instead of kwargs since you might hav + # e.g. 'inverse' as a key in the dict + + # TODO: tags should just be part of kwargs with None values + # (None == any is kinda weird, but probably still better) + + # fix up tags + if tags: + tags = set(tags) + else: + tags = set() + + # make some check functions + if inverse: + + def has_tags(test): + return not tags.intersection(test.keys()) + + def dict_query(test): + for key, value in list(kwargs.items()): + if test.get(key) == value: + return False + return True + + else: + + def has_tags(test): + return tags.issubset(test.keys()) + + def dict_query(test): + for key, value in list(kwargs.items()): + if test.get(key) != value: + return False + return True + + # query the tests + tests = self.query(has_tags, dict_query, tests=tests) + + # if a key is given, return only a list of that key + # useful for keys like 'name' or 'path' + if _key: + return [test[_key] for test in tests] + + # return the tests + return tests + + def manifests(self, tests=None): + """ + return manifests in order in which they appear in the tests + If |tests| is not set, the order of the manifests is unspecified. + """ + if tests is None: + manifests = [] + # Make sure to return all the manifests, even ones without tests. + for manifest in list(self.manifest_defaults.keys()): + if isinstance(manifest, tuple): + parentmanifest, manifest = manifest + if manifest not in manifests: + manifests.append(manifest) + return manifests + + manifests = [] + for test in tests: + manifest = test.get("manifest") + if not manifest: + continue + if manifest not in manifests: + manifests.append(manifest) + return manifests + + def paths(self): + return [i["path"] for i in self.tests] + + # methods for auditing + + def missing(self, tests=None): + """ + return list of tests that do not exist on the filesystem + """ + if tests is None: + tests = self.tests + existing = list(_exists(tests, {})) + return [t for t in tests if t not in existing] + + def check_missing(self, tests=None): + missing = self.missing(tests=tests) + if missing: + missing_paths = [test["path"] for test in missing] + if self.strict: + raise IOError( + "Strict mode enabled, test paths must exist. " + "The following test(s) are missing: %s" + % json.dumps(missing_paths, indent=2) + ) + print( + "Warning: The following test(s) are missing: %s" + % json.dumps(missing_paths, indent=2), + file=sys.stderr, + ) + return missing + + def verifyDirectory(self, directories, pattern=None, extensions=None): + """ + checks what is on the filesystem vs what is in a manifest + returns a 2-tuple of sets: + (missing_from_filesystem, missing_from_manifest) + """ + + files = set([]) + if isinstance(directories, string_types): + directories = [directories] + + # get files in directories + for directory in directories: + for dirpath, dirnames, filenames in os.walk(directory, topdown=True): + + # only add files that match a pattern + if pattern: + filenames = fnmatch.filter(filenames, pattern) + + # only add files that have one of the extensions + if extensions: + filenames = [ + filename + for filename in filenames + if os.path.splitext(filename)[-1] in extensions + ] + + files.update( + [os.path.join(dirpath, filename) for filename in filenames] + ) + + paths = set(self.paths()) + missing_from_filesystem = paths.difference(files) + missing_from_manifest = files.difference(paths) + return (missing_from_filesystem, missing_from_manifest) + + # methods for output + + def write( + self, + fp=sys.stdout, + rootdir=None, + global_tags=None, + global_kwargs=None, + local_tags=None, + local_kwargs=None, + ): + """ + write a manifest given a query + global and local options will be munged to do the query + globals will be written to the top of the file + locals (if given) will be written per test + """ + + # open file if `fp` given as string + close = False + if isinstance(fp, string_types): + fp = open(fp, "w") + close = True + + # root directory + if rootdir is None: + rootdir = self.rootdir + + # sanitize input + global_tags = global_tags or set() + local_tags = local_tags or set() + global_kwargs = global_kwargs or {} + local_kwargs = local_kwargs or {} + + # create the query + tags = set([]) + tags.update(global_tags) + tags.update(local_tags) + kwargs = {} + kwargs.update(global_kwargs) + kwargs.update(local_kwargs) + + # get matching tests + tests = self.get(tags=tags, **kwargs) + + # print the .ini manifest + if global_tags or global_kwargs: + print("[DEFAULT]", file=fp) + for tag in global_tags: + print("%s =" % tag, file=fp) + for key, value in list(global_kwargs.items()): + print("%s = %s" % (key, value), file=fp) + print(file=fp) + + for test in tests: + test = test.copy() # don't overwrite + + path = test["name"] + if not os.path.isabs(path): + path = test["path"] + if self.rootdir: + path = relpath(test["path"], self.rootdir) + path = denormalize_path(path) + print("[%s]" % path, file=fp) + + # reserved keywords: + reserved = [ + "path", + "name", + "here", + "manifest", + "manifest_relpath", + "relpath", + "ancestor_manifest", + ] + for key in sorted(test.keys()): + if key in reserved: + continue + if key in global_kwargs: + continue + if key in global_tags and not test[key]: + continue + print("%s = %s" % (key, test[key]), file=fp) + print(file=fp) + + if close: + # close the created file + fp.close() + + def __str__(self): + fp = StringIO() + self.write(fp=fp) + value = fp.getvalue() + return value + + def copy(self, directory, rootdir=None, *tags, **kwargs): + """ + copy the manifests and associated tests + - directory : directory to copy to + - rootdir : root directory to copy to (if not given from manifests) + - tags : keywords the tests must have + - kwargs : key, values the tests must match + """ + # XXX note that copy does *not* filter the tests out of the + # resulting manifest; it just stupidly copies them over. + # ideally, it would reread the manifests and filter out the + # tests that don't match *tags and **kwargs + + # destination + if not os.path.exists(directory): + os.path.makedirs(directory) + else: + # sanity check + assert os.path.isdir(directory) + + # tests to copy + tests = self.get(tags=tags, **kwargs) + if not tests: + return # nothing to do! + + # root directory + if rootdir is None: + rootdir = self.rootdir + + # copy the manifests + tests + manifests = [relpath(manifest, rootdir) for manifest in self.manifests()] + for manifest in manifests: + destination = os.path.join(directory, manifest) + dirname = os.path.dirname(destination) + if not os.path.exists(dirname): + os.makedirs(dirname) + else: + # sanity check + assert os.path.isdir(dirname) + shutil.copy(os.path.join(rootdir, manifest), destination) + + missing = self.check_missing(tests) + tests = [test for test in tests if test not in missing] + for test in tests: + if os.path.isabs(test["name"]): + continue + source = test["path"] + destination = os.path.join(directory, relpath(test["path"], rootdir)) + shutil.copy(source, destination) + # TODO: ensure that all of the tests are below the from_dir + + def update(self, from_dir, rootdir=None, *tags, **kwargs): + """ + update the tests as listed in a manifest from a directory + - from_dir : directory where the tests live + - rootdir : root directory to copy to (if not given from manifests) + - tags : keys the tests must have + - kwargs : key, values the tests must match + """ + + # get the tests + tests = self.get(tags=tags, **kwargs) + + # get the root directory + if not rootdir: + rootdir = self.rootdir + + # copy them! + for test in tests: + if not os.path.isabs(test["name"]): + _relpath = relpath(test["path"], rootdir) + source = os.path.join(from_dir, _relpath) + if not os.path.exists(source): + message = "Missing test: '%s' does not exist!" + if self.strict: + raise IOError(message) + print(message + " Skipping.", file=sys.stderr) + continue + destination = os.path.join(rootdir, _relpath) + shutil.copy(source, destination) + + # directory importers + + @classmethod + def _walk_directories(cls, directories, callback, pattern=None, ignore=()): + """ + internal function to import directories + """ + + if isinstance(pattern, string_types): + patterns = [pattern] + else: + patterns = pattern + ignore = set(ignore) + + if not patterns: + + def accept_filename(filename): + return True + + else: + + def accept_filename(filename): + for pattern in patterns: + if fnmatch.fnmatch(filename, pattern): + return True + + if not ignore: + + def accept_dirname(dirname): + return True + + else: + + def accept_dirname(dirname): + return dirname not in ignore + + rootdirectories = directories[:] + seen_directories = set() + for rootdirectory in rootdirectories: + # let's recurse directories using list + directories = [os.path.realpath(rootdirectory)] + while directories: + directory = directories.pop(0) + if directory in seen_directories: + # eliminate possible infinite recursion due to + # symbolic links + continue + seen_directories.add(directory) + + files = [] + subdirs = [] + for name in sorted(os.listdir(directory)): + path = os.path.join(directory, name) + if os.path.isfile(path): + # os.path.isfile follow symbolic links, we don't + # need to handle them here. + if accept_filename(name): + files.append(name) + continue + elif os.path.islink(path): + # eliminate symbolic links + path = os.path.realpath(path) + + # we must have a directory here + if accept_dirname(name): + subdirs.append(name) + # this subdir is added for recursion + directories.insert(0, path) + + # here we got all subdirs and files filtered, we can + # call the callback function if directory is not empty + if subdirs or files: + callback(rootdirectory, directory, subdirs, files) + + @classmethod + def populate_directory_manifests( + cls, directories, filename, pattern=None, ignore=(), overwrite=False + ): + """ + walks directories and writes manifests of name `filename` in-place; + returns `cls` instance populated with the given manifests + + filename -- filename of manifests to write + pattern -- shell pattern (glob) or patterns of filenames to match + ignore -- directory names to ignore + overwrite -- whether to overwrite existing files of given name + """ + + manifest_dict = {} + + if os.path.basename(filename) != filename: + raise IOError("filename should not include directory name") + + # no need to hit directories more than once + _directories = directories + directories = [] + for directory in _directories: + if directory not in directories: + directories.append(directory) + + def callback(directory, dirpath, dirnames, filenames): + """write a manifest for each directory""" + + manifest_path = os.path.join(dirpath, filename) + if (dirnames or filenames) and not ( + os.path.exists(manifest_path) and overwrite + ): + with open(manifest_path, "w") as manifest: + for dirname in dirnames: + print( + "[include:%s]" % os.path.join(dirname, filename), + file=manifest, + ) + for _filename in filenames: + print("[%s]" % _filename, file=manifest) + + # add to list of manifests + manifest_dict.setdefault(directory, manifest_path) + + # walk the directories to gather files + cls._walk_directories(directories, callback, pattern=pattern, ignore=ignore) + # get manifests + manifests = [manifest_dict[directory] for directory in _directories] + + # create a `cls` instance with the manifests + return cls(manifests=manifests) + + @classmethod + def from_directories( + cls, directories, pattern=None, ignore=(), write=None, relative_to=None + ): + """ + convert directories to a simple manifest; returns ManifestParser instance + + pattern -- shell pattern (glob) or patterns of filenames to match + ignore -- directory names to ignore + write -- filename or file-like object of manifests to write; + if `None` then a StringIO instance will be created + relative_to -- write paths relative to this path; + if false then the paths are absolute + """ + + # determine output + opened_manifest_file = None # name of opened manifest file + absolute = not relative_to # whether to output absolute path names as names + if isinstance(write, string_types): + opened_manifest_file = write + write = open(write, "w") + if write is None: + write = StringIO() + + # walk the directories, generating manifests + def callback(directory, dirpath, dirnames, filenames): + + # absolute paths + filenames = [os.path.join(dirpath, filename) for filename in filenames] + # ensure new manifest isn't added + filenames = [ + filename for filename in filenames if filename != opened_manifest_file + ] + # normalize paths + if not absolute and relative_to: + filenames = [relpath(filename, relative_to) for filename in filenames] + + # write to manifest + write_content = "\n".join( + ["[{}]".format(denormalize_path(filename)) for filename in filenames] + ) + print(write_content, file=write) + + cls._walk_directories(directories, callback, pattern=pattern, ignore=ignore) + + if opened_manifest_file: + # close file + write.close() + manifests = [opened_manifest_file] + else: + # manifests/write is a file-like object; + # rewind buffer + write.flush() + write.seek(0) + manifests = [write] + + # make a ManifestParser instance + return cls(manifests=manifests) + + +convert = ManifestParser.from_directories + + +class TestManifest(ManifestParser): + """ + apply logic to manifests; this is your integration layer :) + specific harnesses may subclass from this if they need more logic + """ + + def __init__(self, *args, **kwargs): + ManifestParser.__init__(self, *args, **kwargs) + self.filters = filterlist(DEFAULT_FILTERS) + self.last_used_filters = [] + + def active_tests( + self, exists=True, disabled=True, filters=None, noDefaultFilters=False, **values + ): + """ + Run all applied filters on the set of tests. + + :param exists: filter out non-existing tests (default True) + :param disabled: whether to return disabled tests (default True) + :param values: keys and values to filter on (e.g. `os = linux mac`) + :param filters: list of filters to apply to the tests + :returns: list of test objects that were not filtered out + """ + tests = [i.copy() for i in self.tests] # shallow copy + + # mark all tests as passing + for test in tests: + test["expected"] = test.get("expected", "pass") + + # make a copy so original doesn't get modified + if noDefaultFilters: + fltrs = [] + else: + fltrs = self.filters[:] + + if exists: + if self.strict: + self.check_missing(tests) + else: + fltrs.append(_exists) + + if not disabled: + fltrs.append(enabled) + + if filters: + fltrs += filters + + self.last_used_filters = fltrs[:] + for fn in fltrs: + tests = fn(tests, values) + return list(tests) + + def test_paths(self): + return [test["path"] for test in self.active_tests()] + + def fmt_filters(self, filters=None): + filters = filters or self.last_used_filters + names = [] + for f in filters: + if isinstance(f, types.FunctionType): + names.append(f.__name__) + else: + names.append(str(f)) + return ", ".join(names) diff --git a/testing/mozbase/manifestparser/manifestparser/util.py b/testing/mozbase/manifestparser/manifestparser/util.py new file mode 100644 index 0000000000..4830b39547 --- /dev/null +++ b/testing/mozbase/manifestparser/manifestparser/util.py @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import ast +import os + + +def normsep(path): + """ + Normalize path separators, by using forward slashes instead of whatever + :py:const:`os.sep` is. + """ + if os.sep != "/": + # Python 2 is happy to do things like byte_string.replace(u'foo', + # u'bar'), but not Python 3. + if isinstance(path, bytes): + path = path.replace(os.sep.encode("ascii"), b"/") + else: + path = path.replace(os.sep, "/") + if os.altsep and os.altsep != "/": + if isinstance(path, bytes): + path = path.replace(os.altsep.encode("ascii"), b"/") + else: + path = path.replace(os.altsep, "/") + return path + + +def evaluate_list_from_string(list_string): + """ + This is a utility function for converting a string obtained from a manifest + into a list. If the string is not a valid list when converted, an error will be + raised from `ast.eval_literal`. For example, you can convert entries like this + into a list: + ``` + test_settings= + ["hello", "world"], + [1, 10, 100], + values= + 5, + 6, + 7, + 8, + ``` + """ + return ast.literal_eval("[" + "".join(list_string.strip(",").split("\n")) + "]") diff --git a/testing/mozbase/manifestparser/setup.py b/testing/mozbase/manifestparser/setup.py new file mode 100644 index 0000000000..1b907bd772 --- /dev/null +++ b/testing/mozbase/manifestparser/setup.py @@ -0,0 +1,37 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from setuptools import setup + +PACKAGE_NAME = "manifestparser" +PACKAGE_VERSION = "2.1.0" + +DEPS = [ + "mozlog >= 6.0", + "six >= 1.13.0", +] +setup( + name=PACKAGE_NAME, + version=PACKAGE_VERSION, + description="Library to create and manage test manifests", + long_description="see https://firefox-source-docs.mozilla.org/mozbase/index.html", + classifiers=[ + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + ], + # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + keywords="mozilla manifests", + author="Mozilla Automation and Testing Team", + author_email="tools@lists.mozilla.org", + url="https://wiki.mozilla.org/Auto-tools/Projects/Mozbase", + license="MPL", + zip_safe=False, + packages=["manifestparser"], + install_requires=DEPS, + entry_points=""" + [console_scripts] + manifestparser = manifestparser.cli:main + """, +) diff --git a/testing/mozbase/manifestparser/tests/comment-example.ini b/testing/mozbase/manifestparser/tests/comment-example.ini new file mode 100644 index 0000000000..ba03310e4f --- /dev/null +++ b/testing/mozbase/manifestparser/tests/comment-example.ini @@ -0,0 +1,11 @@ +# See https://bugzilla.mozilla.org/show_bug.cgi?id=813674 + +[test_0180_fileInUse_xp_win_complete.js] +[test_0181_fileInUse_xp_win_partial.js] +[test_0182_rmrfdirFileInUse_xp_win_complete.js] +[test_0183_rmrfdirFileInUse_xp_win_partial.js] +[test_0184_fileInUse_xp_win_complete.js] +[test_0185_fileInUse_xp_win_partial.js] +[test_0186_rmrfdirFileInUse_xp_win_complete.js] +[test_0187_rmrfdirFileInUse_xp_win_partial.js] +# [test_0202_app_launch_apply_update_dirlocked.js] # Test disabled, bug 757632 diff --git a/testing/mozbase/manifestparser/tests/default-skipif.ini b/testing/mozbase/manifestparser/tests/default-skipif.ini new file mode 100644 index 0000000000..d3c2687333 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/default-skipif.ini @@ -0,0 +1,22 @@ +[DEFAULT] +skip-if = os == 'win' && debug # a pesky comment + + +[test1] +skip-if = debug + +[test2] +skip-if = os == 'linux' + +[test3] +skip-if = os == 'win' + +[test4] +skip-if = os == 'win' && debug + +[test5] +foo = bar + +[test6] +skip-if = debug # a second pesky comment + diff --git a/testing/mozbase/manifestparser/tests/default-subsuite.ini b/testing/mozbase/manifestparser/tests/default-subsuite.ini new file mode 100644 index 0000000000..f2c2bbd3b1 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/default-subsuite.ini @@ -0,0 +1,5 @@ +[test1] +subsuite = baz + +[test2] +subsuite = foo diff --git a/testing/mozbase/manifestparser/tests/default-suppfiles.ini b/testing/mozbase/manifestparser/tests/default-suppfiles.ini new file mode 100644 index 0000000000..12af247b82 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/default-suppfiles.ini @@ -0,0 +1,9 @@ +[DEFAULT] +support-files = foo.js # a comment + +[test7] +[test8] +support-files = bar.js # another comment +[test9] +foo = bar + diff --git a/testing/mozbase/manifestparser/tests/filter-example.ini b/testing/mozbase/manifestparser/tests/filter-example.ini new file mode 100644 index 0000000000..13a8734c33 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/filter-example.ini @@ -0,0 +1,11 @@ +# illustrate test filters based on various categories + +[windowstest] +skip-if = os != 'win' + +[fleem] +skip-if = os == 'mac' + +[linuxtest] +skip-if = (os == 'mac') || (os == 'win') +fail-if = toolkit == 'cocoa' diff --git a/testing/mozbase/manifestparser/tests/fleem b/testing/mozbase/manifestparser/tests/fleem new file mode 100644 index 0000000000..744817b823 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/fleem @@ -0,0 +1 @@ +# dummy spot for "fleem" test diff --git a/testing/mozbase/manifestparser/tests/include-example.ini b/testing/mozbase/manifestparser/tests/include-example.ini new file mode 100644 index 0000000000..69e728c3bc --- /dev/null +++ b/testing/mozbase/manifestparser/tests/include-example.ini @@ -0,0 +1,11 @@ +[DEFAULT] +foo = bar + +[include:include/bar.ini] + +[fleem] + +[include:include/foo.ini] +red = roses +blue = violets +yellow = daffodils
\ No newline at end of file diff --git a/testing/mozbase/manifestparser/tests/include-invalid.ini b/testing/mozbase/manifestparser/tests/include-invalid.ini new file mode 100644 index 0000000000..e3ed0dd6b3 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/include-invalid.ini @@ -0,0 +1 @@ +[include:invalid.ini] diff --git a/testing/mozbase/manifestparser/tests/include/bar.ini b/testing/mozbase/manifestparser/tests/include/bar.ini new file mode 100644 index 0000000000..bcb312d1db --- /dev/null +++ b/testing/mozbase/manifestparser/tests/include/bar.ini @@ -0,0 +1,4 @@ +[DEFAULT] +foo = fleem + +[crash-handling]
\ No newline at end of file diff --git a/testing/mozbase/manifestparser/tests/include/crash-handling b/testing/mozbase/manifestparser/tests/include/crash-handling new file mode 100644 index 0000000000..8e19a63751 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/include/crash-handling @@ -0,0 +1 @@ +# dummy spot for "crash-handling" test diff --git a/testing/mozbase/manifestparser/tests/include/flowers b/testing/mozbase/manifestparser/tests/include/flowers new file mode 100644 index 0000000000..a25acfbe21 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/include/flowers @@ -0,0 +1 @@ +# dummy spot for "flowers" test diff --git a/testing/mozbase/manifestparser/tests/include/foo.ini b/testing/mozbase/manifestparser/tests/include/foo.ini new file mode 100644 index 0000000000..cfc90ace83 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/include/foo.ini @@ -0,0 +1,5 @@ +[DEFAULT] +blue = ocean + +[flowers] +yellow = submarine
\ No newline at end of file diff --git a/testing/mozbase/manifestparser/tests/just-defaults.ini b/testing/mozbase/manifestparser/tests/just-defaults.ini new file mode 100644 index 0000000000..83a0cec0c6 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/just-defaults.ini @@ -0,0 +1,2 @@ +[DEFAULT] +foo = bar diff --git a/testing/mozbase/manifestparser/tests/manifest.ini b/testing/mozbase/manifestparser/tests/manifest.ini new file mode 100644 index 0000000000..cd8852134c --- /dev/null +++ b/testing/mozbase/manifestparser/tests/manifest.ini @@ -0,0 +1,13 @@ +[DEFAULT] +subsuite = mozbase +[test_expressionparser.py] +[test_manifestparser.py] +[test_testmanifest.py] +[test_read_ini.py] +[test_convert_directory.py] +[test_filters.py] +[test_chunking.py] +[test_convert_symlinks.py] +disabled = https://bugzilla.mozilla.org/show_bug.cgi?id=920938 +[test_default_overrides.py] +[test_util.py] diff --git a/testing/mozbase/manifestparser/tests/missing-path.ini b/testing/mozbase/manifestparser/tests/missing-path.ini new file mode 100644 index 0000000000..919d8e04da --- /dev/null +++ b/testing/mozbase/manifestparser/tests/missing-path.ini @@ -0,0 +1,2 @@ +[foo] +[bar] diff --git a/testing/mozbase/manifestparser/tests/mozmill-example.ini b/testing/mozbase/manifestparser/tests/mozmill-example.ini new file mode 100644 index 0000000000..114cf48c4b --- /dev/null +++ b/testing/mozbase/manifestparser/tests/mozmill-example.ini @@ -0,0 +1,80 @@ +[testAddons/testDisableEnablePlugin.js] +[testAddons/testGetAddons.js] +[testAddons/testSearchAddons.js] +[testAwesomeBar/testAccessLocationBar.js] +[testAwesomeBar/testCheckItemHighlight.js] +[testAwesomeBar/testEscapeAutocomplete.js] +[testAwesomeBar/testFaviconInAutocomplete.js] +[testAwesomeBar/testGoButton.js] +[testAwesomeBar/testLocationBarSearches.js] +[testAwesomeBar/testPasteLocationBar.js] +[testAwesomeBar/testSuggestHistoryBookmarks.js] +[testAwesomeBar/testVisibleItemsMax.js] +[testBookmarks/testAddBookmarkToMenu.js] +[testCookies/testDisableCookies.js] +[testCookies/testEnableCookies.js] +[testCookies/testRemoveAllCookies.js] +[testCookies/testRemoveCookie.js] +[testDownloading/testCloseDownloadManager.js] +[testDownloading/testDownloadStates.js] +[testDownloading/testOpenDownloadManager.js] +[testFindInPage/testFindInPage.js] +[testFormManager/testAutoCompleteOff.js] +[testFormManager/testBasicFormCompletion.js] +[testFormManager/testClearFormHistory.js] +[testFormManager/testDisableFormManager.js] +[testGeneral/testGoogleSuggestions.js] +[testGeneral/testStopReloadButtons.js] +[testInstallation/testBreakpadInstalled.js] +[testLayout/testNavigateFTP.js] +[testPasswordManager/testPasswordNotSaved.js] +[testPasswordManager/testPasswordSavedAndDeleted.js] +[testPopups/testPopupsAllowed.js] +[testPopups/testPopupsBlocked.js] +[testPreferences/testPaneRetention.js] +[testPreferences/testPreferredLanguage.js] +[testPreferences/testRestoreHomepageToDefault.js] +[testPreferences/testSetToCurrentPage.js] +[testPreferences/testSwitchPanes.js] +[testPrivateBrowsing/testAboutPrivateBrowsing.js] +[testPrivateBrowsing/testCloseWindow.js] +[testPrivateBrowsing/testDisabledElements.js] +[testPrivateBrowsing/testDisabledPermissions.js] +[testPrivateBrowsing/testDownloadManagerClosed.js] +[testPrivateBrowsing/testGeolocation.js] +[testPrivateBrowsing/testStartStopPBMode.js] +[testPrivateBrowsing/testTabRestoration.js] +[testPrivateBrowsing/testTabsDismissedOnStop.js] +[testSearch/testAddMozSearchProvider.js] +[testSearch/testFocusAndSearch.js] +[testSearch/testGetMoreSearchEngines.js] +[testSearch/testOpenSearchAutodiscovery.js] +[testSearch/testRemoveSearchEngine.js] +[testSearch/testReorderSearchEngines.js] +[testSearch/testRestoreDefaults.js] +[testSearch/testSearchSelection.js] +[testSearch/testSearchSuggestions.js] +[testSecurity/testBlueLarry.js] +[testSecurity/testDefaultPhishingEnabled.js] +[testSecurity/testDefaultSecurityPrefs.js] +[testSecurity/testEncryptedPageWarning.js] +[testSecurity/testGreenLarry.js] +[testSecurity/testGreyLarry.js] +[testSecurity/testIdentityPopupOpenClose.js] +[testSecurity/testSSLDisabledErrorPage.js] +[testSecurity/testSafeBrowsingNotificationBar.js] +[testSecurity/testSafeBrowsingWarningPages.js] +[testSecurity/testSecurityInfoViaMoreInformation.js] +[testSecurity/testSecurityNotification.js] +[testSecurity/testSubmitUnencryptedInfoWarning.js] +[testSecurity/testUnknownIssuer.js] +[testSecurity/testUntrustedConnectionErrorPage.js] +[testSessionStore/testUndoTabFromContextMenu.js] +[testTabbedBrowsing/testBackgroundTabScrolling.js] +[testTabbedBrowsing/testCloseTab.js] +[testTabbedBrowsing/testNewTab.js] +[testTabbedBrowsing/testNewWindow.js] +[testTabbedBrowsing/testOpenInBackground.js] +[testTabbedBrowsing/testOpenInForeground.js] +[testTechnicalTools/testAccessPageInfoDialog.js] +[testToolbar/testBackForwardButtons.js] diff --git a/testing/mozbase/manifestparser/tests/mozmill-restart-example.ini b/testing/mozbase/manifestparser/tests/mozmill-restart-example.ini new file mode 100644 index 0000000000..2ce30e7c6a --- /dev/null +++ b/testing/mozbase/manifestparser/tests/mozmill-restart-example.ini @@ -0,0 +1,26 @@ +[DEFAULT] +type = restart + +[restartTests/testExtensionInstallUninstall/test2.js] +foo = bar + +[restartTests/testExtensionInstallUninstall/test1.js] +foo = baz + +[restartTests/testExtensionInstallUninstall/test3.js] +[restartTests/testSoftwareUpdateAutoProxy/test2.js] +[restartTests/testSoftwareUpdateAutoProxy/test1.js] +[restartTests/testPrimaryPassword/test1.js] +[restartTests/testExtensionInstallGetAddons/test2.js] +[restartTests/testExtensionInstallGetAddons/test1.js] +[restartTests/testMultipleExtensionInstallation/test2.js] +[restartTests/testMultipleExtensionInstallation/test1.js] +[restartTests/testThemeInstallUninstall/test2.js] +[restartTests/testThemeInstallUninstall/test1.js] +[restartTests/testThemeInstallUninstall/test3.js] +[restartTests/testDefaultBookmarks/test1.js] +[softwareUpdate/testFallbackUpdate/test2.js] +[softwareUpdate/testFallbackUpdate/test1.js] +[softwareUpdate/testFallbackUpdate/test3.js] +[softwareUpdate/testDirectUpdate/test2.js] +[softwareUpdate/testDirectUpdate/test1.js] diff --git a/testing/mozbase/manifestparser/tests/no-tests.ini b/testing/mozbase/manifestparser/tests/no-tests.ini new file mode 100644 index 0000000000..83a0cec0c6 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/no-tests.ini @@ -0,0 +1,2 @@ +[DEFAULT] +foo = bar diff --git a/testing/mozbase/manifestparser/tests/parent/include/first/manifest.ini b/testing/mozbase/manifestparser/tests/parent/include/first/manifest.ini new file mode 100644 index 0000000000..828525c18f --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/include/first/manifest.ini @@ -0,0 +1,3 @@ +[parent:../manifest.ini] + +[testFirst.js] diff --git a/testing/mozbase/manifestparser/tests/parent/include/manifest.ini b/testing/mozbase/manifestparser/tests/parent/include/manifest.ini new file mode 100644 index 0000000000..fb9756d6af --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/include/manifest.ini @@ -0,0 +1,8 @@ +[DEFAULT] +top = data + +[include:first/manifest.ini] +disabled = YES + +[include:second/manifest.ini] +disabled = NO diff --git a/testing/mozbase/manifestparser/tests/parent/include/second/manifest.ini b/testing/mozbase/manifestparser/tests/parent/include/second/manifest.ini new file mode 100644 index 0000000000..31f0537566 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/include/second/manifest.ini @@ -0,0 +1,3 @@ +[parent:../manifest.ini] + +[testSecond.js] diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/level_1.ini b/testing/mozbase/manifestparser/tests/parent/level_1/level_1.ini new file mode 100644 index 0000000000..ac7c370c3e --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/level_1.ini @@ -0,0 +1,5 @@ +[DEFAULT] +x = level_1 + +[test_1] +[test_2] diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_2.ini b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_2.ini new file mode 100644 index 0000000000..ada6a510d7 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_2.ini @@ -0,0 +1,3 @@ +[parent:../level_1.ini] + +[test_2] diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3.ini b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3.ini new file mode 100644 index 0000000000..2edd647fcc --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3.ini @@ -0,0 +1,3 @@ +[parent:../level_2.ini] + +[test_3] diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3_default.ini b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3_default.ini new file mode 100644 index 0000000000..d6aae60ae1 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3_default.ini @@ -0,0 +1,6 @@ +[parent:../level_2.ini] + +[DEFAULT] +x = level_3 + +[test_3] diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/test_3 b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/test_3 new file mode 100644 index 0000000000..f5de587529 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/test_3 @@ -0,0 +1 @@ +# dummy spot for "test_3" test diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/level_2/test_2 b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/test_2 new file mode 100644 index 0000000000..5b77e04f31 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/test_2 @@ -0,0 +1 @@ +# dummy spot for "test_2" test diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/test_1 b/testing/mozbase/manifestparser/tests/parent/level_1/test_1 new file mode 100644 index 0000000000..dccbf04e4d --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/test_1 @@ -0,0 +1 @@ +# dummy spot for "test_1" test diff --git a/testing/mozbase/manifestparser/tests/parent/root/dummy b/testing/mozbase/manifestparser/tests/parent/root/dummy new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/root/dummy diff --git a/testing/mozbase/manifestparser/tests/path-example.ini b/testing/mozbase/manifestparser/tests/path-example.ini new file mode 100644 index 0000000000..366782d95d --- /dev/null +++ b/testing/mozbase/manifestparser/tests/path-example.ini @@ -0,0 +1,2 @@ +[foo] +path = fleem
\ No newline at end of file diff --git a/testing/mozbase/manifestparser/tests/relative-path.ini b/testing/mozbase/manifestparser/tests/relative-path.ini new file mode 100644 index 0000000000..57105489ba --- /dev/null +++ b/testing/mozbase/manifestparser/tests/relative-path.ini @@ -0,0 +1,5 @@ +[foo] +path = ../fleem + +[bar] +path = ../testsSIBLING/example diff --git a/testing/mozbase/manifestparser/tests/subsuite.ini b/testing/mozbase/manifestparser/tests/subsuite.ini new file mode 100644 index 0000000000..c1a70bd44e --- /dev/null +++ b/testing/mozbase/manifestparser/tests/subsuite.ini @@ -0,0 +1,13 @@ +[test1] +subsuite=bar,foo=="bar" # this has a comment + +[test2] +subsuite=bar,foo=="bar" + +[test3] +subsuite=baz + +[test4] +[test5] +[test6] +subsuite=bar,foo=="szy" || foo=="bar"
\ No newline at end of file diff --git a/testing/mozbase/manifestparser/tests/test_chunking.py b/testing/mozbase/manifestparser/tests/test_chunking.py new file mode 100644 index 0000000000..3c649be99f --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_chunking.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python + +import os +import random +from collections import defaultdict +from itertools import chain +from unittest import TestCase + +import mozunit +from manifestparser.filters import chunk_by_dir, chunk_by_runtime, chunk_by_slice +from six import iteritems +from six.moves import range + +here = os.path.dirname(os.path.abspath(__file__)) + + +class ChunkBySlice(TestCase): + """Test chunking related filters""" + + def generate_tests(self, num, disabled=None): + disabled = disabled or [] + tests = [] + for i in range(num): + test = {"name": "test%i" % i} + if i in disabled: + test["disabled"] = "" + tests.append(test) + return tests + + def run_all_combos(self, num_tests, disabled=None): + tests = self.generate_tests(num_tests, disabled=disabled) + + for total in range(1, num_tests + 1): + res = [] + res_disabled = [] + for chunk in range(1, total + 1): + f = chunk_by_slice(chunk, total) + res.append(list(f(tests, {}))) + if disabled: + f.disabled = True + res_disabled.append(list(f(tests, {}))) + + lengths = [len([t for t in c if "disabled" not in t]) for c in res] + # the chunk with the most tests should have at most one more test + # than the chunk with the least tests + self.assertLessEqual(max(lengths) - min(lengths), 1) + + # chaining all chunks back together should equal the original list + # of tests + self.assertEqual(list(chain.from_iterable(res)), list(tests)) + + if disabled: + lengths = [len(c) for c in res_disabled] + self.assertLessEqual(max(lengths) - min(lengths), 1) + self.assertEqual(list(chain.from_iterable(res_disabled)), list(tests)) + + def test_chunk_by_slice(self): + chunk = chunk_by_slice(1, 1) + self.assertEqual(list(chunk([], {})), []) + + self.run_all_combos(num_tests=1) + self.run_all_combos(num_tests=10, disabled=[1, 2]) + + num_tests = 67 + disabled = list(i for i in range(num_tests) if i % 4 == 0) + self.run_all_combos(num_tests=num_tests, disabled=disabled) + + def test_two_times_more_chunks_than_tests(self): + # test case for bug 1182817 + tests = self.generate_tests(5) + + total_chunks = 10 + for i in range(1, total_chunks + 1): + # ensure IndexError is not raised + chunk_by_slice(i, total_chunks)(tests, {}) + + +class ChunkByDir(TestCase): + """Test chunking related filters""" + + def generate_tests(self, dirs): + """ + :param dirs: dict of the form, + { <dir>: <num tests> } + """ + i = 0 + for d, num in iteritems(dirs): + for _ in range(num): + i += 1 + name = "test%i" % i + test = {"name": name, "relpath": os.path.join(d, name)} + yield test + + def run_all_combos(self, dirs): + tests = list(self.generate_tests(dirs)) + + deepest = max(len(t["relpath"].split(os.sep)) - 1 for t in tests) + for depth in range(1, deepest + 1): + + def num_groups(tests): + unique = set() + for p in [t["relpath"] for t in tests]: + p = p.split(os.sep) + p = p[: min(depth, len(p) - 1)] + unique.add(os.sep.join(p)) + return len(unique) + + for total in range(1, num_groups(tests) + 1): + res = [] + for this in range(1, total + 1): + f = chunk_by_dir(this, total, depth) + res.append(list(f(tests, {}))) + + lengths = list(map(num_groups, res)) + # the chunk with the most dirs should have at most one more + # dir than the chunk with the least dirs + self.assertLessEqual(max(lengths) - min(lengths), 1) + + all_chunks = list(chain.from_iterable(res)) + # chunk_by_dir will mess up order, but chained chunks should + # contain all of the original tests and be the same length + self.assertEqual(len(all_chunks), len(tests)) + for t in tests: + self.assertIn(t, all_chunks) + + def test_chunk_by_dir(self): + chunk = chunk_by_dir(1, 1, 1) + self.assertEqual(list(chunk([], {})), []) + + dirs = { + "a": 2, + } + self.run_all_combos(dirs) + + dirs = { + "": 1, + "foo": 1, + "bar": 0, + "/foobar": 1, + } + self.run_all_combos(dirs) + + dirs = { + "a": 1, + "b": 1, + "a/b": 2, + "a/c": 1, + } + self.run_all_combos(dirs) + + dirs = { + "a": 5, + "a/b": 4, + "a/b/c": 7, + "a/b/c/d": 1, + "a/b/c/e": 3, + "b/c": 2, + "b/d": 5, + "b/d/e": 6, + "c": 8, + "c/d/e/f/g/h/i/j/k/l": 5, + "c/d/e/f/g/i/j/k/l/m/n": 2, + "c/e": 1, + } + self.run_all_combos(dirs) + + +class ChunkByRuntime(TestCase): + """Test chunking related filters""" + + def generate_tests(self, dirs): + """ + :param dirs: dict of the form, + { <dir>: <num tests> } + """ + i = 0 + for d, num in iteritems(dirs): + for _ in range(num): + i += 1 + name = "test%i" % i + manifest = os.path.join(d, "manifest.ini") + test = { + "name": name, + "relpath": os.path.join(d, name), + "manifest": manifest, + "manifest_relpath": manifest, + } + yield test + + def get_runtimes(self, tests): + runtimes = defaultdict(int) + for test in tests: + runtimes[test["manifest_relpath"]] += random.randint(0, 100) + return runtimes + + def chunk_by_round_robin(self, tests, total, runtimes): + tests_by_manifest = [] + for manifest, runtime in iteritems(runtimes): + mtests = [t for t in tests if t["manifest_relpath"] == manifest] + tests_by_manifest.append((runtime, mtests)) + tests_by_manifest.sort(key=lambda x: x[0], reverse=False) + + chunks = [[] for i in range(total)] + d = 1 # direction + i = 0 + for runtime, batch in tests_by_manifest: + chunks[i].extend(batch) + + # "draft" style (last pick goes first in the next round) + if (i == 0 and d == -1) or (i == total - 1 and d == 1): + d = -d + else: + i += d + + # make sure this test algorithm is valid + all_chunks = list(chain.from_iterable(chunks)) + self.assertEqual(len(all_chunks), len(tests)) + for t in tests: + self.assertIn(t, all_chunks) + return chunks + + def run_all_combos(self, dirs): + tests = list(self.generate_tests(dirs)) + runtimes = self.get_runtimes(tests) + + for total in range(1, len(dirs) + 1): + chunks = [] + for this in range(1, total + 1): + f = chunk_by_runtime(this, total, runtimes) + ret = list(f(tests, {})) + chunks.append(ret) + + # chunk_by_runtime will mess up order, but chained chunks should + # contain all of the original tests and be the same length + all_chunks = list(chain.from_iterable(chunks)) + self.assertEqual(len(all_chunks), len(tests)) + for t in tests: + self.assertIn(t, all_chunks) + + # calculate delta between slowest and fastest chunks + def runtime_delta(chunks): + totals = [] + for chunk in chunks: + manifests = set([t["manifest_relpath"] for t in chunk]) + total = sum(runtimes[m] for m in manifests) + totals.append(total) + return max(totals) - min(totals) + + delta = runtime_delta(chunks) + + # redo the chunking a second time using a round robin style + # algorithm + chunks = self.chunk_by_round_robin(tests, total, runtimes) + # sanity check the round robin algorithm + all_chunks = list(chain.from_iterable(chunks)) + self.assertEqual(len(all_chunks), len(tests)) + for t in tests: + self.assertIn(t, all_chunks) + + # since chunks will never have exactly equal runtimes, it's hard + # to tell if they were chunked optimally. Make sure it at least + # beats a naive round robin approach. + self.assertLessEqual(delta, runtime_delta(chunks)) + + def test_chunk_by_runtime(self): + random.seed(42) + + chunk = chunk_by_runtime(1, 1, {}) + self.assertEqual(list(chunk([], {})), []) + + dirs = { + "a": 2, + } + self.run_all_combos(dirs) + + dirs = { + "": 1, + "foo": 1, + "bar": 0, + "/foobar": 1, + } + self.run_all_combos(dirs) + + dirs = { + "a": 1, + "b": 1, + "a/b": 2, + "a/c": 1, + } + self.run_all_combos(dirs) + + dirs = { + "a": 5, + "a/b": 4, + "a/b/c": 7, + "a/b/c/d": 1, + "a/b/c/e": 3, + "b/c": 2, + "b/d": 5, + "b/d/e": 6, + "c": 8, + "c/d/e/f/g/h/i/j/k/l": 5, + "c/d/e/f/g/i/j/k/l/m/n": 2, + "c/e": 1, + } + self.run_all_combos(dirs) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/manifestparser/tests/test_convert_directory.py b/testing/mozbase/manifestparser/tests/test_convert_directory.py new file mode 100755 index 0000000000..9656b9c72b --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_convert_directory.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import shutil +import tempfile +import unittest + +import mozunit +from manifestparser import ManifestParser, convert + +here = os.path.dirname(os.path.abspath(__file__)) + +# In some cases tempfile.mkdtemp() may returns a path which contains +# symlinks. Some tests here will then break, as the manifestparser.convert +# function returns paths that does not contains symlinks. +# +# Workaround is to use the following function, if absolute path of temp dir +# must be compared. + + +def create_realpath_tempdir(): + """ + Create a tempdir without symlinks. + """ + return os.path.realpath(tempfile.mkdtemp()) + + +class TestDirectoryConversion(unittest.TestCase): + """test conversion of a directory tree to a manifest structure""" + + def create_stub(self, directory=None): + """stub out a directory with files in it""" + + files = ("foo", "bar", "fleem") + if directory is None: + directory = create_realpath_tempdir() + for i in files: + open(os.path.join(directory, i), "w").write(i) + subdir = os.path.join(directory, "subdir") + os.mkdir(subdir) + open(os.path.join(subdir, "subfile"), "w").write("baz") + return directory + + def test_directory_to_manifest(self): + """ + Test our ability to convert a static directory structure to a + manifest. + """ + + # create a stub directory + stub = self.create_stub() + try: + stub = stub.replace(os.path.sep, "/") + self.assertTrue(os.path.exists(stub) and os.path.isdir(stub)) + + # Make a manifest for it + manifest = convert([stub]) + out_tmpl = """[%(stub)s/bar] + +[%(stub)s/fleem] + +[%(stub)s/foo] + +[%(stub)s/subdir/subfile] + +""" # noqa + self.assertEqual(str(manifest), out_tmpl % dict(stub=stub)) + except BaseException: + raise + finally: + shutil.rmtree(stub) # cleanup + + def test_convert_directory_manifests_in_place(self): + """ + keep the manifests in place + """ + + stub = self.create_stub() + try: + ManifestParser.populate_directory_manifests([stub], filename="manifest.ini") + self.assertEqual( + sorted(os.listdir(stub)), + ["bar", "fleem", "foo", "manifest.ini", "subdir"], + ) + parser = ManifestParser() + parser.read(os.path.join(stub, "manifest.ini")) + self.assertEqual( + [i["name"] for i in parser.tests], ["subfile", "bar", "fleem", "foo"] + ) + parser = ManifestParser() + parser.read(os.path.join(stub, "subdir", "manifest.ini")) + self.assertEqual(len(parser.tests), 1) + self.assertEqual(parser.tests[0]["name"], "subfile") + except BaseException: + raise + finally: + shutil.rmtree(stub) + + def test_manifest_ignore(self): + """test manifest `ignore` parameter for ignoring directories""" + + stub = self.create_stub() + try: + ManifestParser.populate_directory_manifests( + [stub], filename="manifest.ini", ignore=("subdir",) + ) + parser = ManifestParser() + parser.read(os.path.join(stub, "manifest.ini")) + self.assertEqual([i["name"] for i in parser.tests], ["bar", "fleem", "foo"]) + self.assertFalse( + os.path.exists(os.path.join(stub, "subdir", "manifest.ini")) + ) + except BaseException: + raise + finally: + shutil.rmtree(stub) + + def test_pattern(self): + """test directory -> manifest with a file pattern""" + + stub = self.create_stub() + try: + parser = convert([stub], pattern="f*", relative_to=stub) + self.assertEqual([i["name"] for i in parser.tests], ["fleem", "foo"]) + + # test multiple patterns + parser = convert([stub], pattern=("f*", "s*"), relative_to=stub) + self.assertEqual( + [i["name"] for i in parser.tests], ["fleem", "foo", "subdir/subfile"] + ) + except BaseException: + raise + finally: + shutil.rmtree(stub) + + def test_update(self): + """ + Test our ability to update tests from a manifest and a directory of + files + """ + + # boilerplate + tempdir = create_realpath_tempdir() + for i in range(10): + open(os.path.join(tempdir, str(i)), "w").write(str(i)) + + # otherwise empty directory with a manifest file + newtempdir = create_realpath_tempdir() + manifest_file = os.path.join(newtempdir, "manifest.ini") + manifest_contents = str(convert([tempdir], relative_to=tempdir)) + with open(manifest_file, "w") as f: + f.write(manifest_contents) + + # get the manifest + manifest = ManifestParser(manifests=(manifest_file,)) + + # All of the tests are initially missing: + paths = [str(i) for i in range(10)] + self.assertEqual([i["name"] for i in manifest.missing()], paths) + + # But then we copy one over: + self.assertEqual(manifest.get("name", name="1"), ["1"]) + manifest.update(tempdir, name="1") + self.assertEqual(sorted(os.listdir(newtempdir)), ["1", "manifest.ini"]) + + # Update that one file and copy all the "tests": + open(os.path.join(tempdir, "1"), "w").write("secret door") + manifest.update(tempdir) + self.assertEqual( + sorted(os.listdir(newtempdir)), + ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "manifest.ini"], + ) + self.assertEqual( + open(os.path.join(newtempdir, "1")).read().strip(), "secret door" + ) + + # clean up: + shutil.rmtree(tempdir) + shutil.rmtree(newtempdir) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/manifestparser/tests/test_convert_symlinks.py b/testing/mozbase/manifestparser/tests/test_convert_symlinks.py new file mode 100755 index 0000000000..61054c8b78 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_convert_symlinks.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import shutil +import tempfile +import unittest + +import mozunit +from manifestparser import ManifestParser, convert + + +class TestSymlinkConversion(unittest.TestCase): + """ + test conversion of a directory tree with symlinks to a manifest structure + """ + + def create_stub(self, directory=None): + """stub out a directory with files in it""" + + files = ("foo", "bar", "fleem") + if directory is None: + directory = tempfile.mkdtemp() + for i in files: + open(os.path.join(directory, i), "w").write(i) + subdir = os.path.join(directory, "subdir") + os.mkdir(subdir) + open(os.path.join(subdir, "subfile"), "w").write("baz") + return directory + + def test_relpath(self): + """test convert `relative_to` functionality""" + + oldcwd = os.getcwd() + stub = self.create_stub() + try: + # subdir with in-memory manifest + files = ["../bar", "../fleem", "../foo", "subfile"] + subdir = os.path.join(stub, "subdir") + os.chdir(subdir) + parser = convert([stub], relative_to=".") + self.assertEqual([i["name"] for i in parser.tests], files) + except BaseException: + raise + finally: + shutil.rmtree(stub) + os.chdir(oldcwd) + + @unittest.skipIf( + not hasattr(os, "symlink"), "symlinks unavailable on this platform" + ) + def test_relpath_symlink(self): + """ + Ensure `relative_to` works in a symlink. + Not available on windows. + """ + + oldcwd = os.getcwd() + workspace = tempfile.mkdtemp() + try: + tmpdir = os.path.join(workspace, "directory") + os.makedirs(tmpdir) + linkdir = os.path.join(workspace, "link") + os.symlink(tmpdir, linkdir) + self.create_stub(tmpdir) + + # subdir with in-memory manifest + files = ["../bar", "../fleem", "../foo", "subfile"] + subdir = os.path.join(linkdir, "subdir") + os.chdir(os.path.realpath(subdir)) + for directory in (tmpdir, linkdir): + parser = convert([directory], relative_to=".") + self.assertEqual([i["name"] for i in parser.tests], files) + finally: + shutil.rmtree(workspace) + os.chdir(oldcwd) + + # a more complicated example + oldcwd = os.getcwd() + workspace = tempfile.mkdtemp() + try: + tmpdir = os.path.join(workspace, "directory") + os.makedirs(tmpdir) + linkdir = os.path.join(workspace, "link") + os.symlink(tmpdir, linkdir) + self.create_stub(tmpdir) + files = ["../bar", "../fleem", "../foo", "subfile"] + subdir = os.path.join(linkdir, "subdir") + subsubdir = os.path.join(subdir, "sub") + os.makedirs(subsubdir) + linksubdir = os.path.join(linkdir, "linky") + linksubsubdir = os.path.join(subsubdir, "linky") + os.symlink(subdir, linksubdir) + os.symlink(subdir, linksubsubdir) + for dest in (subdir,): + os.chdir(dest) + for directory in (tmpdir, linkdir): + parser = convert([directory], relative_to=".") + self.assertEqual([i["name"] for i in parser.tests], files) + finally: + shutil.rmtree(workspace) + os.chdir(oldcwd) + + @unittest.skipIf( + not hasattr(os, "symlink"), "symlinks unavailable on this platform" + ) + def test_recursion_symlinks(self): + workspace = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, workspace) + + # create two dirs + os.makedirs(os.path.join(workspace, "dir1")) + os.makedirs(os.path.join(workspace, "dir2")) + + # create cyclical symlinks + os.symlink(os.path.join("..", "dir1"), os.path.join(workspace, "dir2", "ldir1")) + os.symlink(os.path.join("..", "dir2"), os.path.join(workspace, "dir1", "ldir2")) + + # create one file in each dir + open(os.path.join(workspace, "dir1", "f1.txt"), "a").close() + open(os.path.join(workspace, "dir1", "ldir2", "f2.txt"), "a").close() + + data = [] + + def callback(rootdirectory, directory, subdirs, files): + for f in files: + data.append(f) + + ManifestParser._walk_directories([workspace], callback) + self.assertEqual(sorted(data), ["f1.txt", "f2.txt"]) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/manifestparser/tests/test_default_overrides.py b/testing/mozbase/manifestparser/tests/test_default_overrides.py new file mode 100755 index 0000000000..bf9c0f7e5d --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_default_overrides.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import unittest + +import mozunit +from manifestparser import ManifestParser, combine_fields + +here = os.path.dirname(os.path.abspath(__file__)) + + +class TestDefaultSkipif(unittest.TestCase): + """Tests applying a skip-if condition in [DEFAULT] and || with the value for the test""" + + def test_defaults(self): + + default = os.path.join(here, "default-skipif.ini") + parser = ManifestParser(manifests=(default,)) + for test in parser.tests: + if test["name"] == "test1": + self.assertEqual(test["skip-if"], "os == 'win' && debug\ndebug") + elif test["name"] == "test2": + self.assertEqual(test["skip-if"], "os == 'win' && debug\nos == 'linux'") + elif test["name"] == "test3": + self.assertEqual(test["skip-if"], "os == 'win' && debug\nos == 'win'") + elif test["name"] == "test4": + self.assertEqual( + test["skip-if"], "os == 'win' && debug\nos == 'win' && debug" + ) + elif test["name"] == "test5": + self.assertEqual(test["skip-if"], "os == 'win' && debug") + elif test["name"] == "test6": + self.assertEqual(test["skip-if"], "os == 'win' && debug\ndebug") + + +class TestDefaultSupportFiles(unittest.TestCase): + """Tests combining support-files field in [DEFAULT] with the value for a test""" + + def test_defaults(self): + + default = os.path.join(here, "default-suppfiles.ini") + parser = ManifestParser(manifests=(default,)) + expected_supp_files = { + "test7": "foo.js", + "test8": "foo.js bar.js", + "test9": "foo.js", + } + for test in parser.tests: + expected = expected_supp_files[test["name"]] + self.assertEqual(test["support-files"], expected) + + +class TestOmitDefaults(unittest.TestCase): + """Tests passing omit-defaults prevents defaults from propagating to definitions.""" + + def test_defaults(self): + manifests = ( + os.path.join(here, "default-suppfiles.ini"), + os.path.join(here, "default-skipif.ini"), + ) + parser = ManifestParser(manifests=manifests, handle_defaults=False) + expected_supp_files = { + "test8": "bar.js", + } + expected_skip_ifs = { + "test1": "debug", + "test2": "os == 'linux'", + "test3": "os == 'win'", + "test4": "os == 'win' && debug", + "test6": "debug", + } + for test in parser.tests: + for field, expectations in ( + ("support-files", expected_supp_files), + ("skip-if", expected_skip_ifs), + ): + expected = expectations.get(test["name"]) + if not expected: + self.assertNotIn(field, test) + else: + self.assertEqual(test[field], expected) + + expected_defaults = { + os.path.join(here, "default-suppfiles.ini"): { + "support-files": "foo.js", + }, + os.path.join(here, "default-skipif.ini"): { + "skip-if": "os == 'win' && debug", + }, + } + for path, defaults in expected_defaults.items(): + self.assertIn(path, parser.manifest_defaults) + actual_defaults = parser.manifest_defaults[path] + for key, value in defaults.items(): + self.assertIn(key, actual_defaults) + self.assertEqual(value, actual_defaults[key]) + + +class TestSubsuiteDefaults(unittest.TestCase): + """Test that subsuites are handled correctly when managing defaults + outside of the manifest parser.""" + + def test_subsuite_defaults(self): + manifest = os.path.join(here, "default-subsuite.ini") + parser = ManifestParser(manifests=(manifest,), handle_defaults=False) + expected_subsuites = { + "test1": "baz", + "test2": "foo", + } + defaults = parser.manifest_defaults[manifest] + for test in parser.tests: + value = combine_fields(defaults, test) + self.assertEqual(expected_subsuites[value["name"]], value["subsuite"]) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/manifestparser/tests/test_expressionparser.py b/testing/mozbase/manifestparser/tests/test_expressionparser.py new file mode 100755 index 0000000000..806f2347ef --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_expressionparser.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python + +import unittest + +import mozunit +from manifestparser import parse + + +class ExpressionParserTest(unittest.TestCase): + """Test the conditional expression parser.""" + + def test_basic(self): + + self.assertEqual(parse("1"), 1) + self.assertEqual(parse("100"), 100) + self.assertEqual(parse("true"), True) + self.assertEqual(parse("false"), False) + self.assertEqual("", parse('""')) + self.assertEqual(parse('"foo bar"'), "foo bar") + self.assertEqual(parse("'foo bar'"), "foo bar") + self.assertEqual(parse("foo", foo=1), 1) + self.assertEqual(parse("bar", bar=True), True) + self.assertEqual(parse("abc123", abc123="xyz"), "xyz") + + def test_equality(self): + + self.assertTrue(parse("true == true")) + self.assertTrue(parse("false == false")) + self.assertTrue(parse("1 == 1")) + self.assertTrue(parse("100 == 100")) + self.assertTrue(parse('"some text" == "some text"')) + self.assertTrue(parse("true != false")) + self.assertTrue(parse("1 != 2")) + self.assertTrue(parse('"text" != "other text"')) + self.assertTrue(parse("foo == true", foo=True)) + self.assertTrue(parse("foo == 1", foo=1)) + self.assertTrue(parse('foo == "bar"', foo="bar")) + self.assertTrue(parse("foo == bar", foo=True, bar=True)) + self.assertTrue(parse("true == foo", foo=True)) + self.assertTrue(parse("foo != true", foo=False)) + self.assertTrue(parse("foo != 2", foo=1)) + self.assertTrue(parse('foo != "bar"', foo="abc")) + self.assertTrue(parse("foo != bar", foo=True, bar=False)) + self.assertTrue(parse("true != foo", foo=False)) + self.assertTrue(parse("!false")) + + def test_conjunctures(self): + self.assertTrue(parse("true && true")) + self.assertTrue(parse("true || false")) + self.assertFalse(parse("false || false")) + self.assertFalse(parse("true && false")) + self.assertTrue(parse("true || false && false")) + + def test_parentheses(self): + self.assertTrue(parse("(true)")) + self.assertEqual(parse("(10)"), 10) + self.assertEqual(parse('("foo")'), "foo") + self.assertEqual(parse("(foo)", foo=1), 1) + self.assertTrue(parse("(true == true)"), True) + self.assertTrue(parse("(true != false)")) + self.assertTrue(parse("(true && true)")) + self.assertTrue(parse("(true || false)")) + self.assertTrue(parse("(true && true || false)")) + self.assertFalse(parse("(true || false) && false")) + self.assertTrue(parse("(true || false) && true")) + self.assertTrue(parse("true && (true || false)")) + self.assertTrue(parse("true && (true || false)")) + self.assertTrue(parse("(true && false) || (true && (true || false))")) + + def test_comments(self): + # comments in expressions work accidentally, via an implementation + # detail - the '#' character doesn't match any of the regular + # expressions we specify as tokens, and thus are ignored. + # However, having explicit tests for them means that should the + # implementation ever change, comments continue to work, even if that + # means a new implementation must handle them explicitly. + self.assertTrue(parse("true == true # it does!")) + self.assertTrue(parse("false == false # it does")) + self.assertTrue(parse("false != true # it doesnt")) + self.assertTrue(parse('"string with #" == "string with #" # really, it does')) + self.assertTrue( + parse('"string with #" != "string with # but not the same" # no match!') + ) + + def test_not(self): + """ + Test the ! operator. + """ + self.assertTrue(parse("!false")) + self.assertTrue(parse("!(false)")) + self.assertFalse(parse("!true")) + self.assertFalse(parse("!(true)")) + self.assertTrue(parse("!true || true)")) + self.assertTrue(parse("true || !true)")) + self.assertFalse(parse("!true && true")) + self.assertFalse(parse("true && !true")) + + def test_lesser_than(self): + """ + Test the < operator. + """ + self.assertTrue(parse("1 < 2")) + self.assertFalse(parse("3 < 2")) + self.assertTrue(parse("false || (1 < 2)")) + self.assertTrue(parse("1 < 2 && true")) + self.assertTrue(parse("true && 1 < 2")) + self.assertTrue(parse("!(5 < 1)")) + self.assertTrue(parse("'abc' < 'def'")) + self.assertFalse(parse("1 < 1")) + self.assertFalse(parse("'abc' < 'abc'")) + + def test_greater_than(self): + """ + Test the > operator. + """ + self.assertTrue(parse("2 > 1")) + self.assertFalse(parse("2 > 3")) + self.assertTrue(parse("false || (2 > 1)")) + self.assertTrue(parse("2 > 1 && true")) + self.assertTrue(parse("true && 2 > 1")) + self.assertTrue(parse("!(1 > 5)")) + self.assertTrue(parse("'def' > 'abc'")) + self.assertFalse(parse("1 > 1")) + self.assertFalse(parse("'abc' > 'abc'")) + + def test_lesser_or_equals_than(self): + """ + Test the <= operator. + """ + self.assertTrue(parse("1 <= 2")) + self.assertFalse(parse("3 <= 2")) + self.assertTrue(parse("false || (1 <= 2)")) + self.assertTrue(parse("1 < 2 && true")) + self.assertTrue(parse("true && 1 <= 2")) + self.assertTrue(parse("!(5 <= 1)")) + self.assertTrue(parse("'abc' <= 'def'")) + self.assertTrue(parse("1 <= 1")) + self.assertTrue(parse("'abc' <= 'abc'")) + + def test_greater_or_equals_than(self): + """ + Test the > operator. + """ + self.assertTrue(parse("2 >= 1")) + self.assertFalse(parse("2 >= 3")) + self.assertTrue(parse("false || (2 >= 1)")) + self.assertTrue(parse("2 >= 1 && true")) + self.assertTrue(parse("true && 2 >= 1")) + self.assertTrue(parse("!(1 >= 5)")) + self.assertTrue(parse("'def' >= 'abc'")) + self.assertTrue(parse("1 >= 1")) + self.assertTrue(parse("'abc' >= 'abc'")) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/manifestparser/tests/test_filters.py b/testing/mozbase/manifestparser/tests/test_filters.py new file mode 100644 index 0000000000..5060c7c395 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_filters.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python + +import os +from copy import deepcopy +from pprint import pprint + +import mozpack.path as mozpath +import mozunit +import pytest +from manifestparser.filters import ( + enabled, + fail_if, + failures, + filterlist, + pathprefix, + run_if, + skip_if, + subsuite, + tags, +) + +here = os.path.dirname(os.path.abspath(__file__)) + + +def test_data_model(): + def foo(x, y): + return x + + def bar(x, y): + return x + + def baz(x, y): + return x + + fl = filterlist() + + fl.extend([foo, bar]) + assert len(fl) == 2 + assert foo in fl + + fl.append(baz) + assert fl[2] == baz + + fl.remove(baz) + assert baz not in fl + + item = fl.pop() + assert item == bar + + assert fl.index(foo) == 0 + + del fl[0] + assert foo not in fl + with pytest.raises(IndexError): + fl[0] + + +def test_add_non_callable_to_list(): + fl = filterlist() + with pytest.raises(TypeError): + fl.append("foo") + + +def test_add_duplicates_to_list(): + def foo(x, y): + return x + + def bar(x, y): + return x + + sub = subsuite("foo") + fl = filterlist([foo, bar, sub]) + assert len(fl) == 3 + assert fl[0] == foo + + with pytest.raises(ValueError): + fl.append(foo) + + with pytest.raises(ValueError): + fl.append(subsuite("bar")) + + +def test_add_two_tags_filters(): + tag1 = tags("foo") + tag2 = tags("bar") + fl = filterlist([tag1]) + + with pytest.raises(ValueError): + fl.append(tag1) + + fl.append(tag2) + assert len(fl) == 2 + + +def test_filters_run_in_order(): + def a(x, y): + return x + + def b(x, y): + return x + + def c(x, y): + return x + + def d(x, y): + return x + + def e(x, y): + return x + + def f(x, y): + return x + + fl = filterlist([a, b]) + fl.append(c) + fl.extend([d, e]) + fl += [f] + assert [i for i in fl] == [a, b, c, d, e, f] + + +@pytest.fixture(scope="module") +def create_tests(): + def inner(*paths, **defaults): + tests = [] + for path in paths: + if isinstance(path, tuple): + path, kwargs = path + else: + kwargs = {} + + path = mozpath.normpath(path) + manifest = kwargs.pop( + "manifest", + defaults.pop( + "manifest", mozpath.join(mozpath.dirname(path), "manifest.ini") + ), + ) + test = { + "name": mozpath.basename(path), + "path": "/root/" + path, + "relpath": path, + "manifest": "/root/" + manifest, + "manifest_relpath": manifest, + } + test.update(**defaults) + test.update(**kwargs) + tests.append(test) + + # dump tests to stdout for easier debugging on failure + print("The 'create_tests' fixture returned:") + pprint(tests, indent=2) + return tests + + return inner + + +@pytest.fixture +def tests(create_tests): + return create_tests( + "test0", + ("test1", {"skip-if": "foo == 'bar'\nintermittent&&!debug"}), + ("test2", {"run-if": "foo == 'bar'"}), + ("test3", {"fail-if": "foo == 'bar'"}), + ("test4", {"disabled": "some reason"}), + ("test5", {"subsuite": "baz"}), + ("test6", {"subsuite": "baz,foo == 'bar'"}), + ("test7", {"tags": "foo bar"}), + ( + "test8", + {"skip-if": "\nbaz\nfoo == 'bar'\nfoo == 'baz'\nintermittent && debug"}, + ), + ) + + +def test_skip_if(tests): + ref = deepcopy(tests) + tests = list(skip_if(tests, {})) + assert len(tests) == len(ref) + + tests = deepcopy(ref) + tests = list(skip_if(tests, {"foo": "bar"})) + assert "disabled" in tests[1] + assert "disabled" in tests[8] + + +def test_run_if(tests): + ref = deepcopy(tests) + tests = list(run_if(tests, {})) + assert "disabled" in tests[2] + + tests = deepcopy(ref) + tests = list(run_if(tests, {"foo": "bar"})) + assert "disabled" not in tests[2] + + +def test_fail_if(tests): + ref = deepcopy(tests) + tests = list(fail_if(tests, {})) + assert "expected" not in tests[3] + + tests = deepcopy(ref) + tests = list(fail_if(tests, {"foo": "bar"})) + assert tests[3]["expected"] == "fail" + + +def test_enabled(tests): + ref = deepcopy(tests) + tests = list(enabled(tests, {})) + assert ref[4] not in tests + + +def test_subsuite(tests): + sub1 = subsuite() + sub2 = subsuite("baz") + + ref = deepcopy(tests) + tests = list(sub1(tests, {})) + assert ref[5] not in tests + assert len(tests) == len(ref) - 1 + + tests = deepcopy(ref) + tests = list(sub2(tests, {})) + assert len(tests) == 1 + assert ref[5] in tests + + +def test_subsuite_condition(tests): + sub1 = subsuite() + sub2 = subsuite("baz") + + ref = deepcopy(tests) + + tests = list(sub1(tests, {"foo": "bar"})) + assert ref[5] not in tests + assert ref[6] not in tests + + tests = deepcopy(ref) + tests = list(sub2(tests, {"foo": "bar"})) + assert len(tests) == 2 + assert tests[0]["name"] == "test5" + assert tests[1]["name"] == "test6" + + +def test_tags(tests): + ftags1 = tags([]) + ftags2 = tags(["bar", "baz"]) + + ref = deepcopy(tests) + tests = list(ftags1(tests, {})) + assert len(tests) == 0 + + tests = deepcopy(ref) + tests = list(ftags2(tests, {})) + assert len(tests) == 1 + assert ref[7] in tests + + +def test_failures(tests): + ref = deepcopy(tests) + fail1 = failures("intermittent") + tests = list(fail1(tests, {"intermittent": True, "debug": True})) + assert len(tests) == 1 + + tests = deepcopy(ref) + tests = list(fail1(tests, {"intermittent": True})) + assert len(tests) == 1 + + tests = deepcopy(ref) + tests = list(fail1(tests, {})) + assert len(tests) == 0 + + tests = deepcopy(ref) + tests = list(fail1(tests, {"intermittent": False, "debug": True})) + assert len(tests) == 0 + + +def test_pathprefix(create_tests): + tests = create_tests( + "test0", + "subdir/test1", + "subdir/test2", + ("subdir/test3", {"manifest": "manifest.ini"}), + ( + "other/test4", + { + "manifest": "manifest-common.ini", + "ancestor_manifest": "other/manifest.ini", + }, + ), + ) + + def names(items): + return sorted(i["name"] for i in items) + + # relative directory + prefix = pathprefix("subdir") + filtered = prefix(tests, {}) + assert names(filtered) == ["test1", "test2", "test3"] + + # absolute directory + prefix = pathprefix(["/root/subdir"]) + filtered = prefix(tests, {}) + assert names(filtered) == ["test1", "test2", "test3"] + + # relative manifest + prefix = pathprefix(["subdir/manifest.ini"]) + filtered = prefix(tests, {}) + assert names(filtered) == ["test1", "test2"] + + # absolute manifest + prefix = pathprefix(["/root/subdir/manifest.ini"]) + filtered = prefix(tests, {}) + assert names(filtered) == ["test1", "test2"] + + # mixed test and manifest + prefix = pathprefix(["subdir/test2", "manifest.ini"]) + filtered = prefix(tests, {}) + assert names(filtered) == ["test0", "test2", "test3"] + + # relative ancestor manifest + prefix = pathprefix(["other/manifest.ini"]) + filtered = prefix(tests, {}) + assert names(filtered) == ["test4"] + + # absolute ancestor manifest + prefix = pathprefix(["/root/other/manifest.ini"]) + filtered = prefix(tests, {}) + assert names(filtered) == ["test4"] + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/manifestparser/tests/test_manifestparser.py b/testing/mozbase/manifestparser/tests/test_manifestparser.py new file mode 100755 index 0000000000..3e6deaceb7 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_manifestparser.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import shutil +import tempfile +import unittest + +import mozunit +from manifestparser import ManifestParser +from six import StringIO + +here = os.path.dirname(os.path.abspath(__file__)) + + +class TestManifestParser(unittest.TestCase): + """ + Test the manifest parser + + You must have manifestparser installed before running these tests. + Run ``python manifestparser.py setup develop`` with setuptools installed. + """ + + def test_sanity(self): + """Ensure basic parser is sane""" + + parser = ManifestParser() + mozmill_example = os.path.join(here, "mozmill-example.ini") + parser.read(mozmill_example) + tests = parser.tests + self.assertEqual( + len(tests), len(open(mozmill_example).read().strip().splitlines()) + ) + + # Ensure that capitalization and order aren't an issue: + lines = ["[%s]" % test["name"] for test in tests] + self.assertEqual(lines, open(mozmill_example).read().strip().splitlines()) + + # Show how you select subsets of tests: + mozmill_restart_example = os.path.join(here, "mozmill-restart-example.ini") + parser.read(mozmill_restart_example) + restart_tests = parser.get(type="restart") + self.assertTrue(len(restart_tests) < len(parser.tests)) + self.assertEqual( + len(restart_tests), len(parser.get(manifest=mozmill_restart_example)) + ) + self.assertFalse( + [ + test + for test in restart_tests + if test["manifest"] != os.path.join(here, "mozmill-restart-example.ini") + ] + ) + self.assertEqual( + parser.get("name", tags=["foo"]), + [ + "restartTests/testExtensionInstallUninstall/test2.js", + "restartTests/testExtensionInstallUninstall/test1.js", + ], + ) + self.assertEqual( + parser.get("name", foo="bar"), + ["restartTests/testExtensionInstallUninstall/test2.js"], + ) + + def test_include(self): + """Illustrate how include works""" + + include_example = os.path.join(here, "include-example.ini") + parser = ManifestParser(manifests=(include_example,)) + + # All of the tests should be included, in order: + self.assertEqual(parser.get("name"), ["crash-handling", "fleem", "flowers"]) + self.assertEqual( + [ + (test["name"], os.path.basename(test["manifest"])) + for test in parser.tests + ], + [ + ("crash-handling", "bar.ini"), + ("fleem", "include-example.ini"), + ("flowers", "foo.ini"), + ], + ) + + # The including manifest is always reported as a part of the generated test object. + self.assertTrue( + all( + [ + t["ancestor_manifest"] == "include-example.ini" + for t in parser.tests + if t["name"] != "fleem" + ] + ) + ) + + # The manifests should be there too: + self.assertEqual(len(parser.manifests()), 3) + + # We already have the root directory: + self.assertEqual(here, parser.rootdir) + + # DEFAULT values should persist across includes, unless they're + # overwritten. In this example, include-example.ini sets foo=bar, but + # it's overridden to fleem in bar.ini + self.assertEqual(parser.get("name", foo="bar"), ["fleem", "flowers"]) + self.assertEqual(parser.get("name", foo="fleem"), ["crash-handling"]) + + # Passing parameters in the include section allows defining variables in + # the submodule scope: + self.assertEqual(parser.get("name", tags=["red"]), ["flowers"]) + + # However, this should be overridable from the DEFAULT section in the + # included file and that overridable via the key directly connected to + # the test: + self.assertEqual(parser.get(name="flowers")[0]["blue"], "ocean") + self.assertEqual(parser.get(name="flowers")[0]["yellow"], "submarine") + + # You can query multiple times if you need to: + flowers = parser.get(foo="bar") + self.assertEqual(len(flowers), 2) + + # Using the inverse flag should invert the set of tests returned: + self.assertEqual( + parser.get("name", inverse=True, tags=["red"]), ["crash-handling", "fleem"] + ) + + # All of the included tests actually exist: + self.assertEqual([i["name"] for i in parser.missing()], []) + + # Write the output to a manifest: + buffer = StringIO() + parser.write(fp=buffer, global_kwargs={"foo": "bar"}) + expected_output = """[DEFAULT] +foo = bar + +[fleem] + +[include/flowers] +blue = ocean +red = roses +yellow = submarine""" # noqa + + self.assertEqual(buffer.getvalue().strip(), expected_output) + + def test_include_manifest_defaults(self): + """ + Test that manifest_defaults and manifests() are correctly populated + when includes are used. + """ + + include_example = os.path.join(here, "include-example.ini") + noinclude_example = os.path.join(here, "just-defaults.ini") + bar_path = os.path.join(here, "include", "bar.ini") + foo_path = os.path.join(here, "include", "foo.ini") + + parser = ManifestParser( + manifests=(include_example, noinclude_example), rootdir=here + ) + + # Standalone manifests must be appear as-is. + self.assertTrue(include_example in parser.manifest_defaults) + self.assertTrue(noinclude_example in parser.manifest_defaults) + + # Included manifests must only appear together with the parent manifest + # that included the manifest. + self.assertFalse(bar_path in parser.manifest_defaults) + self.assertFalse(foo_path in parser.manifest_defaults) + ancestor_ini = os.path.relpath(include_example, parser.rootdir) + self.assertTrue((ancestor_ini, bar_path) in parser.manifest_defaults) + self.assertTrue((ancestor_ini, foo_path) in parser.manifest_defaults) + + # manifests() must only return file paths (strings). + manifests = parser.manifests() + self.assertEqual(len(manifests), 4) + self.assertIn(foo_path, manifests) + self.assertIn(bar_path, manifests) + self.assertIn(include_example, manifests) + self.assertIn(noinclude_example, manifests) + + def test_include_handle_defaults_False(self): + """ + Test that manifest_defaults and manifests() are correct even when + handle_defaults is set to False. + """ + manifest = os.path.join(here, "include-example.ini") + foo_path = os.path.join(here, "include", "foo.ini") + + parser = ManifestParser( + manifests=(manifest,), handle_defaults=False, rootdir=here + ) + ancestor_ini = os.path.relpath(manifest, parser.rootdir) + + self.assertIn(manifest, parser.manifest_defaults) + self.assertNotIn(foo_path, parser.manifest_defaults) + self.assertIn((ancestor_ini, foo_path), parser.manifest_defaults) + self.assertEqual( + parser.manifest_defaults[manifest], + { + "foo": "bar", + "here": here, + }, + ) + self.assertEqual( + parser.manifest_defaults[(ancestor_ini, foo_path)], + { + "here": os.path.join(here, "include"), + "red": "roses", + "blue": "ocean", + "yellow": "daffodils", + }, + ) + + def test_include_repeated(self): + """ + Test that repeatedly included manifests are independent of each other. + """ + include_example = os.path.join(here, "include-example.ini") + included_foo = os.path.join(here, "include", "foo.ini") + + # In the expected output, blue and yellow have the values from foo.ini + # (ocean, submarine) instead of the ones from include-example.ini + # (violets, daffodils), because the defaults in the included file take + # precedence over the values from the parent. + include_output = """[include/crash-handling] +foo = fleem + +[fleem] +foo = bar + +[include/flowers] +blue = ocean +foo = bar +red = roses +yellow = submarine + +""" + included_output = """[include/flowers] +blue = ocean +yellow = submarine + +""" + + parser = ManifestParser(manifests=(include_example, included_foo), rootdir=here) + self.assertEqual( + parser.get("name"), ["crash-handling", "fleem", "flowers", "flowers"] + ) + self.assertEqual( + [ + (test["name"], os.path.basename(test["manifest"])) + for test in parser.tests + ], + [ + ("crash-handling", "bar.ini"), + ("fleem", "include-example.ini"), + ("flowers", "foo.ini"), + ("flowers", "foo.ini"), + ], + ) + self.check_included_repeat( + parser, + parser.tests[3], + parser.tests[2], + "%s%s" % (include_output, included_output), + ) + + # Same tests, but with the load order of the manifests swapped. + parser = ManifestParser(manifests=(included_foo, include_example), rootdir=here) + self.assertEqual( + parser.get("name"), ["flowers", "crash-handling", "fleem", "flowers"] + ) + self.assertEqual( + [ + (test["name"], os.path.basename(test["manifest"])) + for test in parser.tests + ], + [ + ("flowers", "foo.ini"), + ("crash-handling", "bar.ini"), + ("fleem", "include-example.ini"), + ("flowers", "foo.ini"), + ], + ) + self.check_included_repeat( + parser, + parser.tests[0], + parser.tests[3], + "%s%s" % (included_output, include_output), + ) + + def check_included_repeat( + self, parser, isolated_test, included_test, expected_output + ): + include_example = os.path.join(here, "include-example.ini") + included_foo = os.path.join(here, "include", "foo.ini") + ancestor_ini = os.path.relpath(include_example, parser.rootdir) + manifest_default_key = (ancestor_ini, included_foo) + + self.assertFalse("ancestor_manifest" in isolated_test) + self.assertEqual(included_test["ancestor_manifest"], "include-example.ini") + + self.assertTrue(include_example in parser.manifest_defaults) + self.assertTrue(included_foo in parser.manifest_defaults) + self.assertTrue(manifest_default_key in parser.manifest_defaults) + self.assertEqual( + parser.manifest_defaults[manifest_default_key], + { + "foo": "bar", + "here": os.path.join(here, "include"), + "red": "roses", + "blue": "ocean", + "yellow": "daffodils", + }, + ) + + buffer = StringIO() + parser.write(fp=buffer) + self.assertEqual(buffer.getvalue(), expected_output) + + def test_invalid_path(self): + """ + Test invalid path should not throw when not strict + """ + manifest = os.path.join(here, "include-invalid.ini") + ManifestParser(manifests=(manifest,), strict=False) + + def test_copy(self): + """Test our ability to copy a set of manifests""" + + tempdir = tempfile.mkdtemp() + include_example = os.path.join(here, "include-example.ini") + manifest = ManifestParser(manifests=(include_example,)) + manifest.copy(tempdir) + self.assertEqual( + sorted(os.listdir(tempdir)), ["fleem", "include", "include-example.ini"] + ) + self.assertEqual( + sorted(os.listdir(os.path.join(tempdir, "include"))), + ["bar.ini", "crash-handling", "flowers", "foo.ini"], + ) + from_manifest = ManifestParser(manifests=(include_example,)) + to_manifest = os.path.join(tempdir, "include-example.ini") + to_manifest = ManifestParser(manifests=(to_manifest,)) + self.assertEqual(to_manifest.get("name"), from_manifest.get("name")) + shutil.rmtree(tempdir) + + def test_path_override(self): + """You can override the path in the section too. + This shows that you can use a relative path""" + path_example = os.path.join(here, "path-example.ini") + manifest = ManifestParser(manifests=(path_example,)) + self.assertEqual(manifest.tests[0]["path"], os.path.join(here, "fleem")) + + def test_relative_path(self): + """ + Relative test paths are correctly calculated. + """ + relative_path = os.path.join(here, "relative-path.ini") + manifest = ManifestParser(manifests=(relative_path,)) + self.assertEqual( + manifest.tests[0]["path"], os.path.join(os.path.dirname(here), "fleem") + ) + self.assertEqual(manifest.tests[0]["relpath"], os.path.join("..", "fleem")) + self.assertEqual( + manifest.tests[1]["relpath"], os.path.join("..", "testsSIBLING", "example") + ) + + def test_path_from_fd(self): + """ + Test paths are left untouched when manifest is a file-like object. + """ + fp = StringIO("[section]\npath=fleem") + manifest = ManifestParser(manifests=(fp,)) + self.assertEqual(manifest.tests[0]["path"], "fleem") + self.assertEqual(manifest.tests[0]["relpath"], "fleem") + self.assertEqual(manifest.tests[0]["manifest"], None) + + def test_comments(self): + """ + ensure comments work, see + https://bugzilla.mozilla.org/show_bug.cgi?id=813674 + """ + comment_example = os.path.join(here, "comment-example.ini") + manifest = ManifestParser(manifests=(comment_example,)) + self.assertEqual(len(manifest.tests), 8) + names = [i["name"] for i in manifest.tests] + self.assertFalse("test_0202_app_launch_apply_update_dirlocked.js" in names) + + def test_verifyDirectory(self): + + directory = os.path.join(here, "verifyDirectory") + + # correct manifest + manifest_path = os.path.join(directory, "verifyDirectory.ini") + manifest = ManifestParser(manifests=(manifest_path,)) + missing = manifest.verifyDirectory(directory, extensions=(".js",)) + self.assertEqual(missing, (set(), set())) + + # manifest is missing test_1.js + test_1 = os.path.join(directory, "test_1.js") + manifest_path = os.path.join(directory, "verifyDirectory_incomplete.ini") + manifest = ManifestParser(manifests=(manifest_path,)) + missing = manifest.verifyDirectory(directory, extensions=(".js",)) + self.assertEqual(missing, (set(), set([test_1]))) + + # filesystem is missing test_notappearinginthisfilm.js + missing_test = os.path.join(directory, "test_notappearinginthisfilm.js") + manifest_path = os.path.join(directory, "verifyDirectory_toocomplete.ini") + manifest = ManifestParser(manifests=(manifest_path,)) + missing = manifest.verifyDirectory(directory, extensions=(".js",)) + self.assertEqual(missing, (set([missing_test]), set())) + + def test_just_defaults(self): + """Ensure a manifest with just a DEFAULT section exposes that data.""" + + parser = ManifestParser() + manifest = os.path.join(here, "just-defaults.ini") + parser.read(manifest) + self.assertEqual(len(parser.tests), 0) + self.assertTrue(manifest in parser.manifest_defaults) + self.assertEqual(parser.manifest_defaults[manifest]["foo"], "bar") + + def test_manifest_list(self): + """ + Ensure a manifest with just a DEFAULT section still returns + itself from the manifests() method. + """ + + parser = ManifestParser() + manifest = os.path.join(here, "no-tests.ini") + parser.read(manifest) + self.assertEqual(len(parser.tests), 0) + self.assertTrue(len(parser.manifests()) == 1) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/manifestparser/tests/test_read_ini.py b/testing/mozbase/manifestparser/tests/test_read_ini.py new file mode 100755 index 0000000000..257054ee52 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_read_ini.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python + +""" +test .ini parsing + +ensure our .ini parser is doing what we want; to be deprecated for +python's standard ConfigParser when 2.7 is reality so OrderedDict +is the default: + +http://docs.python.org/2/library/configparser.html +""" + +from textwrap import dedent + +import mozunit +import pytest +from manifestparser import read_ini +from six import StringIO + + +@pytest.fixture(scope="module") +def parse_manifest(): + def inner(string, **kwargs): + buf = StringIO() + buf.write(dedent(string)) + buf.seek(0) + return read_ini(buf, **kwargs)[0] + + return inner + + +def test_inline_comments(parse_manifest): + result = parse_manifest( + """ + [test_felinicity.py] + kittens = true # This test requires kittens + cats = false#but not cats + """ + )[0][1] + + # make sure inline comments get stripped out, but comments without a space in front don't + assert result["kittens"] == "true" + assert result["cats"] == "false#but not cats" + + +def test_line_continuation(parse_manifest): + result = parse_manifest( + """ + [test_caninicity.py] + breeds = + sheppard + retriever + terrier + + [test_cats_and_dogs.py] + cats=yep + dogs= + yep + yep + birds=nope + fish=nope + """ + ) + assert result[0][1]["breeds"].split() == ["sheppard", "retriever", "terrier"] + assert result[1][1]["cats"] == "yep" + assert result[1][1]["dogs"].split() == ["yep", "yep"] + assert result[1][1]["birds"].split() == ["nope", "fish=nope"] + + +def test_dupes_error(parse_manifest): + dupes = """ + [test_dupes.py] + foo = bar + foo = baz + """ + with pytest.raises(AssertionError): + parse_manifest(dupes, strict=True) + + with pytest.raises(AssertionError): + parse_manifest(dupes, strict=False) + + +def test_defaults_handling(parse_manifest): + manifest = """ + [DEFAULT] + flower = rose + skip-if = true + + [test_defaults] + """ + + result = parse_manifest(manifest)[0][1] + assert result["flower"] == "rose" + assert result["skip-if"] == "true" + + result = parse_manifest( + manifest, + defaults={ + "flower": "tulip", + "colour": "pink", + "skip-if": "false", + }, + )[0][1] + assert result["flower"] == "rose" + assert result["colour"] == "pink" + assert result["skip-if"] == "false\ntrue" + + result = parse_manifest(manifest.replace("DEFAULT", "default"))[0][1] + assert result["flower"] == "rose" + assert result["skip-if"] == "true" + + +def test_multiline_skip(parse_manifest): + manifest = """ + [test_multiline_skip] + skip-if = + os == "mac" # bug 123 + os == "linux" && debug # bug 456 + """ + + result = parse_manifest(manifest)[0][1] + assert ( + result["skip-if"].replace("\r\n", "\n") + == dedent( + """ + os == "mac" + os == "linux" && debug + """ + ).rstrip() + ) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/manifestparser/tests/test_testmanifest.py b/testing/mozbase/manifestparser/tests/test_testmanifest.py new file mode 100644 index 0000000000..0aef82b00f --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_testmanifest.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python + +import os +import shutil +import tempfile +import unittest + +import mozunit +from manifestparser import ParseError, TestManifest +from manifestparser.filters import subsuite + +here = os.path.dirname(os.path.abspath(__file__)) + + +class TestTestManifest(unittest.TestCase): + """Test the Test Manifest""" + + def test_testmanifest(self): + # Test filtering based on platform: + filter_example = os.path.join(here, "filter-example.ini") + manifest = TestManifest(manifests=(filter_example,), strict=False) + self.assertEqual( + [ + i["name"] + for i in manifest.active_tests(os="win", disabled=False, exists=False) + ], + ["windowstest", "fleem"], + ) + self.assertEqual( + [ + i["name"] + for i in manifest.active_tests(os="linux", disabled=False, exists=False) + ], + ["fleem", "linuxtest"], + ) + + # Look for existing tests. There is only one: + self.assertEqual([i["name"] for i in manifest.active_tests()], ["fleem"]) + + # You should be able to expect failures: + last = manifest.active_tests(exists=False, toolkit="gtk")[-1] + self.assertEqual(last["name"], "linuxtest") + self.assertEqual(last["expected"], "pass") + last = manifest.active_tests(exists=False, toolkit="cocoa")[-1] + self.assertEqual(last["expected"], "fail") + + def test_missing_paths(self): + """ + Test paths that don't exist raise an exception in strict mode. + """ + tempdir = tempfile.mkdtemp() + + missing_path = os.path.join(here, "missing-path.ini") + manifest = TestManifest(manifests=(missing_path,), strict=True) + self.assertRaises(IOError, manifest.active_tests) + self.assertRaises(IOError, manifest.copy, tempdir) + self.assertRaises(IOError, manifest.update, tempdir) + + shutil.rmtree(tempdir) + + def test_comments(self): + """ + ensure comments work, see + https://bugzilla.mozilla.org/show_bug.cgi?id=813674 + """ + comment_example = os.path.join(here, "comment-example.ini") + manifest = TestManifest(manifests=(comment_example,)) + self.assertEqual(len(manifest.tests), 8) + names = [i["name"] for i in manifest.tests] + self.assertFalse("test_0202_app_launch_apply_update_dirlocked.js" in names) + + def test_manifest_subsuites(self): + """ + test subsuites and conditional subsuites + """ + relative_path = os.path.join(here, "subsuite.ini") + manifest = TestManifest(manifests=(relative_path,)) + info = {"foo": "bar"} + + # 6 tests total + tests = manifest.active_tests(exists=False, **info) + self.assertEqual(len(tests), 6) + + # only 3 tests for subsuite bar when foo==bar + tests = manifest.active_tests(exists=False, filters=[subsuite("bar")], **info) + self.assertEqual(len(tests), 3) + + # only 1 test for subsuite baz, regardless of conditions + other = {"something": "else"} + tests = manifest.active_tests(exists=False, filters=[subsuite("baz")], **info) + self.assertEqual(len(tests), 1) + tests = manifest.active_tests(exists=False, filters=[subsuite("baz")], **other) + self.assertEqual(len(tests), 1) + + # 4 tests match when the condition doesn't match (all tests except + # the unconditional subsuite) + info = {"foo": "blah"} + tests = manifest.active_tests(exists=False, filters=[subsuite()], **info) + self.assertEqual(len(tests), 5) + + # test for illegal subsuite value + manifest.tests[0]["subsuite"] = 'subsuite=bar,foo=="bar",type="nothing"' + with self.assertRaises(ParseError): + manifest.active_tests(exists=False, filters=[subsuite("foo")], **info) + + def test_none_and_empty_manifest(self): + """ + Test TestManifest for None and empty manifest, see + https://bugzilla.mozilla.org/show_bug.cgi?id=1087682 + """ + none_manifest = TestManifest(manifests=None, strict=False) + self.assertEqual(len(none_manifest.test_paths()), 0) + self.assertEqual(len(none_manifest.active_tests()), 0) + + empty_manifest = TestManifest(manifests=[], strict=False) + self.assertEqual(len(empty_manifest.test_paths()), 0) + self.assertEqual(len(empty_manifest.active_tests()), 0) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/manifestparser/tests/test_util.py b/testing/mozbase/manifestparser/tests/test_util.py new file mode 100644 index 0000000000..e32ecbab79 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_util.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python + +""" +Test how our utility functions are working. +""" + +from textwrap import dedent + +import mozunit +import pytest +from manifestparser import read_ini +from manifestparser.util import evaluate_list_from_string +from six import StringIO + + +@pytest.fixture(scope="module") +def parse_manifest(): + def inner(string, **kwargs): + buf = StringIO() + buf.write(dedent(string)) + buf.seek(0) + return read_ini(buf, **kwargs)[0] + + return inner + + +@pytest.mark.parametrize( + "test_manifest, expected_list", + [ + [ + """ + [test_felinicity.py] + kittens = true + cats = + "I", + "Am", + "A", + "Cat", + """, + ["I", "Am", "A", "Cat"], + ], + [ + """ + [test_felinicity.py] + kittens = true + cats = + ["I", 1], + ["Am", 2], + ["A", 3], + ["Cat", 4], + """, + [ + ["I", 1], + ["Am", 2], + ["A", 3], + ["Cat", 4], + ], + ], + ], +) +def test_string_to_list_conversion(test_manifest, expected_list, parse_manifest): + parsed_tests = parse_manifest(test_manifest) + assert evaluate_list_from_string(parsed_tests[0][1]["cats"]) == expected_list + + +@pytest.mark.parametrize( + "test_manifest, failure", + [ + [ + """ + # This will fail since the elements are not enlosed in quotes + [test_felinicity.py] + kittens = true + cats = + I, + Am, + A, + Cat, + """, + ValueError, + ], + [ + """ + # This will fail since the syntax is incorrect + [test_felinicity.py] + kittens = true + cats = + ["I", 1, + ["Am", 2, + ["A", 3], + ["Cat", 4], + """, + SyntaxError, + ], + ], +) +def test_string_to_list_conversion_failures(test_manifest, failure, parse_manifest): + parsed_tests = parse_manifest(test_manifest) + with pytest.raises(failure): + evaluate_list_from_string(parsed_tests[0][1]["cats"]) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/subdir/manifest.ini b/testing/mozbase/manifestparser/tests/verifyDirectory/subdir/manifest.ini new file mode 100644 index 0000000000..509ebd62ef --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/subdir/manifest.ini @@ -0,0 +1 @@ +[test_sub.js] diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/subdir/test_sub.js b/testing/mozbase/manifestparser/tests/verifyDirectory/subdir/test_sub.js new file mode 100644 index 0000000000..df48720d9d --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/subdir/test_sub.js @@ -0,0 +1 @@ +// test_sub.js diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/test_1.js b/testing/mozbase/manifestparser/tests/verifyDirectory/test_1.js new file mode 100644 index 0000000000..c5a966f46a --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/test_1.js @@ -0,0 +1 @@ +// test_1.js diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/test_2.js b/testing/mozbase/manifestparser/tests/verifyDirectory/test_2.js new file mode 100644 index 0000000000..d8648599c5 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/test_2.js @@ -0,0 +1 @@ +// test_2.js diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/test_3.js b/testing/mozbase/manifestparser/tests/verifyDirectory/test_3.js new file mode 100644 index 0000000000..794bc2c341 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/test_3.js @@ -0,0 +1 @@ +// test_3.js diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory.ini b/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory.ini new file mode 100644 index 0000000000..10e0c79c81 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory.ini @@ -0,0 +1,4 @@ +[test_1.js] +[test_2.js] +[test_3.js] +[include:subdir/manifest.ini] diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory_incomplete.ini b/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory_incomplete.ini new file mode 100644 index 0000000000..cde526acfc --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory_incomplete.ini @@ -0,0 +1,3 @@ +[test_2.js] +[test_3.js] +[include:subdir/manifest.ini] diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory_toocomplete.ini b/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory_toocomplete.ini new file mode 100644 index 0000000000..88994ae26f --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory_toocomplete.ini @@ -0,0 +1,5 @@ +[test_1.js] +[test_2.js] +[test_3.js] +[test_notappearinginthisfilm.js] +[include:subdir/manifest.ini] |