diff options
Diffstat (limited to 'tools/tryselect/selectors/syntax.py')
| -rw-r--r-- | tools/tryselect/selectors/syntax.py | 708 |
1 files changed, 708 insertions, 0 deletions
diff --git a/tools/tryselect/selectors/syntax.py b/tools/tryselect/selectors/syntax.py new file mode 100644 index 0000000000..ad643f8314 --- /dev/null +++ b/tools/tryselect/selectors/syntax.py @@ -0,0 +1,708 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os +import re +import sys +from collections import defaultdict + +import mozpack.path as mozpath +from moztest.resolve import TestResolver + +from ..cli import BaseTryParser +from ..push import build, push_to_try + +here = os.path.abspath(os.path.dirname(__file__)) + + +class SyntaxParser(BaseTryParser): + name = "syntax" + arguments = [ + [ + ["paths"], + { + "nargs": "*", + "default": [], + "help": "Paths to search for tests to run on try.", + }, + ], + [ + ["-b", "--build"], + { + "dest": "builds", + "default": "do", + "help": "Build types to run (d for debug, o for optimized).", + }, + ], + [ + ["-p", "--platform"], + { + "dest": "platforms", + "action": "append", + "help": "Platforms to run (required if not found in the environment as " + "AUTOTRY_PLATFORM_HINT).", + }, + ], + [ + ["-u", "--unittests"], + { + "dest": "tests", + "action": "append", + "help": "Test suites to run in their entirety.", + }, + ], + [ + ["-t", "--talos"], + { + "action": "append", + "help": "Talos suites to run.", + }, + ], + [ + ["-j", "--jobs"], + { + "action": "append", + "help": "Job tasks to run.", + }, + ], + [ + ["--tag"], + { + "dest": "tags", + "action": "append", + "help": "Restrict tests to the given tag (may be specified multiple times).", + }, + ], + [ + ["--and"], + { + "action": "store_true", + "dest": "intersection", + "help": "When -u and paths are supplied run only the intersection of the " + "tests specified by the two arguments.", + }, + ], + [ + ["--no-artifact"], + { + "action": "store_true", + "help": "Disable artifact builds even if --enable-artifact-builds is set " + "in the mozconfig.", + }, + ], + [ + ["-v", "--verbose"], + { + "dest": "verbose", + "action": "store_true", + "default": False, + "help": "Print detailed information about the resulting test selection " + "and commands performed.", + }, + ], + ] + + # Arguments we will accept on the command line and pass through to try + # syntax with no further intervention. The set is taken from + # http://trychooser.pub.build.mozilla.org with a few additions. + # + # Note that the meaning of store_false and store_true arguments is + # not preserved here, as we're only using these to echo the literal + # arguments to another consumer. Specifying either store_false or + # store_true here will have an equivalent effect. + pass_through_arguments = { + "--rebuild": { + "action": "store", + "dest": "rebuild", + "help": "Re-trigger all test jobs (up to 20 times)", + }, + "--rebuild-talos": { + "action": "store", + "dest": "rebuild_talos", + "help": "Re-trigger all talos jobs", + }, + "--interactive": { + "action": "store_true", + "dest": "interactive", + "help": "Allow ssh-like access to running test containers", + }, + "--no-retry": { + "action": "store_true", + "dest": "no_retry", + "help": "Do not retrigger failed tests", + }, + "--setenv": { + "action": "append", + "dest": "setenv", + "help": "Set the corresponding variable in the test environment for " + "applicable harnesses.", + }, + "-f": { + "action": "store_true", + "dest": "failure_emails", + "help": "Request failure emails only", + }, + "--failure-emails": { + "action": "store_true", + "dest": "failure_emails", + "help": "Request failure emails only", + }, + "-e": { + "action": "store_true", + "dest": "all_emails", + "help": "Request all emails", + }, + "--all-emails": { + "action": "store_true", + "dest": "all_emails", + "help": "Request all emails", + }, + "--artifact": { + "action": "store_true", + "dest": "artifact", + "help": "Force artifact builds where possible.", + }, + "--upload-xdbs": { + "action": "store_true", + "dest": "upload_xdbs", + "help": "Upload XDB compilation db files generated by hazard build", + }, + } + task_configs = [] + + def __init__(self, *args, **kwargs): + BaseTryParser.__init__(self, *args, **kwargs) + + group = self.add_argument_group("pass-through arguments") + for arg, opts in self.pass_through_arguments.items(): + group.add_argument(arg, **opts) + + +class TryArgumentTokenizer: + symbols = [ + ("separator", ","), + ("list_start", r"\["), + ("list_end", r"\]"), + ("item", r"([^,\[\]\s][^,\[\]]+)"), + ("space", r"\s+"), + ] + token_re = re.compile("|".join("(?P<%s>%s)" % item for item in symbols)) + + def tokenize(self, data): + for match in self.token_re.finditer(data): + symbol = match.lastgroup + data = match.group(symbol) + if symbol == "space": + pass + else: + yield symbol, data + + +class TryArgumentParser: + """Simple three-state parser for handling expressions + of the from "foo[sub item, another], bar,baz". This takes + input from the TryArgumentTokenizer and runs through a small + state machine, returning a dictionary of {top-level-item:[sub_items]} + i.e. the above would result in + {"foo":["sub item", "another"], "bar": [], "baz": []} + In the case of invalid input a ValueError is raised.""" + + EOF = object() + + def __init__(self): + self.reset() + + def reset(self): + self.tokens = None + self.current_item = None + self.data = {} + self.token = None + self.state = None + + def parse(self, tokens): + self.reset() + self.tokens = tokens + self.consume() + self.state = self.item_state + while self.token[0] != self.EOF: + self.state() + return self.data + + def consume(self): + try: + self.token = next(self.tokens) + except StopIteration: + self.token = (self.EOF, None) + + def expect(self, *types): + if self.token[0] not in types: + raise ValueError( + "Error parsing try string, unexpected %s" % (self.token[0]) + ) + + def item_state(self): + self.expect("item") + value = self.token[1].strip() + if value not in self.data: + self.data[value] = [] + self.current_item = value + self.consume() + if self.token[0] == "separator": + self.consume() + elif self.token[0] == "list_start": + self.consume() + self.state = self.subitem_state + elif self.token[0] == self.EOF: + pass + else: + raise ValueError + + def subitem_state(self): + self.expect("item") + value = self.token[1].strip() + self.data[self.current_item].append(value) + self.consume() + if self.token[0] == "separator": + self.consume() + elif self.token[0] == "list_end": + self.consume() + self.state = self.after_list_end_state + else: + raise ValueError + + def after_list_end_state(self): + self.expect("separator") + self.consume() + self.state = self.item_state + + +def parse_arg(arg): + tokenizer = TryArgumentTokenizer() + parser = TryArgumentParser() + return parser.parse(tokenizer.tokenize(arg)) + + +class AutoTry: + + # Maps from flavors to the job names needed to run that flavour + flavor_jobs = { + "mochitest": ["mochitest-1", "mochitest-e10s-1"], + "xpcshell": ["xpcshell"], + "chrome": ["mochitest-o"], + "browser-a11y": ["mochitest-ba"], + "browser-media": ["mochitest-bmda"], + "browser-chrome": [ + "mochitest-browser-chrome-1", + "mochitest-e10s-browser-chrome-1", + "mochitest-browser-chrome-e10s-1", + ], + "devtools-chrome": [ + "mochitest-devtools-chrome-1", + "mochitest-e10s-devtools-chrome-1", + "mochitest-devtools-chrome-e10s-1", + ], + "crashtest": ["crashtest", "crashtest-e10s"], + "reftest": ["reftest", "reftest-e10s"], + "remote": ["mochitest-remote"], + "web-platform-tests": ["web-platform-tests-1"], + } + + flavor_suites = { + "mochitest": "mochitests", + "xpcshell": "xpcshell", + "chrome": "mochitest-o", + "browser-chrome": "mochitest-bc", + "browser-a11y": "mochitest-ba", + "browser-media": "mochitest-bmda", + "devtools-chrome": "mochitest-dt", + "crashtest": "crashtest", + "reftest": "reftest", + "web-platform-tests": "web-platform-tests", + } + + compiled_suites = [ + "cppunit", + "gtest", + "jittest", + ] + + common_suites = [ + "cppunit", + "crashtest", + "firefox-ui-functional", + "geckoview", + "geckoview-junit", + "gtest", + "jittest", + "jsreftest", + "marionette", + "marionette-e10s", + "mochitests", + "reftest", + "robocop", + "web-platform-tests", + "xpcshell", + ] + + def __init__(self): + self.topsrcdir = build.topsrcdir + self._resolver = None + + @property + def resolver(self): + if self._resolver is None: + self._resolver = TestResolver.from_environment(cwd=here) + return self._resolver + + @classmethod + def split_try_string(cls, data): + return re.findall(r"(?:\[.*?\]|\S)+", data) + + def paths_by_flavor(self, paths=None, tags=None): + paths_by_flavor = defaultdict(set) + + if not (paths or tags): + return dict(paths_by_flavor) + + tests = list(self.resolver.resolve_tests(paths=paths, tags=tags)) + + for t in tests: + if t["flavor"] in self.flavor_suites: + flavor = t["flavor"] + if "subsuite" in t and t["subsuite"] == "devtools": + flavor = "devtools-chrome" + + if "subsuite" in t and t["subsuite"] == "a11y": + flavor = "browser-a11y" + + if "subsuite" in t and t["subsuite"] == "media-bc": + flavor = "browser-media" + + if flavor in ["crashtest", "reftest"]: + manifest_relpath = os.path.relpath(t["manifest"], self.topsrcdir) + paths_by_flavor[flavor].add(os.path.dirname(manifest_relpath)) + elif "dir_relpath" in t: + paths_by_flavor[flavor].add(t["dir_relpath"]) + else: + file_relpath = os.path.relpath(t["path"], self.topsrcdir) + dir_relpath = os.path.dirname(file_relpath) + paths_by_flavor[flavor].add(dir_relpath) + + for flavor, path_set in paths_by_flavor.items(): + paths_by_flavor[flavor] = self.deduplicate_prefixes(path_set, paths) + + return dict(paths_by_flavor) + + def deduplicate_prefixes(self, path_set, input_paths): + # Removes paths redundant to test selection in the given path set. + # If a path was passed on the commandline that is the prefix of a + # path in our set, we only need to include the specified prefix to + # run the intended tests (every test in "layout/base" will run if + # "layout" is passed to the reftest harness). + removals = set() + additions = set() + + for path in path_set: + full_path = path + while path: + path, _ = os.path.split(path) + if path in input_paths: + removals.add(full_path) + additions.add(path) + + return additions | (path_set - removals) + + def remove_duplicates(self, paths_by_flavor, tests): + rv = {} + for item in paths_by_flavor: + if self.flavor_suites[item] not in tests: + rv[item] = paths_by_flavor[item].copy() + return rv + + def calc_try_syntax( + self, + platforms, + tests, + talos, + jobs, + builds, + paths_by_flavor, + tags, + extras, + intersection, + ): + parts = ["try:"] + + if platforms: + parts.extend(["-b", builds, "-p", ",".join(platforms)]) + + suites = tests if not intersection else {} + paths = set() + for flavor, flavor_tests in paths_by_flavor.items(): + suite = self.flavor_suites[flavor] + if suite not in suites and (not intersection or suite in tests): + for job_name in self.flavor_jobs[flavor]: + for test in flavor_tests: + paths.add("{}:{}".format(flavor, test)) + suites[job_name] = tests.get(suite, []) + + # intersection implies tests are expected + if intersection and not suites: + raise ValueError("No tests found matching filters") + + if extras.get("artifact") and any([p.endswith("-nightly") for p in platforms]): + print( + 'You asked for |--artifact| but "-nightly" platforms don\'t have artifacts. ' + "Running without |--artifact| instead." + ) + del extras["artifact"] + + if extras.get("artifact"): + rejected = [] + for suite in suites.keys(): + if any([suite.startswith(c) for c in self.compiled_suites]): + rejected.append(suite) + if rejected: + raise ValueError( + "You can't run {} with " + "--artifact option.".format(", ".join(rejected)) + ) + + if extras.get("artifact") and "all" in suites.keys(): + non_compiled_suites = set(self.common_suites) - set(self.compiled_suites) + message = ( + "You asked for |-u all| with |--artifact| but compiled-code tests ({tests})" + " can't run against an artifact build. Running (-u {non_compiled_suites}) " + "instead." + ) + string_format = { + "tests": ",".join(self.compiled_suites), + "non_compiled_suites": ",".join(non_compiled_suites), + } + print(message.format(**string_format)) + del suites["all"] + suites.update({suite_name: None for suite_name in non_compiled_suites}) + + if suites: + parts.append("-u") + parts.append( + ",".join( + "{}{}".format(k, "[%s]" % ",".join(v) if v else "") + for k, v in sorted(suites.items()) + ) + ) + + if talos: + parts.append("-t") + parts.append( + ",".join( + "{}{}".format(k, "[%s]" % ",".join(v) if v else "") + for k, v in sorted(talos.items()) + ) + ) + + if jobs: + parts.append("-j") + parts.append(",".join(jobs)) + + if tags: + parts.append(" ".join("--tag %s" % t for t in tags)) + + if paths: + parts.append("--try-test-paths %s" % " ".join(sorted(paths))) + + args_by_dest = { + v["dest"]: k for k, v in SyntaxParser.pass_through_arguments.items() + } + for dest, value in extras.items(): + assert dest in args_by_dest + arg = args_by_dest[dest] + action = SyntaxParser.pass_through_arguments[arg]["action"] + if action == "store": + parts.append(arg) + parts.append(value) + if action == "append": + for e in value: + parts.append(arg) + parts.append(e) + if action in ("store_true", "store_false"): + parts.append(arg) + + return " ".join(parts) + + def normalise_list(self, items, allow_subitems=False): + rv = defaultdict(list) + for item in items: + parsed = parse_arg(item) + for key, values in parsed.items(): + rv[key].extend(values) + + if not allow_subitems: + if not all(item == [] for item in rv.values()): + raise ValueError("Unexpected subitems in argument") + return rv.keys() + else: + return rv + + def validate_args(self, **kwargs): + tests_selected = kwargs["tests"] or kwargs["paths"] or kwargs["tags"] + if kwargs["platforms"] is None and (kwargs["jobs"] is None or tests_selected): + if "AUTOTRY_PLATFORM_HINT" in os.environ: + kwargs["platforms"] = [os.environ["AUTOTRY_PLATFORM_HINT"]] + elif tests_selected: + print("Must specify platform when selecting tests.") + sys.exit(1) + else: + print( + "Either platforms or jobs must be specified as an argument to autotry." + ) + sys.exit(1) + + try: + platforms = ( + self.normalise_list(kwargs["platforms"]) if kwargs["platforms"] else {} + ) + except ValueError as e: + print("Error parsing -p argument:\n%s" % e) + sys.exit(1) + + try: + tests = ( + self.normalise_list(kwargs["tests"], allow_subitems=True) + if kwargs["tests"] + else {} + ) + except ValueError as e: + print("Error parsing -u argument ({}):\n{}".format(kwargs["tests"], e)) + sys.exit(1) + + try: + talos = ( + self.normalise_list(kwargs["talos"], allow_subitems=True) + if kwargs["talos"] + else [] + ) + except ValueError as e: + print("Error parsing -t argument:\n%s" % e) + sys.exit(1) + + try: + jobs = self.normalise_list(kwargs["jobs"]) if kwargs["jobs"] else {} + except ValueError as e: + print("Error parsing -j argument:\n%s" % e) + sys.exit(1) + + paths = [] + for p in kwargs["paths"]: + p = mozpath.normpath(os.path.abspath(p)) + if not (os.path.isdir(p) and p.startswith(self.topsrcdir)): + print( + 'Specified path "%s" is not a directory under the srcdir,' + " unable to specify tests outside of the srcdir" % p + ) + sys.exit(1) + if len(p) <= len(self.topsrcdir): + print( + 'Specified path "%s" is at the top of the srcdir and would' + " select all tests." % p + ) + sys.exit(1) + paths.append(os.path.relpath(p, self.topsrcdir)) + + try: + tags = self.normalise_list(kwargs["tags"]) if kwargs["tags"] else [] + except ValueError as e: + print("Error parsing --tags argument:\n%s" % e) + sys.exit(1) + + extra_values = {k["dest"] for k in SyntaxParser.pass_through_arguments.values()} + extra_args = {k: v for k, v in kwargs.items() if k in extra_values and v} + + return kwargs["builds"], platforms, tests, talos, jobs, paths, tags, extra_args + + def run(self, **kwargs): + if not any(kwargs[item] for item in ("paths", "tests", "tags")): + kwargs["paths"] = set() + kwargs["tags"] = set() + + builds, platforms, tests, talos, jobs, paths, tags, extra = self.validate_args( + **kwargs + ) + + if paths or tags: + paths = [ + os.path.relpath(os.path.normpath(os.path.abspath(item)), self.topsrcdir) + for item in paths + ] + paths_by_flavor = self.paths_by_flavor(paths=paths, tags=tags) + + if not paths_by_flavor and not tests: + print( + "No tests were found when attempting to resolve paths:\n\n\t%s" + % paths + ) + sys.exit(1) + + if not kwargs["intersection"]: + paths_by_flavor = self.remove_duplicates(paths_by_flavor, tests) + else: + paths_by_flavor = {} + + # No point in dealing with artifacts if we aren't running any builds + local_artifact_build = False + if platforms: + local_artifact_build = kwargs.get("local_artifact_build", False) + + # Add --artifact if --enable-artifact-builds is set ... + if local_artifact_build: + extra["artifact"] = True + # ... unless --no-artifact is explicitly given. + if kwargs["no_artifact"]: + if "artifact" in extra: + del extra["artifact"] + + try: + msg = self.calc_try_syntax( + platforms, + tests, + talos, + jobs, + builds, + paths_by_flavor, + tags, + extra, + kwargs["intersection"], + ) + except ValueError as e: + print(e) + sys.exit(1) + + if local_artifact_build and not kwargs["no_artifact"]: + print( + "mozconfig has --enable-artifact-builds; including " + "--artifact flag in try syntax (use --no-artifact " + "to override)" + ) + + if kwargs["verbose"] and paths_by_flavor: + print("The following tests will be selected: ") + for flavor, paths in paths_by_flavor.items(): + print("{}: {}".format(flavor, ",".join(paths))) + + if kwargs["verbose"]: + print("The following try syntax was calculated:\n%s" % msg) + + push_to_try( + "syntax", + kwargs["message"].format(msg=msg), + stage_changes=kwargs["stage_changes"], + dry_run=kwargs["dry_run"], + closed_tree=kwargs["closed_tree"], + ) + + +def run(**kwargs): + at = AutoTry() + return at.run(**kwargs) |