diff options
Diffstat (limited to 'taskcluster/taskgraph/transforms/perftest.py')
| -rw-r--r-- | taskcluster/taskgraph/transforms/perftest.py | 296 |
1 files changed, 296 insertions, 0 deletions
diff --git a/taskcluster/taskgraph/transforms/perftest.py b/taskcluster/taskgraph/transforms/perftest.py new file mode 100644 index 0000000000..dcbb9c5bcc --- /dev/null +++ b/taskcluster/taskgraph/transforms/perftest.py @@ -0,0 +1,296 @@ +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +This transform passes options from `mach perftest` to the corresponding task. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from copy import deepcopy +from datetime import date, timedelta +import json + +from six import ensure_text, text_type + +from voluptuous import ( + Any, + Optional, + Extra, +) + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema +from taskgraph.util.treeherder import split_symbol, join_symbol + + +transforms = TransformSequence() + + +perftest_description_schema = Schema( + { + # The test names and the symbols to use for them: [test-symbol, test-path] + Optional("perftest"): [[text_type]], + # Metrics to gather for the test. These will be merged + # with options specified through perftest-perfherder-global + Optional("perftest-metrics"): optionally_keyed_by( + "perftest", + Any( + [text_type], + {text_type: Any(None, {text_type: Any(None, text_type, [text_type])})}, + ), + ), + # Perfherder data options that will be applied to + # all metrics gathered. + Optional("perftest-perfherder-global"): optionally_keyed_by( + "perftest", {text_type: Any(None, text_type, [text_type])} + ), + # Extra options to add to the test's command + Optional("perftest-extra-options"): optionally_keyed_by( + "perftest", [text_type] + ), + # Variants of the test to make based on extra browsertime + # arguments. Expecting: + # [variant-suffix, options-to-use] + # If variant-suffix is `null` then the options will be added + # to the existing task. Otherwise, a new variant is created + # with the given suffix and with its options replaced. + Optional("perftest-btime-variants"): optionally_keyed_by( + "perftest", [[Any(None, text_type)]] + ), + # These options will be parsed in the next schemas + Extra: object, + } +) + + +transforms.add_validate(perftest_description_schema) + + +@transforms.add +def split_tests(config, jobs): + for job in jobs: + if job.get("perftest") is None: + yield job + continue + + for test_symbol, test_name in job.pop("perftest"): + job_new = deepcopy(job) + + job_new["perftest"] = test_symbol + job_new["name"] += "-" + test_symbol + job_new["treeherder"]["symbol"] = job["treeherder"]["symbol"].format( + symbol=test_symbol + ) + job_new["run"]["command"] = job["run"]["command"].replace( + "{perftest_testname}", test_name + ) + + yield job_new + + +@transforms.add +def handle_keyed_by_perftest(config, jobs): + fields = ["perftest-metrics", "perftest-extra-options", "perftest-btime-variants"] + for job in jobs: + if job.get("perftest") is None: + yield job + continue + + for field in fields: + resolve_keyed_by(job, field, item_name=job["name"]) + + job.pop("perftest") + yield job + + +@transforms.add +def parse_perftest_metrics(config, jobs): + """Parse the metrics into a dictionary immediately. + + This way we can modify the extraOptions field (and others) entry through the + transforms that come later. The metrics aren't formatted until the end of the + transforms. + """ + for job in jobs: + if job.get("perftest-metrics") is None: + yield job + continue + perftest_metrics = job.pop("perftest-metrics") + + # If perftest metrics is a string, split it up first + if isinstance(perftest_metrics, list): + new_metrics_info = [{"name": metric} for metric in perftest_metrics] + else: + new_metrics_info = [] + for metric, options in perftest_metrics.items(): + entry = {"name": metric} + entry.update(options) + new_metrics_info.append(entry) + + job["perftest-metrics"] = new_metrics_info + yield job + + +@transforms.add +def split_perftest_variants(config, jobs): + for job in jobs: + if job.get("variants") is None: + yield job + continue + + for variant in job.pop("variants"): + job_new = deepcopy(job) + + group, symbol = split_symbol(job_new["treeherder"]["symbol"]) + group += "-" + variant + job_new["treeherder"]["symbol"] = join_symbol(group, symbol) + job_new["name"] += "-" + variant + job_new.setdefault("perftest-perfherder-global", {}).setdefault( + "extraOptions", [] + ).append(variant) + job_new[variant] = True + + yield job_new + + yield job + + +@transforms.add +def split_btime_variants(config, jobs): + for job in jobs: + if job.get("perftest-btime-variants") is None: + yield job + continue + + variants = job.pop("perftest-btime-variants") + if not variants: + yield job + continue + + yield_existing = False + for suffix, options in variants: + if suffix is None: + # Append options to the existing job + job.setdefault("perftest-btime-variants", []).append(options) + yield_existing = True + else: + job_new = deepcopy(job) + group, symbol = split_symbol(job_new["treeherder"]["symbol"]) + symbol += "-" + suffix + job_new["treeherder"]["symbol"] = join_symbol(group, symbol) + job_new["name"] += "-" + suffix + job_new.setdefault("perftest-perfherder-global", {}).setdefault( + "extraOptions", [] + ).append(suffix) + # Replace the existing options with the new ones + job_new["perftest-btime-variants"] = [options] + yield job_new + + # The existing job has been modified so we should also return it + if yield_existing: + yield job + + +@transforms.add +def setup_http3_tests(config, jobs): + for job in jobs: + if job.get("http3") is None or not job.pop("http3"): + yield job + continue + job.setdefault("perftest-btime-variants", []).append( + "firefox.preference=network.http.http3.enabled:true" + ) + yield job + + +@transforms.add +def setup_perftest_metrics(config, jobs): + for job in jobs: + if job.get("perftest-metrics") is None: + yield job + continue + perftest_metrics = job.pop("perftest-metrics") + + # Options to apply to each metric + global_options = job.pop("perftest-perfherder-global", {}) + for metric_info in perftest_metrics: + for opt, val in global_options.items(): + if isinstance(val, list) and opt in metric_info: + metric_info[opt].extend(val) + elif not (isinstance(val, list) and len(val) == 0): + metric_info[opt] = val + + quote_escape = '\\"' + if "win" in job.get("platform", ""): + # Escaping is a bit different on windows platforms + quote_escape = '\\\\\\"' + + job["run"]["command"] = job["run"]["command"].replace( + "{perftest_metrics}", + " ".join( + [ + ",".join( + [ + ":".join( + [ + option, + str(value) + .replace(" ", "") + .replace("'", quote_escape), + ] + ) + for option, value in metric_info.items() + ] + ) + for metric_info in perftest_metrics + ] + ), + ) + + yield job + + +@transforms.add +def setup_perftest_browsertime_variants(config, jobs): + for job in jobs: + if job.get("perftest-btime-variants") is None: + yield job + continue + + job["run"]["command"] += " --browsertime-extra-options %s" % ",".join( + [opt.strip() for opt in job.pop("perftest-btime-variants")] + ) + + yield job + + +@transforms.add +def setup_perftest_extra_options(config, jobs): + for job in jobs: + if job.get("perftest-extra-options") is None: + yield job + continue + job["run"]["command"] += " " + " ".join(job.pop("perftest-extra-options")) + yield job + + +@transforms.add +def pass_perftest_options(config, jobs): + for job in jobs: + env = job.setdefault("worker", {}).setdefault("env", {}) + env["PERFTEST_OPTIONS"] = ensure_text( + json.dumps(config.params["try_task_config"].get("perftest-options")) + ) + yield job + + +@transforms.add +def setup_perftest_test_date(config, jobs): + for job in jobs: + if ( + job.get("attributes", {}).get("batch", False) + and "--test-date" not in job["run"]["command"] + ): + yesterday = (date.today() - timedelta(1)).strftime("%Y.%m.%d") + job["run"]["command"] += " --test-date %s" % yesterday + yield job |