diff options
Diffstat (limited to 'testing/runtimes/writeruntimes')
| -rwxr-xr-x | testing/runtimes/writeruntimes | 226 |
1 files changed, 226 insertions, 0 deletions
diff --git a/testing/runtimes/writeruntimes b/testing/runtimes/writeruntimes new file mode 100755 index 0000000000..67f6290701 --- /dev/null +++ b/testing/runtimes/writeruntimes @@ -0,0 +1,226 @@ +#!/bin/sh +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# The beginning of this script is both valid shell and valid python, +# such that the script starts with the shell and is reexecuted python +''':' +which mach > /dev/null 2>&1 && exec mach python "$0" "$@" || +echo "mach not found, either add it to your \$PATH or run this script via ./mach python testing/runtimes/writeruntimes"; exit # noqa +''' + +from __future__ import absolute_import, print_function + +import datetime +import json +import os +import sys +import time +from argparse import ArgumentParser +from collections import defaultdict + +import requests + +from moztest.resolve import ( + TestManifestLoader, + TestResolver, + TEST_SUITES, +) + +here = os.path.abspath(os.path.dirname(__file__)) +ACTIVE_DATA_URL = "https://activedata.allizom.org/query" +EXCEED_LIMIT = [ + # Suites that exceed 10,000 ActiveData result limit will be defined here. + 'web-platform-tests', + 'web-platform-tests-reftest', +] +MAX_RETRIES = 10 +RETRY_INTERVAL = 10 + + +def construct_query(suite, platform): + if platform in ('windows', 'android'): + platform_clause = '{"find":{"run.machine.platform": "%s"}}' % platform + else: + # Bundle macosx and linux results together - they are not too different. + platform_clause = ''' + { + "not": { + "or": [ + {"find":{"run.machine.platform": "windows"}}, + {"find":{"run.machine.platform": "android"}} + ] + } + } + ''' + + # Only use this if the suite being queried exceeeds 10,000 results. + output_clause = '"destination": "url",\n"format": "list",' if suite in EXCEED_LIMIT else '' + + query = """ +{ + "from":"unittest", + "limit":200000, + "groupby":["result.test"], + "select":{"value":"result.duration","aggregate":"median"}, + %s + "where":{"and":[ + {"eq":{"repo.branch.name": "mozilla-central"}}, + {"in":{"result.status": ["OK", "PASS", "FAIL"]}}, + {"gt":{"run.timestamp": {"date": "today-week"}}}, + {"eq":{"run.suite.fullname":"%s"}}, + %s + ]} +} +""" % (output_clause, suite, platform_clause) + + return query + + +def query_activedata(suite, platform): + query = construct_query(suite, platform) + print("Querying ActiveData for '{}' tests on '{}' platforms.. " + .format(suite, platform), end='') + sys.stdout.flush() + response = requests.post(ACTIVE_DATA_URL, + data=query, + stream=True) + response.raise_for_status() + + # Presence of destination clause in the query requires additional processing + # to produce the dataset that can be used. + if suite in EXCEED_LIMIT: + # The output_url is where result of the query will be stored. + output_url = response.json()["url"] + + tried = 0 + while tried < MAX_RETRIES: + # Use the requests.Session object to manage requests, since the output_url + # can often return 403 Forbidden. + session = requests.Session() + response = session.get(output_url) + if response.status_code == 200: + break + # A non-200 status code means we should retry after some wait. + time.sleep(RETRY_INTERVAL) + tried += 1 + + # Data returned from destination is in format of: + # {data: [result: {test: test_name, duration: duration}]} + # Normalize it to the format expected by compute_manifest_runtimes. + raw_data = response.json()["data"] + data = dict([[item['result']['test'], item['result']['duration']] for item in raw_data]) + else: + data = dict(response.json()["data"]) + + print("{} found".format(len(data))) + return data + + +def write_runtimes(manifest_runtimes, platform, suite, outdir=here): + if not os.path.exists(outdir): + os.makedirs(outdir) + + outfilename = os.path.join(outdir, "manifest-runtimes-{}.json".format(platform)) + # If file is not present, initialize a file with empty JSON object. + if not os.path.exists(outfilename): + with open(outfilename, 'w+') as f: + json.dump({}, f) + + # Load the entire file. + with open(outfilename, 'r') as f: + data = json.load(f) + + # Update the specific suite with the new runtime information and write to file. + data[suite] = manifest_runtimes + with open(outfilename, 'w') as f: + json.dump(data, f, indent=2, sort_keys=True) + + +def compute_manifest_runtimes(suite, platform): + resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader) + + crashtest_prefixes = { + 'http': '/tests/', + 'chrome': '/reftest/content/', + 'file': '/reftest/tests/', + } + manifest_runtimes = defaultdict(float) + data = query_activedata(suite, platform) + + if "web-platform-tests" in suite: + wpt_groups = {t["name"]: t["manifest"] + for t in resolver.resolve_tests(flavor="web-platform-tests")} + + for path, duration in data.items(): + # Returned data did not contain a test path, so go to next result. + if not path: + continue + + if suite in ('reftest', 'crashtest') and ' ' in path: + path = path.split()[0] + + if suite == 'crashtest' and '://' in path: + # Crashtest paths are URLs with various schemes and prefixes. + # Normalize it to become relative to mozilla-central. + scheme = path[:path.index('://')] + if ':' in scheme: + scheme = scheme.split(':')[-1] + prefix = crashtest_prefixes[scheme] + path = path.split(prefix, 1)[-1] + elif suite == 'xpcshell' and ':' in path: + path = path.split(':', 1)[-1] + + if "web-platform-tests" in suite: + if path in wpt_groups: + manifest_runtimes[wpt_groups[path]] += duration + continue + + if path not in resolver.tests_by_path: + continue + + for test in resolver.tests_by_path[path]: + manifest = test.get('ancestor_manifest') or test['manifest_relpath'] + manifest_runtimes[manifest] += duration + + manifest_runtimes = {k: round(v, 2) for k, v in manifest_runtimes.items()} + return manifest_runtimes + + +def cli(args=sys.argv[1:]): + default_suites = [suite for suite, obj in TEST_SUITES.items() if 'build_flavor' in obj] + default_platforms = ['android', 'windows', 'unix'] + + parser = ArgumentParser() + parser.add_argument('-o', '--output-directory', dest='outdir', default=here, + help="Directory to save runtime data.") + parser.add_argument('-s', '--suite', dest='suites', action='append', + default=None, choices=default_suites, + help="Suite(s) to include in the data set (default: all)") + parser.add_argument('-p', '--platform', dest='platforms', action='append', + default=None, choices=default_platforms, + help="Platform(s) to gather runtime information on " + "(default: all).") + args = parser.parse_args(args) + + # If a suite was specified, use that. Otherwise, use the full default set. + suites = args.suites or default_suites + # Same as above, but for the platform clause. + platforms = args.platforms or default_platforms + + for platform in platforms: + for suite in suites: + runtimes = compute_manifest_runtimes(suite, platform) + if not runtimes: + print("Not writing runtime data for '{}' for '{}' as no data was found".format(suite, platform)) + continue + + write_runtimes(runtimes, platform, suite, outdir=args.outdir) + + +if __name__ == "__main__": + sys.exit(cli()) |