diff options
Diffstat (limited to '')
| -rw-r--r-- | js/src/tests/parsemark.py | 257 |
1 files changed, 257 insertions, 0 deletions
diff --git a/js/src/tests/parsemark.py b/js/src/tests/parsemark.py new file mode 100644 index 0000000000..199747e7dd --- /dev/null +++ b/js/src/tests/parsemark.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python + +"""%prog [options] shellpath dirpath + +Pulls performance data on parsing via the js shell. +Displays the average number of milliseconds it took to parse each file. + +For comparison, something apparently approximating a t-test is performed: +"Faster" means that: + + t_baseline_goodrun = (t_baseline_avg - t_baseline_stddev) + t_current_badrun = (t_current_avg + t_current_stddev) + t_current_badrun < t_baseline_goodrun + +Effectively, a bad run from the current data is better than a good run from the +baseline data, we're probably faster. A similar computation is used for +determining the "slower" designation. + +Arguments: + shellpath executable JavaScript shell + dirpath directory filled with parsilicious js files +""" + +import json +import math +import optparse +import os +import subprocess as subp +import sys +from string import Template + +try: + import compare_bench +except ImportError: + compare_bench = None + + +_DIR = os.path.dirname(__file__) +JS_CODE_TEMPLATE = Template( + """ +if (typeof snarf !== 'undefined') read = snarf +var contents = read("$filepath"); +$prepare +for (var i = 0; i < $warmup_run_count; i++) + $func(contents, $options); +var results = []; +for (var i = 0; i < $real_run_count; i++) { + var start = elapsed() / 1000; + $func(contents, $options); + var end = elapsed() / 1000; + results.push(end - start); +} +print(results); +""" +) + + +def gen_filepaths(dirpath, target_ext=".js"): + for filename in os.listdir(dirpath): + if filename.endswith(target_ext): + yield os.path.join(dirpath, filename) + + +def avg(seq): + return sum(seq) / len(seq) + + +def stddev(seq, mean): + diffs = ((float(item) - mean) ** 2 for item in seq) + return math.sqrt(sum(diffs) / len(seq)) + + +def bench( + shellpath, filepath, warmup_runs, counted_runs, prepare, func, options, stfu=False +): + """Return a list of milliseconds for the counted runs.""" + assert '"' not in filepath + code = JS_CODE_TEMPLATE.substitute( + filepath=filepath, + warmup_run_count=warmup_runs, + real_run_count=counted_runs, + prepare=prepare, + func=func, + options=options, + ) + proc = subp.Popen([shellpath, "-e", code], stdout=subp.PIPE) + stdout, _ = proc.communicate() + milliseconds = [float(val) for val in stdout.decode().split(",")] + mean = avg(milliseconds) + sigma = stddev(milliseconds, mean) + if not stfu: + print("Runs:", [int(ms) for ms in milliseconds]) + print("Mean:", mean) + print("Stddev: {:.2f} ({:.2f}% of mean)".format(sigma, sigma / mean * 100)) + return mean, sigma + + +def parsemark(filepaths, fbench, stfu=False): + """:param fbench: fbench(filename) -> float""" + bench_map = {} # {filename: (avg, stddev)} + for filepath in filepaths: + filename = os.path.split(filepath)[-1] + if not stfu: + print("Parsemarking {}...".format(filename)) + bench_map[filename] = fbench(filepath) + print("{") + for i, (filename, (avg, stddev)) in enumerate(iter(bench_map.items())): + assert '"' not in filename + fmt = ' {:30s}: {{"average_ms": {:6.2f}, "stddev_ms": {:6.2f}}}' + if i != len(bench_map) - 1: + fmt += "," + filename_str = '"{}"'.format(filename) + print(fmt.format(filename_str, avg, stddev)) + print("}") + return dict( + (filename, dict(average_ms=avg, stddev_ms=stddev)) + for filename, (avg, stddev) in iter(bench_map.items()) + ) + + +def main(): + parser = optparse.OptionParser(usage=__doc__.strip()) + parser.add_option( + "-w", + "--warmup-runs", + metavar="COUNT", + type=int, + default=5, + help="used to minimize test instability [%default]", + ) + parser.add_option( + "-c", + "--counted-runs", + metavar="COUNT", + type=int, + default=50, + help="timed data runs that count towards the average" " [%default]", + ) + parser.add_option( + "-s", + "--shell", + metavar="PATH", + help="explicit shell location; when omitted, will look" " in likely places", + ) + parser.add_option( + "-b", + "--baseline", + metavar="JSON_PATH", + dest="baseline_path", + help="json file with baseline values to " "compare against", + ) + parser.add_option( + "--mode", + dest="mode", + type="choice", + choices=("parse", "dumpStencil", "compile", "decode"), + default="parse", + help="The target of the benchmark (parse/dumpStencil/compile/decode), defaults to parse", + ) + parser.add_option( + "--lazy", + dest="lazy", + action="store_true", + default=False, + help="Use lazy parsing when compiling", + ) + parser.add_option( + "-q", + "--quiet", + dest="stfu", + action="store_true", + default=False, + help="only print JSON to stdout [%default]", + ) + options, args = parser.parse_args() + try: + shellpath = args.pop(0) + except IndexError: + parser.print_help() + print() + print("error: shellpath required", file=sys.stderr) + return -1 + try: + dirpath = args.pop(0) + except IndexError: + parser.print_help() + print() + print("error: dirpath required", file=sys.stderr) + return -1 + if not shellpath or not os.path.exists(shellpath): + print("error: could not find shell:", shellpath, file=sys.stderr) + return -1 + if options.baseline_path: + if not os.path.isfile(options.baseline_path): + print("error: baseline file does not exist", file=sys.stderr) + return -1 + if not compare_bench: + print( + "error: JSON support is missing, cannot compare benchmarks", + file=sys.stderr, + ) + return -1 + + if options.lazy and options.mode == "parse": + print( + "error: parse mode doesn't support lazy", + file=sys.stderr, + ) + return -1 + + funcOpt = {} + if options.mode == "decode": + encodeOpt = {} + encodeOpt["execute"] = False + encodeOpt["saveIncrementalBytecode"] = True + if not options.lazy: + encodeOpt["forceFullParse"] = True + + # In order to test the decoding, we first have to encode the content. + prepare = Template( + """ +contents = cacheEntry(contents); +evaluate(contents, $options); +""" + ).substitute(options=json.dumps(encodeOpt)) + + func = "evaluate" + funcOpt["execute"] = False + funcOpt["loadBytecode"] = True + if not options.lazy: + funcOpt["forceFullParse"] = True + else: + prepare = "" + func = options.mode + if not options.lazy: + funcOpt["forceFullParse"] = True + + def benchfile(filepath): + return bench( + shellpath, + filepath, + options.warmup_runs, + options.counted_runs, + prepare, + func, + json.dumps(funcOpt), + stfu=options.stfu, + ) + + bench_map = parsemark(gen_filepaths(dirpath), benchfile, options.stfu) + if options.baseline_path: + compare_bench.compare_immediate(bench_map, options.baseline_path) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) |