1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
#!/usr/bin/env python3
# Copyright Hans Dembinski 2019
# Distributed under the Boost Software License, Version 1.0.
# See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt
"""
This script runs the benchmarks on previous versions of this library to track changes
in performance.
Run this from a special build directory that uses the benchmark folder as root
cd my_build_dir
cmake ../benchmark
../run_benchmarks.py
This creates a database, benchmark_results. Plot it:
../plot_benchmarks.py
The script leaves the include folder in a modified state. To clean up, do:
git checkout HEAD -- ../include
git clean -f -- ../include
"""
import subprocess as subp
import tempfile
import os
import shelve
import json
import argparse
def get_commits():
commits = []
comments = {}
for line in subp.check_output(("git", "log", "--oneline")).decode("ascii").split("\n"):
if line:
ispace = line.index(" ")
hash = line[:ispace]
commits.append(hash)
comments[hash] = line[ispace+1:]
commits = commits[::-1]
return commits, comments
def recursion(results, commits, comments, ia, ib):
ic = int((ia + ib) / 2)
if ic == ia:
return
run(results, comments, commits[ic], False)
if all([results[commits[i]] is None for i in (ia, ib, ic)]):
return
recursion(results, commits, comments, ic, ib)
recursion(results, commits, comments, ia, ic)
def run(results, comments, hash, update):
if not update and hash in results:
return
print(hash, comments[hash])
subp.call(("rm", "-rf", "../include"))
if subp.call(("git", "checkout", hash, "--", "../include")) != 0:
print("[Benchmark] Cannot checkout include folder\n")
return
print(hash, "make")
with tempfile.TemporaryFile() as out:
if subp.call(("make", "-j4", "histogram_filling"), stdout=out, stderr=out) != 0:
print("[Benchmark] Cannot make benchmarks\n")
out.seek(0)
print(out.read().decode("utf-8") + "\n")
return
print(hash, "run")
s = subp.check_output(("./histogram_filling", "--benchmark_format=json", "--benchmark_filter=normal"))
d = json.loads(s)
if update and hash in results and results[hash] is not None:
d2 = results[hash]
for i, (b, b2) in enumerate(zip(d["benchmarks"], d2["benchmarks"])):
d["benchmarks"][i] = b if b["cpu_time"] < b2["cpu_time"] else b2
results[hash] = d
for benchmark in d["benchmarks"]:
print(benchmark["name"], min(benchmark["real_time"], benchmark["cpu_time"]))
def main():
commits, comments = get_commits()
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("first", type=str, default="begin",
help="first commit in range, special value `begin` is allowed")
parser.add_argument("last", type=str, default="end",
help="last commit in range, special value `end` is allowed")
parser.add_argument("-f", action="store_true",
help="override previous results")
args = parser.parse_args()
if args.first == "begin":
args.first = commits[0]
if args.last == "end":
args.last = commits[-1]
with shelve.open("benchmark_results") as results:
a = commits.index(args.first)
b = commits.index(args.last)
if args.f:
for hash in commits[a:b+1]:
del results[hash]
run(results, comments, args.first, False)
run(results, comments, args.last, False)
recursion(results, commits, comments, a, b)
if __name__ == "__main__":
main()
|
