1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import errno
import json
import os
import sys
from pathlib import Path
from mozfile import which
from mozperftest.layers import Layer
from mozperftest.utils import run_script, silence
METRICS_FIELDS = (
"SpeedIndex",
"FirstVisualChange",
"LastVisualChange",
"VisualProgress",
"videoRecordingStart",
)
class VisualData:
def open_data(self, data):
res = {
"name": "visualmetrics",
"subtest": data["name"],
"data": [
{"file": "visualmetrics", "value": value, "xaxis": xaxis}
for xaxis, value in enumerate(data["values"])
],
}
return res
def transform(self, data):
return data
def merge(self, data):
return data
class VisualMetrics(Layer):
"""Wrapper around Browsertime's visualmetrics.py script"""
name = "visualmetrics"
activated = False
arguments = {}
def setup(self):
self.metrics = {}
self.metrics_fields = []
# making sure we have ffmpeg and imagemagick available
for tool in ("ffmpeg", "convert"):
if sys.platform in ("win32", "msys"):
tool += ".exe"
path = which(tool)
if not path:
raise OSError(errno.ENOENT, f"Could not find {tool}")
def run(self, metadata):
if "VISUALMETRICS_PY" not in os.environ:
raise OSError(
"The VISUALMETRICS_PY environment variable is not set."
"Make sure you run the browsertime layer"
)
path = Path(os.environ["VISUALMETRICS_PY"])
if not path.exists():
raise FileNotFoundError(str(path))
self.visualmetrics = path
treated = 0
for result in metadata.get_results():
result_dir = result.get("results")
if result_dir is None:
continue
result_dir = Path(result_dir)
if not result_dir.is_dir():
continue
browsertime_json = Path(result_dir, "browsertime.json")
if not browsertime_json.exists():
continue
treated += self.run_visual_metrics(browsertime_json)
self.info(f"Treated {treated} videos.")
if len(self.metrics) > 0:
metadata.add_result(
{
"name": metadata.script["name"] + "-vm",
"framework": {"name": "mozperftest"},
"transformer": "mozperftest.metrics.visualmetrics:VisualData",
"results": list(self.metrics.values()),
}
)
# we also extend --perfherder-metrics and --console-metrics if they
# are activated
def add_to_option(name):
existing = self.get_arg(name, [])
for field in self.metrics_fields:
existing.append({"name": field, "unit": "ms"})
self.env.set_arg(name, existing)
if self.get_arg("perfherder"):
add_to_option("perfherder-metrics")
if self.get_arg("console"):
add_to_option("console-metrics")
else:
self.warning("No video was treated.")
return metadata
def run_visual_metrics(self, browsertime_json):
verbose = self.get_arg("verbose")
self.info(f"Looking at {browsertime_json}")
venv = self.mach_cmd.virtualenv_manager
class _display:
def __enter__(self, *args, **kw):
return self
__exit__ = __enter__
may_silence = not verbose and silence or _display
with browsertime_json.open() as f:
browsertime_json_data = json.loads(f.read())
videos = 0
global_options = [
str(self.visualmetrics),
"--orange",
"--perceptual",
"--contentful",
"--force",
"--renderignore",
"5",
"--viewport",
]
if verbose:
global_options += ["-vvv"]
for site in browsertime_json_data:
# collecting metrics from browserScripts
# because it can be used in splitting
for index, bs in enumerate(site["browserScripts"]):
for name, val in bs.items():
if not isinstance(val, (str, int)):
continue
self.append_metrics(index, name, val)
extra = {"lowerIsBetter": True, "unit": "ms"}
for index, video in enumerate(site["files"]["video"]):
videos += 1
video_path = browsertime_json.parent / video
output = "[]"
with may_silence():
res, output = run_script(
venv.python_path,
global_options + ["--video", str(video_path), "--json"],
verbose=verbose,
label="visual metrics",
display=False,
)
if not res:
self.error(f"Failed {res}")
continue
output = output.strip()
if verbose:
self.info(str(output))
try:
output = json.loads(output)
except json.JSONDecodeError:
self.error("Could not read the json output from visualmetrics.py")
continue
for name, value in output.items():
if name.endswith(
"Progress",
):
self._expand_visual_progress(index, name, value, **extra)
else:
self.append_metrics(index, name, value, **extra)
return videos
def _expand_visual_progress(self, index, name, value, **fields):
def _split_percent(val):
# value is of the form "567=94%"
val = val.split("=")
value, percent = val[0].strip(), val[1].strip()
if percent.endswith("%"):
percent = percent[:-1]
return int(percent), int(value)
percents = [_split_percent(elmt) for elmt in value.split(",")]
# we want to keep the first added value for each percent
# so the trick here is to create a dict() with the reversed list
percents = dict(reversed(percents))
# we are keeping the last 5 percents
percents = list(percents.items())
percents.sort()
for percent, value in percents[:5]:
self.append_metrics(index, f"{name}{percent}", value, **fields)
def append_metrics(self, index, name, value, **fields):
if name not in self.metrics_fields:
self.metrics_fields.append(name)
if name not in self.metrics:
self.metrics[name] = {"name": name, "values": []}
self.metrics[name]["values"].append(value)
self.metrics[name].update(**fields)
|
