1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
|
#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import shutil
from logger.logger import RaptorLogger
from mozpower import MozPower
from mozrunner import runners
from outputhandler import OutputHandler
from perftest import PerftestDesktop
from .base import WebExtension
LOG = RaptorLogger(component="raptor-webext-desktop")
class WebExtensionDesktop(PerftestDesktop, WebExtension):
def __init__(self, *args, **kwargs):
super(WebExtensionDesktop, self).__init__(*args, **kwargs)
# create the desktop browser runner
LOG.info("creating browser runner using mozrunner")
self.output_handler = OutputHandler(verbose=self.config["verbose"])
process_args = {"processOutputLine": [self.output_handler]}
firefox_args = ["--allow-downgrade"]
runner_cls = runners[self.config["app"]]
self.runner = runner_cls(
self.config["binary"],
profile=self.profile,
cmdargs=firefox_args,
process_args=process_args,
symbols_path=self.config["symbols_path"],
)
# Force Firefox to immediately exit for content crashes
self.runner.env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
self.runner.env.update(self.config.get("environment", {}))
def launch_desktop_browser(self, test):
raise NotImplementedError
def start_runner_proc(self):
# launch the browser via our previously-created runner
self.runner.start()
proc = self.runner.process_handler
self.output_handler.proc = proc
# give our control server the browser process so it can shut it down later
self.control_server.browser_proc = proc
def process_exists(self):
return self.runner.is_running()
def run_test(self, test, timeout):
# tests will be run warm (i.e. NO browser restart between page-cycles)
# unless otheriwse specified in the test INI by using 'cold = true'
mozpower_measurer = None
if self.config.get("power_test", False):
powertest_name = test["name"].replace("/", "-").replace("\\", "-")
output_dir = os.path.join(
self.artifact_dir, "power-measurements-%s" % powertest_name
)
test_dir = os.path.join(output_dir, powertest_name)
try:
if not os.path.exists(output_dir):
os.mkdir(output_dir)
if not os.path.exists(test_dir):
os.mkdir(test_dir)
except Exception:
LOG.critical(
"Could not create directories to store power testing data."
)
raise
# Start power measurements with IPG creating a power usage log
# every 30 seconds with 1 data point per second (or a 1000 milli-
# second sampling rate).
mozpower_measurer = MozPower(
ipg_measure_duration=30,
sampling_rate=1000,
output_file_path=os.path.join(test_dir, "power-usage"),
)
mozpower_measurer.initialize_power_measurements()
if self.config.get("cold") or test.get("cold"):
self.__run_test_cold(test, timeout)
else:
self.__run_test_warm(test, timeout)
if mozpower_measurer:
mozpower_measurer.finalize_power_measurements(test_name=test["name"])
perfherder_data = mozpower_measurer.get_perfherder_data()
if not self.config.get("run_local", False):
# when not running locally, zip the data and delete the folder which
# was placed in the zip
powertest_name = test["name"].replace("/", "-").replace("\\", "-")
power_data_path = os.path.join(
self.artifact_dir, "power-measurements-%s" % powertest_name
)
shutil.make_archive(power_data_path, "zip", power_data_path)
shutil.rmtree(power_data_path)
for data_type in perfherder_data:
self.control_server.submit_supporting_data(perfherder_data[data_type])
def __run_test_cold(self, test, timeout):
"""
Run the Raptor test but restart the entire browser app between page-cycles.
Note: For page-load tests, playback will only be started once - at the beginning of all
browser cycles, and then stopped after all cycles are finished. That includes the import
of the mozproxy ssl cert and turning on the browser proxy.
Since we're running in cold-mode, before this point (in manifest.py) the
'expected-browser-cycles' value was already set to the initial 'page-cycles' value;
and the 'page-cycles' value was set to 1 as we want to perform one page-cycle per
browser restart.
The 'browser-cycle' value is the current overall browser start iteration. The control
server will receive the current 'browser-cycle' and the 'expected-browser-cycles' in
each results set received; and will pass that on as part of the results so that the
results processing will know results for multiple browser cycles are being received.
The default will be to run in warm mode; unless 'cold = true' is set in the test INI.
"""
LOG.info(
"test %s is running in cold mode; browser WILL be restarted between "
"page cycles" % test["name"]
)
for test["browser_cycle"] in range(1, test["expected_browser_cycles"] + 1):
LOG.info(
"begin browser cycle %d of %d for test %s"
% (test["browser_cycle"], test["expected_browser_cycles"], test["name"])
)
self.run_test_setup(test)
if test["browser_cycle"] == 1:
if not self.is_localhost:
self.delete_proxy_settings_from_profile()
else:
# initial browser profile was already created before run_test was called;
# now additional browser cycles we want to create a new one each time
self.build_browser_profile()
# Update runner profile
self.runner.profile = self.profile
self.run_test_setup(test)
# now start the browser/app under test
self.launch_desktop_browser(test)
# set our control server flag to indicate we are running the browser/app
self.control_server._finished = False
self.wait_for_test_finish(test, timeout, self.process_exists)
def __run_test_warm(self, test, timeout):
self.run_test_setup(test)
if not self.is_localhost:
self.delete_proxy_settings_from_profile()
# start the browser/app under test
self.launch_desktop_browser(test)
# set our control server flag to indicate we are running the browser/app
self.control_server._finished = False
self.wait_for_test_finish(test, timeout, self.process_exists)
def run_test_teardown(self, test):
# browser should be closed by now but this is a backup-shutdown (if not in debug-mode)
if not self.debug_mode:
# If the runner was not started in the first place, stop() will silently
# catch RunnerNotStartedError
self.runner.stop()
else:
# in debug mode, and running locally, leave the browser running
if self.config["run_local"]:
LOG.info(
"* debug-mode enabled - please shutdown the browser manually..."
)
self.runner.wait(timeout=None)
super(WebExtensionDesktop, self).run_test_teardown(test)
def check_for_crashes(self):
super(WebExtensionDesktop, self).check_for_crashes()
try:
self.runner.check_for_crashes()
except NotImplementedError: # not implemented for Chrome
pass
self.crashes += self.runner.crashed
def clean_up(self):
self.runner.stop()
super(WebExtensionDesktop, self).clean_up()
class WebExtensionFirefox(WebExtensionDesktop):
def launch_desktop_browser(self, test):
LOG.info("starting %s" % self.config["app"])
if self.config["is_release_build"]:
self.disable_non_local_connections()
# if running debug-mode, tell Firefox to open the browser console on startup
if self.debug_mode:
self.runner.cmdargs.extend(["-jsconsole"])
self.start_runner_proc()
if self.config["is_release_build"] and test.get("playback") is not None:
self.enable_non_local_connections()
# if geckoProfile is enabled, initialize it
if self.config["gecko_profile"] is True:
self._init_gecko_profiling(test)
# tell the control server the gecko_profile dir; the control server
# will receive the filename of the stored gecko profile from the web
# extension, and will move it out of the browser user profile to
# this directory; where it is picked-up by gecko_profile.symbolicate
self.control_server.gecko_profile_dir = (
self.gecko_profiler.gecko_profile_dir
)
class WebExtensionDesktopChrome(WebExtensionDesktop):
def setup_chrome_args(self, test):
# Setup chrome args and add them to the runner's args
chrome_args = self.desktop_chrome_args(test)
if " ".join(chrome_args) not in " ".join(self.runner.cmdargs):
self.runner.cmdargs.extend(chrome_args)
def launch_desktop_browser(self, test):
LOG.info("starting %s" % self.config["app"])
# Setup chrome/chromium specific arguments then start the runner
self.setup_chrome_args(test)
self.start_runner_proc()
def set_browser_test_prefs(self, raw_prefs):
# add test-specific preferences
LOG.info(
"preferences were configured for the test, however \
we currently do not install them on non-Firefox browsers."
)
|
