1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
module to handle Gecko profiling.
"""
import json
import os
import tempfile
import zipfile
import mozfile
from mozgeckoprofiler import ProfileSymbolicator, save_gecko_profile
from mozlog import get_proxy_logger
LOG = get_proxy_logger()
class GeckoProfile(object):
"""
Handle Gecko profiling.
This allow to collect Gecko profiling data and to zip results in one file.
"""
def __init__(self, upload_dir, browser_config, test_config):
self.upload_dir = upload_dir
self.browser_config, self.test_config = browser_config, test_config
self.cleanup = True
# Create a temporary directory into which the tests can put
# their profiles. These files will be assembled into one big
# zip file later on, which is put into the MOZ_UPLOAD_DIR.
gecko_profile_dir = tempfile.mkdtemp()
gecko_profile_interval = test_config.get("gecko_profile_interval", 1)
# Default number of entries is 128MiB.
# This value is calculated by dividing the 128MiB of memory by 8 because
# the profiler uses 8 bytes per entry.
gecko_profile_entries = test_config.get(
"gecko_profile_entries", int(128 * 1024 * 1024 / 8)
)
gecko_profile_features = test_config.get(
"gecko_profile_features", "js,stackwalk,cpu,screenshots"
)
gecko_profile_threads = test_config.get(
"gecko_profile_threads", "GeckoMain,Compositor,Renderer"
)
gecko_profile_extra_threads = test_config.get(
"gecko_profile_extra_threads", None
)
if gecko_profile_extra_threads:
gecko_profile_threads += "," + gecko_profile_extra_threads
# Make sure no archive already exists in the location where
# we plan to output our profiler archive
# If individual talos is ran (--activeTest) instead of suite (--suite)
# the "suite" key will be empty and we'll name the profile after
# the test name
self.profile_arcname = os.path.join(
self.upload_dir,
"profile_{0}.zip".format(test_config.get("suite", test_config["name"])),
)
# We delete the archive if the current test is the first in the suite
if test_config.get("is_first_test", False):
LOG.info("Clearing archive {0}".format(self.profile_arcname))
mozfile.remove(self.profile_arcname)
self.symbol_paths = {
"FIREFOX": tempfile.mkdtemp(),
"THUNDERBIRD": tempfile.mkdtemp(),
"WINDOWS": tempfile.mkdtemp(),
}
LOG.info(
"Activating Gecko Profiling. Temp. profile dir:"
" {0}, interval: {1}, entries: {2}".format(
gecko_profile_dir, gecko_profile_interval, gecko_profile_entries
)
)
self.profiling_info = {
"gecko_profile_interval": gecko_profile_interval,
"gecko_profile_entries": gecko_profile_entries,
"gecko_profile_dir": gecko_profile_dir,
"gecko_profile_features": gecko_profile_features,
"gecko_profile_threads": gecko_profile_threads,
}
def option(self, name):
return self.profiling_info["gecko_profile_" + name]
def update_env(self, env):
"""
update the given env to update some env vars if required.
"""
if not self.test_config.get("gecko_profile_startup"):
return
# Set environment variables which will cause profiling to
# start as early as possible. These are consumed by Gecko
# itself, not by Talos JS code.
env.update(
{
"MOZ_PROFILER_STARTUP": "1",
# Temporary: Don't run Base Profiler, see bug 1630448.
# TODO: Remove when fix lands in bug 1648324 or bug 1648325.
"MOZ_PROFILER_STARTUP_NO_BASE": "1",
"MOZ_PROFILER_STARTUP_INTERVAL": str(self.option("interval")),
"MOZ_PROFILER_STARTUP_ENTRIES": str(self.option("entries")),
"MOZ_PROFILER_STARTUP_FEATURES": str(self.option("features")),
"MOZ_PROFILER_STARTUP_FILTERS": str(self.option("threads")),
}
)
def _save_gecko_profile(
self, cycle, symbolicator, missing_symbols_zip, profile_path
):
try:
with open(profile_path, "r", encoding="utf-8") as profile_file:
profile = json.load(profile_file)
symbolicator.dump_and_integrate_missing_symbols(
profile, missing_symbols_zip
)
symbolicator.symbolicate_profile(profile)
save_gecko_profile(profile, profile_path)
except MemoryError:
LOG.critical(
"Ran out of memory while trying"
" to symbolicate profile {0} (cycle {1})".format(profile_path, cycle),
exc_info=True,
)
except Exception:
LOG.critical(
"Encountered an exception during profile"
" symbolication {0} (cycle {1})".format(profile_path, cycle),
exc_info=True,
)
def symbolicate(self, cycle):
"""
Symbolicate Gecko profiling data for one cycle.
:param cycle: the number of the cycle of the test currently run.
"""
symbolicator = ProfileSymbolicator(
{
# Trace-level logging (verbose)
"enableTracing": 0,
# Fallback server if symbol is not found locally
"remoteSymbolServer": "https://symbolication.services.mozilla.com/symbolicate/v4",
# Maximum number of symbol files to keep in memory
"maxCacheEntries": 2000000,
# Frequency of checking for recent symbols to
# cache (in hours)
"prefetchInterval": 12,
# Oldest file age to prefetch (in hours)
"prefetchThreshold": 48,
# Maximum number of library versions to pre-fetch
# per library
"prefetchMaxSymbolsPerLib": 3,
# Default symbol lookup directories
"defaultApp": "FIREFOX",
"defaultOs": "WINDOWS",
# Paths to .SYM files, expressed internally as a
# mapping of app or platform names to directories
# Note: App & OS names from requests are converted
# to all-uppercase internally
"symbolPaths": self.symbol_paths,
}
)
if self.browser_config["symbols_path"]:
if mozfile.is_url(self.browser_config["symbols_path"]):
symbolicator.integrate_symbol_zip_from_url(
self.browser_config["symbols_path"]
)
elif os.path.isfile(self.browser_config["symbols_path"]):
symbolicator.integrate_symbol_zip_from_file(
self.browser_config["symbols_path"]
)
elif os.path.isdir(self.browser_config["symbols_path"]):
sym_path = self.browser_config["symbols_path"]
symbolicator.options["symbolPaths"]["FIREFOX"] = sym_path
self.cleanup = False
missing_symbols_zip = os.path.join(self.upload_dir, "missingsymbols.zip")
try:
mode = zipfile.ZIP_DEFLATED
except NameError:
mode = zipfile.ZIP_STORED
gecko_profile_dir = self.option("dir")
with zipfile.ZipFile(self.profile_arcname, "a", mode) as arc:
# Collect all individual profiles that the test
# has put into gecko_profile_dir.
for profile_filename in os.listdir(gecko_profile_dir):
testname = profile_filename
if testname.endswith(".profile"):
testname = testname[0:-8]
profile_path = os.path.join(gecko_profile_dir, profile_filename)
self._save_gecko_profile(
cycle, symbolicator, missing_symbols_zip, profile_path
)
# Our zip will contain one directory per subtest,
# and each subtest directory will contain one or
# more cycle_i.profile files. For example, with
# test_config['name'] == 'tscrollx',
# profile_filename == 'iframe.svg.profile', i == 0,
# we'll get path_in_zip ==
# 'profile_tscrollx/iframe.svg/cycle_0.profile'.
cycle_name = "cycle_{0}.profile".format(cycle)
path_in_zip = os.path.join(
"profile_{0}".format(self.test_config["name"]), testname, cycle_name
)
LOG.info(
"Adding profile {0} to archive {1}".format(
path_in_zip, self.profile_arcname
)
)
try:
arc.write(profile_path, path_in_zip)
except Exception:
LOG.exception(
"Failed to copy profile {0} as {1} to"
" archive {2}".format(
profile_path, path_in_zip, self.profile_arcname
)
)
# save the latest gecko profile archive to an env var, so later on
# it can be viewed automatically via the view-gecko-profile tool
os.environ["TALOS_LATEST_GECKO_PROFILE_ARCHIVE"] = self.profile_arcname
def clean(self):
"""
Clean up temp folders created with the instance creation.
"""
mozfile.remove(self.option("dir"))
if self.cleanup:
for symbol_path in self.symbol_paths.values():
mozfile.remove(symbol_path)
|
