1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# This module needs to stay Python 2 and 3 compatible
#
import datetime
import functools
import os
import shutil
import tarfile
import tempfile
import time
from mozprofile.prefs import Preferences
from condprof import progress
from condprof.changelog import Changelog
from condprof.util import (
TASK_CLUSTER,
ArchiveNotFound,
check_exists,
download_file,
logger,
)
TC_SERVICE = "https://firefox-ci-tc.services.mozilla.com"
ROOT_URL = TC_SERVICE + "/api/index"
INDEX_PATH = "gecko.v2.%(repo)s.latest.firefox.condprof-%(platform)s-%(scenario)s"
INDEX_BY_DATE_PATH = "gecko.v2.%(repo)s.pushdate.%(date)s.latest.firefox.condprof-%(platform)s-%(scenario)s"
PUBLIC_DIR = "artifacts/public/condprof"
TC_LINK = ROOT_URL + "/v1/task/" + INDEX_PATH + "/" + PUBLIC_DIR + "/"
TC_LINK_BY_DATE = ROOT_URL + "/v1/task/" + INDEX_BY_DATE_PATH + "/" + PUBLIC_DIR + "/"
ARTIFACT_NAME = "profile%(version)s-%(platform)s-%(scenario)s-%(customization)s.tgz"
CHANGELOG_LINK = (
ROOT_URL + "/v1/task/" + INDEX_PATH + "/" + PUBLIC_DIR + "/changelog.json"
)
ARTIFACTS_SERVICE = "https://taskcluster-artifacts.net"
DIRECT_LINK = ARTIFACTS_SERVICE + "/%(task_id)s/0/public/condprof/"
CONDPROF_CACHE = "~/.condprof-cache"
RETRIES = 3
RETRY_PAUSE = 45
class ServiceUnreachableError(Exception):
pass
class ProfileNotFoundError(Exception):
pass
class RetriesError(Exception):
pass
def _check_service(url):
"""Sanity check to see if we can reach the service root url."""
def _check():
exists, _ = check_exists(url, all_types=True)
if not exists:
raise ServiceUnreachableError(url)
try:
return _retries(_check)
except RetriesError:
raise ServiceUnreachableError(url)
def _check_profile(profile_dir):
"""Checks for prefs we need to remove or set."""
to_remove = ("gfx.blacklist.", "marionette.")
def _keep_pref(name, value):
for item in to_remove:
if not name.startswith(item):
continue
logger.info("Removing pref %s: %s" % (name, value))
return False
return True
def _clean_pref_file(name):
js_file = os.path.join(profile_dir, name)
prefs = Preferences.read_prefs(js_file)
cleaned_prefs = dict([pref for pref in prefs if _keep_pref(*pref)])
if name == "prefs.js":
# When we start Firefox, forces startupScanScopes to SCOPE_PROFILE (1)
# otherwise, side loading will be deactivated and the
# Raptor web extension won't be able to run.
cleaned_prefs["extensions.startupScanScopes"] = 1
# adding a marker so we know it's a conditioned profile
cleaned_prefs["profile.conditioned"] = True
with open(js_file, "w") as f:
Preferences.write(f, cleaned_prefs)
_clean_pref_file("prefs.js")
_clean_pref_file("user.js")
def _retries(callable, onerror=None, retries=RETRIES):
_retry_count = 0
pause = RETRY_PAUSE
while _retry_count < retries:
try:
return callable()
except Exception as e:
if onerror is not None:
onerror(e)
logger.info("Failed, retrying")
_retry_count += 1
time.sleep(pause)
pause *= 1.5
# If we reach that point, it means all attempts failed
if _retry_count >= RETRIES:
logger.error("All attempt failed")
else:
logger.info("Retried %s attempts and failed" % _retry_count)
raise RetriesError()
def get_profile(
target_dir,
platform,
scenario,
customization="default",
task_id=None,
download_cache=True,
repo="mozilla-central",
remote_test_root="/sdcard/test_root/",
version=None,
retries=RETRIES,
):
"""Extract a conditioned profile in the target directory.
If task_id is provided, will grab the profile from that task. when not
provided (default) will grab the latest profile.
"""
# XXX assert values
if version:
version = "-v%s" % version
else:
version = ""
# when we bump the Firefox version on trunk, autoland still needs to catch up
# in this case we want to download an older profile- 2 days to account for closures/etc.
oldday = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=2)
params = {
"platform": platform,
"scenario": scenario,
"customization": customization,
"task_id": task_id,
"repo": repo,
"version": version,
"date": str(oldday.date()).replace("-", "."),
}
logger.info("Getting conditioned profile with arguments: %s" % params)
filename = ARTIFACT_NAME % params
if task_id is None:
url = TC_LINK % params + filename
_check_service(TC_SERVICE)
else:
url = DIRECT_LINK % params + filename
_check_service(ARTIFACTS_SERVICE)
logger.info("preparing download dir")
if not download_cache:
download_dir = tempfile.mkdtemp()
else:
# using a cache dir in the user home dir
download_dir = os.path.expanduser(CONDPROF_CACHE)
if not os.path.exists(download_dir):
os.makedirs(download_dir)
downloaded_archive = os.path.join(download_dir, filename)
logger.info("Downloaded archive path: %s" % downloaded_archive)
def _get_profile():
logger.info("Getting %s" % url)
try:
archive = download_file(url, target=downloaded_archive)
except ArchiveNotFound:
raise ProfileNotFoundError(url)
try:
with tarfile.open(archive, "r:gz") as tar:
logger.info("Extracting the tarball content in %s" % target_dir)
size = len(list(tar))
with progress.Bar(expected_size=size) as bar:
def _extract(self, *args, **kw):
if not TASK_CLUSTER:
bar.show(bar.last_progress + 1)
return self.old(*args, **kw)
tar.old = tar.extract
tar.extract = functools.partial(_extract, tar)
tar.extractall(target_dir)
except (OSError, tarfile.ReadError) as e:
logger.info("Failed to extract the tarball")
if download_cache and os.path.exists(archive):
logger.info("Removing cached file to attempt a new download")
os.remove(archive)
raise ProfileNotFoundError(str(e))
finally:
if not download_cache:
shutil.rmtree(download_dir)
_check_profile(target_dir)
logger.info("Success, we have a profile to work with")
return target_dir
def onerror(error):
logger.info("Failed to get the profile.")
if os.path.exists(downloaded_archive):
try:
os.remove(downloaded_archive)
except Exception:
logger.error("Could not remove the file")
try:
return _retries(_get_profile, onerror, retries)
except RetriesError:
# look for older profile 2 days previously
filename = ARTIFACT_NAME % params
url = TC_LINK_BY_DATE % params + filename
try:
return _retries(_get_profile, onerror, retries)
except RetriesError:
raise ProfileNotFoundError(url)
def read_changelog(platform, repo="mozilla-central", scenario="settled"):
params = {"platform": platform, "repo": repo, "scenario": scenario}
changelog_url = CHANGELOG_LINK % params
logger.info("Getting %s" % changelog_url)
download_dir = tempfile.mkdtemp()
downloaded_changelog = os.path.join(download_dir, "changelog.json")
def _get_changelog():
try:
download_file(changelog_url, target=downloaded_changelog)
except ArchiveNotFound:
shutil.rmtree(download_dir)
raise ProfileNotFoundError(changelog_url)
return Changelog(download_dir)
try:
return _retries(_get_changelog)
except Exception:
raise ProfileNotFoundError(changelog_url)
|
