# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # # This module needs to stay Python 2 and 3 compatible # import datetime import functools import os import shutil import tarfile import tempfile import time from mozprofile.prefs import Preferences from condprof import progress from condprof.changelog import Changelog from condprof.util import ( TASK_CLUSTER, ArchiveNotFound, check_exists, download_file, logger, ) TC_SERVICE = "https://firefox-ci-tc.services.mozilla.com" ROOT_URL = TC_SERVICE + "/api/index" INDEX_PATH = "gecko.v2.%(repo)s.latest.firefox.condprof-%(platform)s-%(scenario)s" INDEX_BY_DATE_PATH = "gecko.v2.%(repo)s.pushdate.%(date)s.latest.firefox.condprof-%(platform)s-%(scenario)s" PUBLIC_DIR = "artifacts/public/condprof" TC_LINK = ROOT_URL + "/v1/task/" + INDEX_PATH + "/" + PUBLIC_DIR + "/" TC_LINK_BY_DATE = ROOT_URL + "/v1/task/" + INDEX_BY_DATE_PATH + "/" + PUBLIC_DIR + "/" ARTIFACT_NAME = "profile%(version)s-%(platform)s-%(scenario)s-%(customization)s.tgz" CHANGELOG_LINK = ( ROOT_URL + "/v1/task/" + INDEX_PATH + "/" + PUBLIC_DIR + "/changelog.json" ) ARTIFACTS_SERVICE = "https://taskcluster-artifacts.net" DIRECT_LINK = ARTIFACTS_SERVICE + "/%(task_id)s/0/public/condprof/" CONDPROF_CACHE = "~/.condprof-cache" RETRIES = 3 RETRY_PAUSE = 45 class ServiceUnreachableError(Exception): pass class ProfileNotFoundError(Exception): pass class RetriesError(Exception): pass def _check_service(url): """Sanity check to see if we can reach the service root url.""" def _check(): exists, _ = check_exists(url, all_types=True) if not exists: raise ServiceUnreachableError(url) try: return _retries(_check) except RetriesError: raise ServiceUnreachableError(url) def _check_profile(profile_dir): """Checks for prefs we need to remove or set.""" to_remove = ("gfx.blacklist.", "marionette.") def _keep_pref(name, value): for item in to_remove: if not name.startswith(item): continue logger.info("Removing pref %s: %s" % (name, value)) return False return True def _clean_pref_file(name): js_file = os.path.join(profile_dir, name) prefs = Preferences.read_prefs(js_file) cleaned_prefs = dict([pref for pref in prefs if _keep_pref(*pref)]) if name == "prefs.js": # When we start Firefox, forces startupScanScopes to SCOPE_PROFILE (1) # otherwise, side loading will be deactivated and the # Raptor web extension won't be able to run. cleaned_prefs["extensions.startupScanScopes"] = 1 # adding a marker so we know it's a conditioned profile cleaned_prefs["profile.conditioned"] = True with open(js_file, "w") as f: Preferences.write(f, cleaned_prefs) _clean_pref_file("prefs.js") _clean_pref_file("user.js") def _retries(callable, onerror=None, retries=RETRIES): _retry_count = 0 pause = RETRY_PAUSE while _retry_count < retries: try: return callable() except Exception as e: if onerror is not None: onerror(e) logger.info("Failed, retrying") _retry_count += 1 time.sleep(pause) pause *= 1.5 # If we reach that point, it means all attempts failed if _retry_count >= RETRIES: logger.error("All attempt failed") else: logger.info("Retried %s attempts and failed" % _retry_count) raise RetriesError() def get_profile( target_dir, platform, scenario, customization="default", task_id=None, download_cache=True, repo="mozilla-central", remote_test_root="/sdcard/test_root/", version=None, retries=RETRIES, ): """Extract a conditioned profile in the target directory. If task_id is provided, will grab the profile from that task. when not provided (default) will grab the latest profile. """ # XXX assert values if version: version = "-v%s" % version else: version = "" # when we bump the Firefox version on trunk, autoland still needs to catch up # in this case we want to download an older profile- 2 days to account for closures/etc. oldday = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=2) params = { "platform": platform, "scenario": scenario, "customization": customization, "task_id": task_id, "repo": repo, "version": version, "date": str(oldday.date()).replace("-", "."), } logger.info("Getting conditioned profile with arguments: %s" % params) filename = ARTIFACT_NAME % params if task_id is None: url = TC_LINK % params + filename _check_service(TC_SERVICE) else: url = DIRECT_LINK % params + filename _check_service(ARTIFACTS_SERVICE) logger.info("preparing download dir") if not download_cache: download_dir = tempfile.mkdtemp() else: # using a cache dir in the user home dir download_dir = os.path.expanduser(CONDPROF_CACHE) if not os.path.exists(download_dir): os.makedirs(download_dir) downloaded_archive = os.path.join(download_dir, filename) logger.info("Downloaded archive path: %s" % downloaded_archive) def _get_profile(): logger.info("Getting %s" % url) try: archive = download_file(url, target=downloaded_archive) except ArchiveNotFound: raise ProfileNotFoundError(url) try: with tarfile.open(archive, "r:gz") as tar: logger.info("Extracting the tarball content in %s" % target_dir) size = len(list(tar)) with progress.Bar(expected_size=size) as bar: def _extract(self, *args, **kw): if not TASK_CLUSTER: bar.show(bar.last_progress + 1) return self.old(*args, **kw) tar.old = tar.extract tar.extract = functools.partial(_extract, tar) tar.extractall(target_dir) except (OSError, tarfile.ReadError) as e: logger.info("Failed to extract the tarball") if download_cache and os.path.exists(archive): logger.info("Removing cached file to attempt a new download") os.remove(archive) raise ProfileNotFoundError(str(e)) finally: if not download_cache: shutil.rmtree(download_dir) _check_profile(target_dir) logger.info("Success, we have a profile to work with") return target_dir def onerror(error): logger.info("Failed to get the profile.") if os.path.exists(downloaded_archive): try: os.remove(downloaded_archive) except Exception: logger.error("Could not remove the file") try: return _retries(_get_profile, onerror, retries) except RetriesError: # look for older profile 2 days previously filename = ARTIFACT_NAME % params url = TC_LINK_BY_DATE % params + filename try: return _retries(_get_profile, onerror, retries) except RetriesError: raise ProfileNotFoundError(url) def read_changelog(platform, repo="mozilla-central", scenario="settled"): params = {"platform": platform, "repo": repo, "scenario": scenario} changelog_url = CHANGELOG_LINK % params logger.info("Getting %s" % changelog_url) download_dir = tempfile.mkdtemp() downloaded_changelog = os.path.join(download_dir, "changelog.json") def _get_changelog(): try: download_file(changelog_url, target=downloaded_changelog) except ArchiveNotFound: shutil.rmtree(download_dir) raise ProfileNotFoundError(changelog_url) return Changelog(download_dir) try: return _retries(_get_changelog) except Exception: raise ProfileNotFoundError(changelog_url)