Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 255 insertions, 0 deletions

diff --git a/testing/condprofile/condprof/client.py b/testing/condprofile/condprof/client.py
new file mode 100644
index 0000000000..186e8cf4c4
--- /dev/null
+++ b/testing/condprofile/condprof/client.py
@@ -0,0 +1,255 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This module needs to stay Python 2 and 3 compatible
+#
+import datetime
+import functools
+import os
+import shutil
+import tarfile
+import tempfile
+import time
+
+from mozprofile.prefs import Preferences
+
+from condprof import progress
+from condprof.changelog import Changelog
+from condprof.util import (
+    TASK_CLUSTER,
+    ArchiveNotFound,
+    check_exists,
+    download_file,
+    logger,
+)
+
+TC_SERVICE = "https://firefox-ci-tc.services.mozilla.com"
+ROOT_URL = TC_SERVICE + "/api/index"
+INDEX_PATH = "gecko.v2.%(repo)s.latest.firefox.condprof-%(platform)s-%(scenario)s"
+INDEX_BY_DATE_PATH = "gecko.v2.%(repo)s.pushdate.%(date)s.latest.firefox.condprof-%(platform)s-%(scenario)s"
+PUBLIC_DIR = "artifacts/public/condprof"
+TC_LINK = ROOT_URL + "/v1/task/" + INDEX_PATH + "/" + PUBLIC_DIR + "/"
+TC_LINK_BY_DATE = ROOT_URL + "/v1/task/" + INDEX_BY_DATE_PATH + "/" + PUBLIC_DIR + "/"
+ARTIFACT_NAME = "profile%(version)s-%(platform)s-%(scenario)s-%(customization)s.tgz"
+CHANGELOG_LINK = (
+    ROOT_URL + "/v1/task/" + INDEX_PATH + "/" + PUBLIC_DIR + "/changelog.json"
+)
+ARTIFACTS_SERVICE = "https://taskcluster-artifacts.net"
+DIRECT_LINK = ARTIFACTS_SERVICE + "/%(task_id)s/0/public/condprof/"
+CONDPROF_CACHE = "~/.condprof-cache"
+RETRIES = 3
+RETRY_PAUSE = 45
+
+
+class ServiceUnreachableError(Exception):
+    pass
+
+
+class ProfileNotFoundError(Exception):
+    pass
+
+
+class RetriesError(Exception):
+    pass
+
+
+def _check_service(url):
+    """Sanity check to see if we can reach the service root url."""
+
+    def _check():
+        exists, _ = check_exists(url, all_types=True)
+        if not exists:
+            raise ServiceUnreachableError(url)
+
+    try:
+        return _retries(_check)
+    except RetriesError:
+        raise ServiceUnreachableError(url)
+
+
+def _check_profile(profile_dir):
+    """Checks for prefs we need to remove or set."""
+    to_remove = ("gfx.blacklist.", "marionette.")
+
+    def _keep_pref(name, value):
+        for item in to_remove:
+            if not name.startswith(item):
+                continue
+            logger.info("Removing pref %s: %s" % (name, value))
+            return False
+        return True
+
+    def _clean_pref_file(name):
+        js_file = os.path.join(profile_dir, name)
+        prefs = Preferences.read_prefs(js_file)
+        cleaned_prefs = dict([pref for pref in prefs if _keep_pref(*pref)])
+        if name == "prefs.js":
+            # When we start Firefox, forces startupScanScopes to SCOPE_PROFILE (1)
+            # otherwise, side loading will be deactivated and the
+            # Raptor web extension won't be able to run.
+            cleaned_prefs["extensions.startupScanScopes"] = 1
+
+            # adding a marker so we know it's a conditioned profile
+            cleaned_prefs["profile.conditioned"] = True
+
+        with open(js_file, "w") as f:
+            Preferences.write(f, cleaned_prefs)
+
+    _clean_pref_file("prefs.js")
+    _clean_pref_file("user.js")
+
+
+def _retries(callable, onerror=None, retries=RETRIES):
+    _retry_count = 0
+    pause = RETRY_PAUSE
+
+    while _retry_count < retries:
+        try:
+            return callable()
+        except Exception as e:
+            if onerror is not None:
+                onerror(e)
+            logger.info("Failed, retrying")
+            _retry_count += 1
+            time.sleep(pause)
+            pause *= 1.5
+
+    # If we reach that point, it means all attempts failed
+    if _retry_count >= RETRIES:
+        logger.error("All attempt failed")
+    else:
+        logger.info("Retried %s attempts and failed" % _retry_count)
+    raise RetriesError()
+
+
+def get_profile(
+    target_dir,
+    platform,
+    scenario,
+    customization="default",
+    task_id=None,
+    download_cache=True,
+    repo="mozilla-central",
+    remote_test_root="/sdcard/test_root/",
+    version=None,
+    retries=RETRIES,
+):
+    """Extract a conditioned profile in the target directory.
+
+    If task_id is provided, will grab the profile from that task. when not
+    provided (default) will grab the latest profile.
+    """
+
+    # XXX assert values
+    if version:
+        version = "-v%s" % version
+    else:
+        version = ""
+
+    # when we bump the Firefox version on trunk, autoland still needs to catch up
+    # in this case we want to download an older profile- 2 days to account for closures/etc.
+    oldday = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=2)
+    params = {
+        "platform": platform,
+        "scenario": scenario,
+        "customization": customization,
+        "task_id": task_id,
+        "repo": repo,
+        "version": version,
+        "date": str(oldday.date()).replace("-", "."),
+    }
+    logger.info("Getting conditioned profile with arguments: %s" % params)
+    filename = ARTIFACT_NAME % params
+    if task_id is None:
+        url = TC_LINK % params + filename
+        _check_service(TC_SERVICE)
+    else:
+        url = DIRECT_LINK % params + filename
+        _check_service(ARTIFACTS_SERVICE)
+
+    logger.info("preparing download dir")
+    if not download_cache:
+        download_dir = tempfile.mkdtemp()
+    else:
+        # using a cache dir in the user home dir
+        download_dir = os.path.expanduser(CONDPROF_CACHE)
+        if not os.path.exists(download_dir):
+            os.makedirs(download_dir)
+
+    downloaded_archive = os.path.join(download_dir, filename)
+    logger.info("Downloaded archive path: %s" % downloaded_archive)
+
+    def _get_profile():
+        logger.info("Getting %s" % url)
+        try:
+            archive = download_file(url, target=downloaded_archive)
+        except ArchiveNotFound:
+            raise ProfileNotFoundError(url)
+        try:
+            with tarfile.open(archive, "r:gz") as tar:
+                logger.info("Extracting the tarball content in %s" % target_dir)
+                size = len(list(tar))
+                with progress.Bar(expected_size=size) as bar:
+
+                    def _extract(self, *args, **kw):
+                        if not TASK_CLUSTER:
+                            bar.show(bar.last_progress + 1)
+                        return self.old(*args, **kw)
+
+                    tar.old = tar.extract
+                    tar.extract = functools.partial(_extract, tar)
+                    tar.extractall(target_dir)
+        except (OSError, tarfile.ReadError) as e:
+            logger.info("Failed to extract the tarball")
+            if download_cache and os.path.exists(archive):
+                logger.info("Removing cached file to attempt a new download")
+                os.remove(archive)
+            raise ProfileNotFoundError(str(e))
+        finally:
+            if not download_cache:
+                shutil.rmtree(download_dir)
+
+        _check_profile(target_dir)
+        logger.info("Success, we have a profile to work with")
+        return target_dir
+
+    def onerror(error):
+        logger.info("Failed to get the profile.")
+        if os.path.exists(downloaded_archive):
+            try:
+                os.remove(downloaded_archive)
+            except Exception:
+                logger.error("Could not remove the file")
+
+    try:
+        return _retries(_get_profile, onerror, retries)
+    except RetriesError:
+        # look for older profile 2 days previously
+        filename = ARTIFACT_NAME % params
+        url = TC_LINK_BY_DATE % params + filename
+        try:
+            return _retries(_get_profile, onerror, retries)
+        except RetriesError:
+            raise ProfileNotFoundError(url)
+
+
+def read_changelog(platform, repo="mozilla-central", scenario="settled"):
+    params = {"platform": platform, "repo": repo, "scenario": scenario}
+    changelog_url = CHANGELOG_LINK % params
+    logger.info("Getting %s" % changelog_url)
+    download_dir = tempfile.mkdtemp()
+    downloaded_changelog = os.path.join(download_dir, "changelog.json")
+
+    def _get_changelog():
+        try:
+            download_file(changelog_url, target=downloaded_changelog)
+        except ArchiveNotFound:
+            shutil.rmtree(download_dir)
+            raise ProfileNotFoundError(changelog_url)
+        return Changelog(download_dir)
+
+    try:
+        return _retries(_get_changelog)
+    except Exception:
+        raise ProfileNotFoundError(changelog_url)