1 files changed, 466 insertions, 0 deletions
diff --git a/testing/condprofile/condprof/util.py b/testing/condprofile/condprof/util.py
new file mode 100644
index 0000000000..6dfdc7d526
--- /dev/null
+++ b/testing/condprofile/condprof/util.py
@@ -0,0 +1,466 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This module needs to stay Python 2 and 3 compatible
+#
+from __future__ import absolute_import, division, print_function
+
+import platform
+import time
+import os
+import shutil
+import contextlib
+import yaml
+from subprocess import Popen, PIPE
+import sys
+import tempfile
+
+import requests
+from requests.exceptions import ConnectionError
+from requests.packages.urllib3.util.retry import Retry
+
+import mozlog
+
+from condprof import progress
+
+
+TASK_CLUSTER = "TASK_ID" in os.environ.keys()
+DOWNLOAD_TIMEOUT = 30
+
+
+class ArchiveNotFound(Exception):
+    pass
+
+
+DEFAULT_PREFS = {
+    "focusmanager.testmode": True,
+    "marionette.defaultPrefs.port": 2828,
+    "marionette.port": 2828,
+    "marionette.enabled": True,
+    "marionette.log.level": "Trace",
+    "marionette.log.truncate": False,
+    "marionette.contentListener": False,
+    "extensions.autoDisableScopes": 10,
+    "devtools.debugger.remote-enabled": True,
+    "devtools.console.stdout.content": True,
+    "devtools.console.stdout.chrome": True,
+}
+
+DEFAULT_CUSTOMIZATION = os.path.join(
+    os.path.dirname(__file__), "customization", "default.json"
+)
+STRUCTLOG_PAD_SIZE = 20
+
+
+class BridgeLogger:
+    def __init__(self, logger):
+        self.logger = logger
+
+    def _find(self, text, *names):
+        # structlog's ConsoleRenderer pads values
+        for name in names:
+            if name + " " * STRUCTLOG_PAD_SIZE in text:
+                return True
+        return False
+
+    def _convert(self, message):
+        return obfuscate(message)[1]
+
+    def info(self, message, *args, **kw):
+        if not isinstance(message, str):
+            message = str(message)
+        # converting Arsenic request/response struct log
+        if self._find(message, "request", "response"):
+            self.logger.debug(self._convert(message), *args, **kw)
+        else:
+            self.logger.info(self._convert(message), *args, **kw)
+
+    def error(self, message, *args, **kw):
+        self.logger.error(self._convert(message), *args, **kw)
+
+    def warning(self, message, *args, **kw):
+        self.logger.warning(self._convert(message), *args, **kw)
+
+
+logger = None
+
+
+def get_logger():
+    global logger
+    if logger is not None:
+        return logger
+    new_logger = mozlog.get_default_logger("condprof")
+    if new_logger is None:
+        new_logger = mozlog.unstructured.getLogger("condprof")
+
+    # wrap the logger into the BridgeLogger
+    new_logger = BridgeLogger(new_logger)
+
+    # bridge for Arsenic
+    if sys.version_info.major == 3:
+        try:
+            from arsenic import connection
+            from structlog import wrap_logger
+
+            connection.log = wrap_logger(new_logger)
+        except ImportError:
+            # Arsenic is not installed for client-only usage
+            pass
+    logger = new_logger
+    return logger
+
+
+# initializing the logger right away
+get_logger()
+
+
+def fresh_profile(profile, customization_data):
+    from mozprofile import create_profile  # NOQA
+
+    # XXX on android we mgiht need to run it on the device?
+    logger.info("Creating a fresh profile")
+    new_profile = create_profile(app="firefox")
+    prefs = customization_data["prefs"]
+    prefs.update(DEFAULT_PREFS)
+    logger.info("Setting prefs %s" % str(prefs.items()))
+    new_profile.set_preferences(prefs)
+    extensions = []
+    for name, url in customization_data["addons"].items():
+        logger.info("Downloading addon %s" % name)
+        extension = download_file(url)
+        extensions.append(extension)
+    logger.info("Installing addons")
+    new_profile.addons.install(extensions, unpack=True)
+    new_profile.addons.install(extensions)
+    shutil.copytree(new_profile.profile, profile)
+    return profile
+
+
+link = "https://ftp.mozilla.org/pub/firefox/nightly/latest-mozilla-central/"
+
+
+def get_firefox_download_link():
+    try:
+        from bs4 import BeautifulSoup
+    except ImportError:
+        raise ImportError("You need to run pip install beautifulsoup4")
+    if platform.system() == "Darwin":
+        extension = ".dmg"
+    elif platform.system() == "Linux":
+        arch = platform.machine()
+        extension = ".linux-%s.tar.bz2" % arch
+    else:
+        raise NotImplementedError(platform.system())
+
+    page = requests.get(link).text
+    soup = BeautifulSoup(page, "html.parser")
+    for node in soup.find_all("a", href=True):
+        href = node["href"]
+        if href.endswith(extension):
+            return "https://ftp.mozilla.org" + href
+
+    raise Exception()
+
+
+def check_exists(archive, server=None, all_types=False):
+    if server is not None:
+        archive = server + "/" + archive
+    try:
+        logger.info("Getting headers at %s" % archive)
+        resp = requests.head(archive, timeout=DOWNLOAD_TIMEOUT)
+    except ConnectionError:
+        return False, {}
+
+    if resp.status_code in (302, 303):
+        logger.info("Redirected")
+        return check_exists(resp.headers["Location"])
+
+    # see Bug 1574854
+    if (
+        not all_types
+        and resp.status_code == 200
+        and "text/html" in resp.headers["Content-Type"]
+    ):
+        logger.info("Got an html page back")
+        exists = False
+    else:
+        logger.info("Response code is %d" % resp.status_code)
+        exists = resp.status_code
+
+    return exists, resp.headers
+
+
+def download_file(url, target=None):
+    present, headers = check_exists(url)
+    if not present:
+        logger.info("Cannot find %r" % url)
+        raise ArchiveNotFound(url)
+
+    etag = headers.get("ETag")
+    if target is None:
+        target = url.split("/")[-1]
+
+    logger.info("Checking for existence of: %s" % target)
+    if os.path.exists(target):
+        # XXX for now, reusing downloads without checking them
+        # when we don't have an .etag file
+        if etag is None or not os.path.exists(target + ".etag"):
+            logger.info("No existing etag downloads.")
+            return target
+        with open(target + ".etag") as f:
+            current_etag = f.read()
+        if etag == current_etag:
+            logger.info("Already Downloaded.")
+            # should at least check the size?
+            return target
+        else:
+            logger.info("Changed!")
+    else:
+        logger.info("Could not find an existing archive.")
+        # Add some debugging logs for the directory content
+        try:
+            archivedir = os.path.dirname(target)
+            logger.info(
+                "Content in cache directory %s: %s"
+                % (archivedir, os.listdir(archivedir))
+            )
+        except Exception:
+            logger.info("Failed to list cache directory contents")
+
+    logger.info("Downloading %s" % url)
+    req = requests.get(url, stream=True, timeout=DOWNLOAD_TIMEOUT)
+    total_length = int(req.headers.get("content-length"))
+    target_dir = os.path.dirname(target)
+    if target_dir != "" and not os.path.exists(target_dir):
+        logger.info("Creating dir %s" % target_dir)
+        os.makedirs(target_dir)
+
+    with open(target, "wb") as f:
+        if TASK_CLUSTER:
+            for chunk in req.iter_content(chunk_size=1024):
+                if chunk:
+                    f.write(chunk)
+                    f.flush()
+        else:
+            iter = req.iter_content(chunk_size=1024)
+            # pylint --py3k W1619
+            size = total_length / 1024 + 1
+            for chunk in progress.bar(iter, expected_size=size):
+                if chunk:
+                    f.write(chunk)
+                    f.flush()
+
+    if etag is not None:
+        with open(target + ".etag", "w") as f:
+            f.write(etag)
+
+    return target
+
+
+def extract_from_dmg(dmg, target):
+    mount = tempfile.mkdtemp()
+    cmd = "hdiutil attach -nobrowse -mountpoint %s %s"
+    os.system(cmd % (mount, dmg))
+    try:
+        found = False
+        for f in os.listdir(mount):
+            if not f.endswith(".app"):
+                continue
+            app = os.path.join(mount, f)
+            shutil.copytree(app, target)
+            found = True
+            break
+    finally:
+        os.system("hdiutil detach " + mount)
+        shutil.rmtree(mount)
+    if not found:
+        raise IOError("No app file found in %s" % dmg)
+
+
+@contextlib.contextmanager
+def latest_nightly(binary=None):
+
+    if binary is None:
+        # we want to use the latest nightly
+        nightly_archive = get_firefox_download_link()
+        logger.info("Downloading %s" % nightly_archive)
+        target = download_file(nightly_archive)
+        # on macOs we just mount the DMG
+        # XXX replace with extract_from_dmg
+        if platform.system() == "Darwin":
+            cmd = "hdiutil attach -mountpoint /Volumes/Nightly %s"
+            os.system(cmd % target)
+            binary = "/Volumes/Nightly/Firefox Nightly.app/Contents/MacOS/firefox"
+        # on linux we unpack it
+        elif platform.system() == "Linux":
+            cmd = "bunzip2 %s" % target
+            os.system(cmd)
+            cmd = "tar -xvf %s" % target[: -len(".bz2")]
+            os.system(cmd)
+            binary = "firefox/firefox"
+
+        mounted = True
+    else:
+        mounted = False
+    try:
+        yield binary
+    finally:
+        # XXX replace with extract_from_dmg
+        if mounted:
+            if platform.system() == "Darwin":
+                logger.info("Unmounting Firefox")
+                time.sleep(10)
+                os.system("hdiutil detach /Volumes/Nightly")
+            elif platform.system() == "Linux":
+                # XXX we should keep it for next time
+                shutil.rmtree("firefox")
+
+
+def write_yml_file(yml_file, yml_data):
+    logger.info("writing %s to %s" % (yml_data, yml_file))
+    try:
+        with open(yml_file, "w") as outfile:
+            yaml.dump(yml_data, outfile, default_flow_style=False)
+    except Exception:
+        logger.error("failed to write yaml file", exc_info=True)
+
+
+def get_version(firefox):
+    p = Popen([firefox, "--version"], stdin=PIPE, stdout=PIPE, stderr=PIPE)
+    output, __ = p.communicate()
+    res = output.strip().split()[-1]
+    return res.decode("utf-8")
+
+
+def get_current_platform():
+    """Returns a combination of system and arch info that matches TC standards.
+
+    e.g. macosx64, win32, linux64, etc..
+    """
+    arch = sys.maxsize == 2 ** 63 - 1 and "64" or "32"
+    plat = platform.system().lower()
+    if plat == "windows":
+        plat = "win"
+    elif plat == "darwin":
+        plat = "macosx"
+    return plat + arch
+
+
+class BaseEnv:
+    def __init__(self, profile, firefox, geckodriver, archive, device_name):
+        self.profile = profile
+        self.firefox = firefox
+        self.geckodriver = geckodriver
+        if profile is None:
+            self.profile = os.path.join(tempfile.mkdtemp(), "profile")
+        else:
+            self.profile = profile
+        self.archive = archive
+        self.device_name = device_name
+
+    @property
+    def target_platform(self):
+        return self.get_target_platform()
+
+    def get_target_platform(self):
+        raise NotImplementedError()
+
+    def get_device(self, *args, **kw):
+        raise NotImplementedError()
+
+    @contextlib.contextmanager
+    def get_browser(self, path):
+        raise NotImplementedError()
+
+    def get_browser_args(self, headless):
+        raise NotImplementedError()
+
+    def prepare(self, logfile):
+        pass
+
+    def check_session(self, session):
+        pass
+
+    def dump_logs(self):
+        pass
+
+    def get_browser_version(self):
+        raise NotImplementedError()
+
+    def get_geckodriver(self, log_file):
+        raise NotImplementedError()
+
+    def collect_profile(self):
+        pass
+
+    def stop_browser(self):
+        pass
+
+
+_URL = (
+    "{0}/secrets/v1/secret/project"
+    "{1}releng{1}gecko{1}build{1}level-{2}{1}conditioned-profiles"
+)
+_DEFAULT_SERVER = "https://firefox-ci-tc.services.mozilla.com"
+
+
+def get_tc_secret():
+    if not TASK_CLUSTER:
+        raise OSError("Not running in Taskcluster")
+    session = requests.Session()
+    retry = Retry(total=5, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504])
+    http_adapter = requests.adapters.HTTPAdapter(max_retries=retry)
+    session.mount("https://", http_adapter)
+    session.mount("http://", http_adapter)
+    secrets_url = _URL.format(
+        os.environ.get("TASKCLUSTER_PROXY_URL", _DEFAULT_SERVER),
+        "%2F",
+        os.environ.get("MOZ_SCM_LEVEL", "1"),
+    )
+    res = session.get(secrets_url, timeout=DOWNLOAD_TIMEOUT)
+    res.raise_for_status()
+    return res.json()["secret"]
+
+
+_CACHED = {}
+
+
+def obfuscate(text):
+    if "CONDPROF_RUNNER" not in os.environ:
+        return True, text
+    username, password = get_credentials()
+    if username is None:
+        return False, text
+    if username not in text and password not in text:
+        return False, text
+    text = text.replace(password, "<PASSWORD>")
+    text = text.replace(username, "<USERNAME>")
+    return True, text
+
+
+def obfuscate_file(path):
+    if "CONDPROF_RUNNER" not in os.environ:
+        return
+    with open(path) as f:
+        data = f.read()
+    hit, data = obfuscate(data)
+    if not hit:
+        return
+    with open(path, "w") as f:
+        f.write(data)
+
+
+def get_credentials():
+    if "creds" in _CACHED:
+        return _CACHED["creds"]
+    password = os.environ.get("FXA_PASSWORD")
+    username = os.environ.get("FXA_USERNAME")
+    if username is None or password is None:
+        if not TASK_CLUSTER:
+            return None, None
+        secret = get_tc_secret()
+        password = secret["password"]
+        username = secret["username"]
+    _CACHED["creds"] = username, password
+    return username, password