summaryrefslogtreecommitdiffstats
path: root/testing/condprofile
diff options
context:
space:
mode:
Diffstat (limited to 'testing/condprofile')
-rw-r--r--testing/condprofile/Makefile23
-rw-r--r--testing/condprofile/README.rst114
-rw-r--r--testing/condprofile/condprof/__init__.py3
-rw-r--r--testing/condprofile/condprof/android.py294
-rw-r--r--testing/condprofile/condprof/archiver.py76
-rw-r--r--testing/condprofile/condprof/changelog.py56
-rw-r--r--testing/condprofile/condprof/check_install.py71
-rw-r--r--testing/condprofile/condprof/client.py255
-rw-r--r--testing/condprofile/condprof/creator.py226
-rw-r--r--testing/condprofile/condprof/customization/__init__.py35
-rw-r--r--testing/condprofile/condprof/customization/default.json10
-rw-r--r--testing/condprofile/condprof/customization/webext.json15
-rw-r--r--testing/condprofile/condprof/customization/youtube.json12
-rw-r--r--testing/condprofile/condprof/desktop.py90
-rw-r--r--testing/condprofile/condprof/helpers.py110
-rw-r--r--testing/condprofile/condprof/main.py86
-rw-r--r--testing/condprofile/condprof/metadata.py83
-rw-r--r--testing/condprofile/condprof/patch.py47
-rw-r--r--testing/condprofile/condprof/progress.py223
-rw-r--r--testing/condprofile/condprof/runner.py211
-rw-r--r--testing/condprofile/condprof/scenarii/__init__.py15
-rw-r--r--testing/condprofile/condprof/scenarii/bookmark.js52
-rw-r--r--testing/condprofile/condprof/scenarii/full.py138
-rw-r--r--testing/condprofile/condprof/scenarii/settled.py11
-rw-r--r--testing/condprofile/condprof/scenarii/settled2.py19
-rw-r--r--testing/condprofile/condprof/scenarii/settled_youtube.py20
-rw-r--r--testing/condprofile/condprof/scenarii/sync.js21
-rw-r--r--testing/condprofile/condprof/scenarii/urls.txt7
-rw-r--r--testing/condprofile/condprof/scenarii/words.txt4555
-rw-r--r--testing/condprofile/condprof/tests/__init__.py0
-rwxr-xr-xtesting/condprofile/condprof/tests/fakefirefox.py7
-rwxr-xr-xtesting/condprofile/condprof/tests/fakegeckodriver.py144
-rw-r--r--testing/condprofile/condprof/tests/ftp_mozilla.html1484
-rw-r--r--testing/condprofile/condprof/tests/profile/prefs.js1
-rw-r--r--testing/condprofile/condprof/tests/profile/user.js9
-rw-r--r--testing/condprofile/condprof/tests/python.toml4
-rw-r--r--testing/condprofile/condprof/tests/test_client.py126
-rw-r--r--testing/condprofile/condprof/tests/test_runner.py123
-rw-r--r--testing/condprofile/condprof/util.py484
-rw-r--r--testing/condprofile/mach_commands.py121
-rw-r--r--testing/condprofile/moz.build7
-rw-r--r--testing/condprofile/requirements/base.txt5
-rw-r--r--testing/condprofile/requirements/ci-client.txt10
-rw-r--r--testing/condprofile/requirements/ci.txt7
-rw-r--r--testing/condprofile/requirements/local-client.txt6
-rw-r--r--testing/condprofile/requirements/local.txt5
-rw-r--r--testing/condprofile/requirements/tox.txt8
-rw-r--r--testing/condprofile/setup.py32
-rw-r--r--testing/condprofile/tox.ini16
49 files changed, 9477 insertions, 0 deletions
diff --git a/testing/condprofile/Makefile b/testing/condprofile/Makefile
new file mode 100644
index 0000000000..b626f5c7e9
--- /dev/null
+++ b/testing/condprofile/Makefile
@@ -0,0 +1,23 @@
+HERE = $(shell pwd)
+BIN = $(HERE)/bin
+PYTHON = $(BIN)/python
+INSTALL = $(BIN)/pip install --no-deps
+BUILD_DIRS = bin build include lib lib64 man share
+VIRTUALENV = virtualenv
+
+.PHONY: all test build docs
+
+all: build
+
+$(PYTHON):
+ $(VIRTUALENV) $(VTENV_OPTS) .
+
+build: $(PYTHON)
+ $(PYTHON) setup.py develop
+ $(BIN)/pip install tox
+
+test: build
+ $(BIN)/tox
+
+docs: build
+ $(BIN)/tox -e docs
diff --git a/testing/condprofile/README.rst b/testing/condprofile/README.rst
new file mode 100644
index 0000000000..01b7fb73be
--- /dev/null
+++ b/testing/condprofile/README.rst
@@ -0,0 +1,114 @@
+Conditioned Profile
+===================
+
+This project provides a command-line tool that is used to generate and maintain
+a collection of Gecko profiles.
+
+Unlike testing/profiles, the **conditioned profiles** are a collection of full
+Gecko profiles that are dynamically updated every day.
+
+Each profile is created or updated using a **scenario** and a
+**customization**, and eventually uploaded as an artifact in TaskCluster.
+
+The goal of the project is to build a collection of profiles that we can use in
+our performance or functional tests instead of the empty profile that we
+usually create on the fly with **mozprofile**.
+
+Having a collection of realistic profiles we can use when running some tests
+gives us the ability to check the impact of user profiles on page loads or
+other tests.
+
+A full cycle of how this tool is used in Taskcluster looks like this:
+
+For each combination of scenario, customization and platform:
+
+- grabs an existing profile in Taskcluster
+- browses the web using the scenario, via the WebDriver client
+- recreates a tarball with the updated profile
+- uploads it as an index artifact into TaskCluster - maintains a changelog of each change
+
+It's based on the Arsenic webdriver client https://github.com/HDE/arsenic
+
+The project provides two **Mach** commands to interact with the conditioned
+profile:
+
+- **fetch-condprofile**: downloads a conditioned profile and deecompress it
+- **run-condprofile**: runs on or all conditioned profiles scenarii locally
+
+How to download a conditioned profile
+=====================================
+
+From your mozilla-central root, run:
+
+::
+
+ $ ./mach fetch-condprofile
+
+This will grab the latest conditioned profile for your platform. But
+you can also grab a specific profile built from any scenario or platform.
+
+You can look at all the options with --help
+
+How to run a conditioned profile
+================================
+
+If you want to play a scenario locally to modify it, run for example:
+
+::
+
+ $ ./mach run-condprofile --scenario settled --visible /path/to/generated/profile
+
+The project will run a webdriver session against Firefox and generate the profile.
+You can look at all the options with --help
+
+Architecture
+============
+
+The conditioned profile project is organized into webdriver **scenarii** and
+**customization** files.
+
+Scenarii
+--------
+
+Scenarii are coroutines registered under a unique name in condprof/scenarii/__init__.py.
+
+They get a **session** object and some **options**.
+
+The scenario can do whatever it wants with the browser, through the webdriver session
+instance.
+
+See Arsenic's `API documentation <https://arsenic.readthedocs.io/en/latest/reference/session.html>`_ for the session class.
+
+Adding a new scenario is done by adding a module in condprof/scenarii/
+and register it in condprof/scenarii/__init__.py
+
+
+Customization
+-------------
+
+A customization is a configuration file that can be used to set some
+prefs in the browser and install some webextensions.
+
+Customizations are JSON files registered into condprof/customizations,
+and they provide four keys:
+
+- **name**: the name of the customization
+- **addons**: a mapping of add-ons to install.
+- **prefs**: a mapping of prefs to set
+- **scenario**: a mapping of options to pass to a specific scenario
+
+In the example below, we install uBlock, set a pref, and pass the
+**max_urls** option to the **heavy** scenario.
+
+ {
+ "name": "intermediate",
+ "addons":{
+ "uBlock":"https://addons.mozilla.org/firefox/downloads/file/3361355/ublock_origin-1.21.2-an+fx.xpi"
+ },
+ "prefs":{
+ "accessibility.tabfocus": 9
+ },
+ "scenario": {
+ "heavy": {"max_urls": 10}
+ }
+ }
diff --git a/testing/condprofile/condprof/__init__.py b/testing/condprofile/condprof/__init__.py
new file mode 100644
index 0000000000..6fbe8159b2
--- /dev/null
+++ b/testing/condprofile/condprof/__init__.py
@@ -0,0 +1,3 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
diff --git a/testing/condprofile/condprof/android.py b/testing/condprofile/condprof/android.py
new file mode 100644
index 0000000000..b56d0b906c
--- /dev/null
+++ b/testing/condprofile/condprof/android.py
@@ -0,0 +1,294 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+""" Drives an android device.
+"""
+import os
+import posixpath
+import tempfile
+import contextlib
+import time
+import logging
+
+import attr
+from arsenic.services import Geckodriver, free_port, subprocess_based_service
+from mozdevice import ADBDeviceFactory, ADBError
+
+from condprof.util import write_yml_file, logger, DEFAULT_PREFS, BaseEnv
+
+
+# XXX most of this code should migrate into mozdevice - see Bug 1574849
+class AndroidDevice:
+ def __init__(
+ self,
+ app_name,
+ marionette_port=2828,
+ verbose=False,
+ remote_test_root="/sdcard/test_root/",
+ ):
+ self.app_name = app_name
+
+ # XXX make that an option
+ if "fenix" in app_name:
+ self.activity = "org.mozilla.fenix.IntentReceiverActivity"
+ else:
+ self.activity = "org.mozilla.geckoview_example.GeckoViewActivity"
+ self.verbose = verbose
+ self.device = None
+ self.marionette_port = marionette_port
+ self.profile = None
+ self.remote_profile = None
+ self.log_file = None
+ self.remote_test_root = remote_test_root
+ self._adb_fh = None
+
+ def _set_adb_logger(self, log_file):
+ self.log_file = log_file
+ if self.log_file is None:
+ return
+ logger.info("Setting ADB log file to %s" % self.log_file)
+ adb_logger = logging.getLogger("adb")
+ adb_logger.setLevel(logging.DEBUG)
+ self._adb_fh = logging.FileHandler(self.log_file)
+ self._adb_fh.setLevel(logging.DEBUG)
+ adb_logger.addHandler(self._adb_fh)
+
+ def _unset_adb_logger(self):
+ if self._adb_fh is None:
+ return
+ logging.getLogger("adb").removeHandler(self._adb_fh)
+ self._adb_fh = None
+
+ def clear_logcat(self, timeout=None, buffers=[]):
+ if not self.device:
+ return
+ self.device.clear_logcat(timeout, buffers)
+
+ def get_logcat(self):
+ if not self.device:
+ return None
+ # we don't want to have ADBCommand dump the command
+ # in the debug stream so we reduce its verbosity here
+ # temporarely
+ old_verbose = self.device._verbose
+ self.device._verbose = False
+ try:
+ return self.device.get_logcat()
+ finally:
+ self.device._verbose = old_verbose
+
+ def prepare(self, profile, logfile):
+ self._set_adb_logger(logfile)
+ try:
+ # See android_emulator_pgo.py run_tests for more
+ # details on why test_root must be /sdcard/test_root
+ # for android pgo due to Android 4.3.
+ self.device = ADBDeviceFactory(
+ verbose=self.verbose, logger_name="adb", test_root=self.remote_test_root
+ )
+ except Exception:
+ logger.error("Cannot initialize device")
+ raise
+ device = self.device
+ self.profile = profile
+
+ # checking that the app is installed
+ if not device.is_app_installed(self.app_name):
+ raise Exception("%s is not installed" % self.app_name)
+
+ # debug flag
+ logger.info("Setting %s as the debug app on the phone" % self.app_name)
+ device.shell(
+ "am set-debug-app --persistent %s" % self.app_name,
+ stdout_callback=logger.info,
+ )
+
+ # creating the profile on the device
+ logger.info("Creating the profile on the device")
+
+ remote_profile = posixpath.join(self.remote_test_root, "profile")
+ logger.info("The profile on the phone will be at %s" % remote_profile)
+
+ device.rm(remote_profile, force=True, recursive=True)
+ logger.info("Pushing %s on the phone" % self.profile)
+ device.push(profile, remote_profile)
+ device.chmod(remote_profile, recursive=True)
+ self.profile = profile
+ self.remote_profile = remote_profile
+
+ # creating the yml file
+ yml_data = {
+ "args": ["-marionette", "-profile", self.remote_profile],
+ "prefs": DEFAULT_PREFS,
+ "env": {"LOG_VERBOSE": 1, "R_LOG_LEVEL": 6, "MOZ_LOG": ""},
+ }
+
+ yml_name = "%s-geckoview-config.yaml" % self.app_name
+ yml_on_host = posixpath.join(tempfile.mkdtemp(), yml_name)
+ write_yml_file(yml_on_host, yml_data)
+ tmp_on_device = posixpath.join("/data", "local", "tmp")
+ if not device.exists(tmp_on_device):
+ raise IOError("%s does not exists on the device" % tmp_on_device)
+ yml_on_device = posixpath.join(tmp_on_device, yml_name)
+ try:
+ device.rm(yml_on_device, force=True, recursive=True)
+ device.push(yml_on_host, yml_on_device)
+ device.chmod(yml_on_device, recursive=True)
+ except Exception:
+ logger.info("could not create the yaml file on device. Permission issue?")
+ raise
+
+ # command line 'extra' args not used with geckoview apps; instead we use
+ # an on-device config.yml file
+ intent = "android.intent.action.VIEW"
+ device.stop_application(self.app_name)
+ device.launch_application(
+ self.app_name, self.activity, intent, extras=None, url="about:blank"
+ )
+ if not device.process_exist(self.app_name):
+ raise Exception("Could not start %s" % self.app_name)
+
+ logger.info("Creating socket forwarding on port %d" % self.marionette_port)
+ device.forward(
+ local="tcp:%d" % self.marionette_port,
+ remote="tcp:%d" % self.marionette_port,
+ )
+
+ # we don't have a clean way for now to check that GV or Fenix
+ # is ready to handle our tests. So here we just wait 30s
+ logger.info("Sleeping for 30s")
+ time.sleep(30)
+
+ def stop_browser(self):
+ logger.info("Stopping %s" % self.app_name)
+ try:
+ self.device.stop_application(self.app_name)
+ except ADBError:
+ logger.info("Could not stop the application using force-stop")
+
+ time.sleep(5)
+ if self.device.process_exist(self.app_name):
+ logger.info("%s still running, trying SIGKILL" % self.app_name)
+ num_tries = 0
+ while self.device.process_exist(self.app_name) and num_tries < 5:
+ try:
+ self.device.pkill(self.app_name)
+ except ADBError:
+ pass
+ num_tries += 1
+ time.sleep(1)
+ logger.info("%s stopped" % self.app_name)
+
+ def collect_profile(self):
+ logger.info("Collecting profile from %s" % self.remote_profile)
+ self.device.pull(self.remote_profile, self.profile)
+
+ def close(self):
+ self._unset_adb_logger()
+ if self.device is None:
+ return
+ try:
+ self.device.remove_forwards("tcp:%d" % self.marionette_port)
+ except ADBError:
+ logger.warning("Could not remove forward port")
+
+
+# XXX redundant, remove
+@contextlib.contextmanager
+def device(
+ app_name, marionette_port=2828, verbose=True, remote_test_root="/sdcard/test_root/"
+):
+ device_ = AndroidDevice(
+ app_name, marionette_port, verbose, remote_test_root=remote_test_root
+ )
+ try:
+ yield device_
+ finally:
+ device_.close()
+
+
+@attr.s
+class AndroidGeckodriver(Geckodriver):
+ async def start(self):
+ port = free_port()
+ await self._check_version()
+ logger.info("Running Webdriver on port %d" % port)
+ logger.info("Running Marionette on port 2828")
+ pargs = [
+ self.binary,
+ "--log",
+ "trace",
+ "--port",
+ str(port),
+ "--marionette-port",
+ "2828",
+ ]
+ logger.info("Connecting on Android device")
+ pargs.append("--connect-existing")
+ return await subprocess_based_service(
+ pargs, f"http://localhost:{port}", self.log_file
+ )
+
+
+class AndroidEnv(BaseEnv):
+ @contextlib.contextmanager
+ def get_device(self, *args, **kw):
+ with device(self.firefox, *args, **kw) as d:
+ self.device = d
+ yield self.device
+
+ def get_target_platform(self):
+ app = self.firefox.split("org.mozilla.")[-1]
+ if self.device_name is None:
+ return app
+ return "%s-%s" % (self.device_name, app)
+
+ def dump_logs(self):
+ logger.info("Dumping Android logs")
+ try:
+ logcat = self.device.get_logcat()
+ if logcat:
+ # local path, not using posixpath
+ logfile = os.path.join(self.archive, "logcat.log")
+ logger.info("Writing logcat at %s" % logfile)
+ with open(logfile, "wb") as f:
+ for line in logcat:
+ f.write(line.encode("utf8", errors="replace") + b"\n")
+ else:
+ logger.info("logcat came back empty")
+ except Exception:
+ logger.error("Could not extract the logcat", exc_info=True)
+
+ @contextlib.contextmanager
+ def get_browser(self):
+ yield
+
+ def get_browser_args(self, headless, prefs=None):
+ # XXX merge with DesktopEnv.get_browser_args
+ options = ["--allow-downgrade"]
+ if headless:
+ options.append("-headless")
+ if prefs is None:
+ prefs = {}
+ return {"moz:firefoxOptions": {"args": options, "prefs": prefs}}
+
+ def prepare(self, logfile):
+ self.device.prepare(self.profile, logfile)
+
+ def get_browser_version(self):
+ return self.target_platform + "-XXXneedtograbversion"
+
+ def get_geckodriver(self, log_file):
+ return AndroidGeckodriver(binary=self.geckodriver, log_file=log_file)
+
+ def check_session(self, session):
+ async def fake_close(*args):
+ pass
+
+ session.close = fake_close
+
+ def collect_profile(self):
+ self.device.collect_profile()
+
+ def stop_browser(self):
+ self.device.stop_browser()
diff --git a/testing/condprofile/condprof/archiver.py b/testing/condprofile/condprof/archiver.py
new file mode 100644
index 0000000000..db7fc94347
--- /dev/null
+++ b/testing/condprofile/condprof/archiver.py
@@ -0,0 +1,76 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""Helper to create tarballs.
+"""
+import copy
+import glob
+import os
+import tarfile
+
+from condprof import progress
+from condprof.util import TASK_CLUSTER
+
+
+def _tarinfo2mem(tar, tarinfo):
+ metadata = copy.copy(tarinfo)
+ try:
+ data = tar.extractfile(tarinfo)
+ if data is not None:
+ data = data.read()
+ except Exception:
+ data = None
+
+ return metadata, data
+
+
+class Archiver(object):
+ def __init__(self, scenario, profile_dir, archives_dir):
+ self.profile_dir = profile_dir
+ self.archives_dir = archives_dir
+ self.scenario = scenario
+
+ def _strftime(self, date, template="-%Y-%m-%d-hp.tar.gz"):
+ return date.strftime(self.scenario + template)
+
+ def _get_archive_path(self, when):
+ archive = self._strftime(when)
+ return os.path.join(self.archives_dir, archive), archive
+
+ def create_archive(self, when, iterator=None):
+ if iterator is None:
+
+ def _files(tar):
+ files = glob.glob(os.path.join(self.profile_dir, "*"))
+ yield len(files)
+ for filename in files:
+ try:
+ tar.add(filename, os.path.basename(filename))
+ yield filename
+ except FileNotFoundError: # NOQA
+ # locks and such
+ pass
+
+ iterator = _files
+
+ if isinstance(when, str):
+ archive = when
+ else:
+ archive, __ = self._get_archive_path(when)
+
+ with tarfile.open(archive, "w:gz", dereference=True) as tar:
+ it = iterator(tar)
+ size = next(it)
+ with progress.Bar(expected_size=size) as bar:
+ for filename in it:
+ if not TASK_CLUSTER:
+ bar.show(bar.last_progress + 1)
+
+ return archive
+
+ def _read_tar(self, filename):
+ files = {}
+ with tarfile.open(filename, "r:gz") as tar:
+ for tarinfo in tar:
+ files[tarinfo.name] = _tarinfo2mem(tar, tarinfo)
+ return files
diff --git a/testing/condprofile/condprof/changelog.py b/testing/condprofile/condprof/changelog.py
new file mode 100644
index 0000000000..72c8e7e8ea
--- /dev/null
+++ b/testing/condprofile/condprof/changelog.py
@@ -0,0 +1,56 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This module needs to stay Python 2 and 3 compatible
+#
+"""
+Maintains a unique file that lists all artifacts operations.
+"""
+import json
+import os
+import sys
+from datetime import datetime
+
+
+# XXX we should do one per platform and use platform-changelog.json as a name
+class Changelog:
+ def __init__(self, archives_dir):
+ self.archives_dir = archives_dir
+ self.location = os.path.join(archives_dir, "changelog.json")
+ if os.path.exists(self.location):
+ with open(self.location) as f:
+ self._data = json.loads(f.read())
+ if "changes" not in self._data:
+ self._data["changes"] = []
+ else:
+ self._data = {"changes": []}
+
+ def append(self, action, platform=sys.platform, **metadata):
+ now = datetime.timestamp(datetime.now())
+ log = {"action": action, "platform": platform, "when": now}
+ log.update(metadata)
+ # adding taskcluster specific info if we see it in the env
+ for key in (
+ "TC_SCHEDULER_ID",
+ "TASK_ID",
+ "TC_OWNER",
+ "TC_SOURCE",
+ "TC_PROJECT",
+ ):
+ if key in os.environ:
+ log[key] = os.environ[key]
+ self._data["changes"].append(log)
+
+ def save(self, archives_dir=None):
+ if archives_dir is not None and archives_dir != self.archives_dir:
+ self.location = os.path.join(archives_dir, "changelog.json")
+ # we need to resolve potential r/w conflicts on TC here
+ with open(self.location, "w") as f:
+ f.write(json.dumps(self._data))
+
+ def history(self):
+ """From older to newer"""
+ return sorted(
+ self._data["changes"], key=lambda entry: entry["when"], reverse=True
+ )
diff --git a/testing/condprofile/condprof/check_install.py b/testing/condprofile/condprof/check_install.py
new file mode 100644
index 0000000000..0209db3a55
--- /dev/null
+++ b/testing/condprofile/condprof/check_install.py
@@ -0,0 +1,71 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+""" Installs dependencies at runtime to simplify deployment.
+
+This module tries to make sure we have all dependencies installed on
+all our environments.
+"""
+import os
+import subprocess
+import sys
+
+PY3 = sys.version_info.major == 3
+TOPDIR = os.path.join(os.path.dirname(__file__), "..")
+
+
+def install_reqs():
+ """We install requirements one by one, with no cache, and in isolated mode."""
+ try:
+ import yaml # NOQA
+
+ return False
+ except Exception:
+ # we're detecting here that this is running in Taskcluster
+ # by checking for the presence of the mozfile directory
+ # that was decompressed from target.condprof.tests.tar.gz
+ run_in_ci = os.path.exists(os.path.join(TOPDIR, "mozfile"))
+
+ # On Python 2 we only install what's required for condprof.client
+ # On Python 3 it's the full thing
+ if not run_in_ci:
+ req_files = PY3 and ["base.txt", "local.txt"] or ["local-client.txt"]
+ else:
+ req_files = PY3 and ["base.txt", "ci.txt"] or ["ci-client.txt"]
+
+ for req_file in req_files:
+ req_file = os.path.join(TOPDIR, "requirements", req_file)
+
+ with open(req_file) as f:
+ reqs = [
+ req
+ for req in f.read().split("\n")
+ if req.strip() != "" and not req.startswith("#")
+ ]
+ for req in reqs:
+ subprocess.check_call(
+ [
+ sys.executable,
+ "-m",
+ "pip",
+ "install",
+ "--no-cache-dir",
+ "--isolated",
+ "--find-links",
+ "https://pypi.pub.build.mozilla.org/pub/",
+ req,
+ ]
+ )
+
+ return True
+
+
+def check():
+ """Called by the runner.
+
+ The check function will restart the app after
+ all deps have been installed.
+ """
+ if install_reqs():
+ os.execl(sys.executable, sys.executable, *sys.argv)
+ os._exit(0)
diff --git a/testing/condprofile/condprof/client.py b/testing/condprofile/condprof/client.py
new file mode 100644
index 0000000000..186e8cf4c4
--- /dev/null
+++ b/testing/condprofile/condprof/client.py
@@ -0,0 +1,255 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This module needs to stay Python 2 and 3 compatible
+#
+import datetime
+import functools
+import os
+import shutil
+import tarfile
+import tempfile
+import time
+
+from mozprofile.prefs import Preferences
+
+from condprof import progress
+from condprof.changelog import Changelog
+from condprof.util import (
+ TASK_CLUSTER,
+ ArchiveNotFound,
+ check_exists,
+ download_file,
+ logger,
+)
+
+TC_SERVICE = "https://firefox-ci-tc.services.mozilla.com"
+ROOT_URL = TC_SERVICE + "/api/index"
+INDEX_PATH = "gecko.v2.%(repo)s.latest.firefox.condprof-%(platform)s-%(scenario)s"
+INDEX_BY_DATE_PATH = "gecko.v2.%(repo)s.pushdate.%(date)s.latest.firefox.condprof-%(platform)s-%(scenario)s"
+PUBLIC_DIR = "artifacts/public/condprof"
+TC_LINK = ROOT_URL + "/v1/task/" + INDEX_PATH + "/" + PUBLIC_DIR + "/"
+TC_LINK_BY_DATE = ROOT_URL + "/v1/task/" + INDEX_BY_DATE_PATH + "/" + PUBLIC_DIR + "/"
+ARTIFACT_NAME = "profile%(version)s-%(platform)s-%(scenario)s-%(customization)s.tgz"
+CHANGELOG_LINK = (
+ ROOT_URL + "/v1/task/" + INDEX_PATH + "/" + PUBLIC_DIR + "/changelog.json"
+)
+ARTIFACTS_SERVICE = "https://taskcluster-artifacts.net"
+DIRECT_LINK = ARTIFACTS_SERVICE + "/%(task_id)s/0/public/condprof/"
+CONDPROF_CACHE = "~/.condprof-cache"
+RETRIES = 3
+RETRY_PAUSE = 45
+
+
+class ServiceUnreachableError(Exception):
+ pass
+
+
+class ProfileNotFoundError(Exception):
+ pass
+
+
+class RetriesError(Exception):
+ pass
+
+
+def _check_service(url):
+ """Sanity check to see if we can reach the service root url."""
+
+ def _check():
+ exists, _ = check_exists(url, all_types=True)
+ if not exists:
+ raise ServiceUnreachableError(url)
+
+ try:
+ return _retries(_check)
+ except RetriesError:
+ raise ServiceUnreachableError(url)
+
+
+def _check_profile(profile_dir):
+ """Checks for prefs we need to remove or set."""
+ to_remove = ("gfx.blacklist.", "marionette.")
+
+ def _keep_pref(name, value):
+ for item in to_remove:
+ if not name.startswith(item):
+ continue
+ logger.info("Removing pref %s: %s" % (name, value))
+ return False
+ return True
+
+ def _clean_pref_file(name):
+ js_file = os.path.join(profile_dir, name)
+ prefs = Preferences.read_prefs(js_file)
+ cleaned_prefs = dict([pref for pref in prefs if _keep_pref(*pref)])
+ if name == "prefs.js":
+ # When we start Firefox, forces startupScanScopes to SCOPE_PROFILE (1)
+ # otherwise, side loading will be deactivated and the
+ # Raptor web extension won't be able to run.
+ cleaned_prefs["extensions.startupScanScopes"] = 1
+
+ # adding a marker so we know it's a conditioned profile
+ cleaned_prefs["profile.conditioned"] = True
+
+ with open(js_file, "w") as f:
+ Preferences.write(f, cleaned_prefs)
+
+ _clean_pref_file("prefs.js")
+ _clean_pref_file("user.js")
+
+
+def _retries(callable, onerror=None, retries=RETRIES):
+ _retry_count = 0
+ pause = RETRY_PAUSE
+
+ while _retry_count < retries:
+ try:
+ return callable()
+ except Exception as e:
+ if onerror is not None:
+ onerror(e)
+ logger.info("Failed, retrying")
+ _retry_count += 1
+ time.sleep(pause)
+ pause *= 1.5
+
+ # If we reach that point, it means all attempts failed
+ if _retry_count >= RETRIES:
+ logger.error("All attempt failed")
+ else:
+ logger.info("Retried %s attempts and failed" % _retry_count)
+ raise RetriesError()
+
+
+def get_profile(
+ target_dir,
+ platform,
+ scenario,
+ customization="default",
+ task_id=None,
+ download_cache=True,
+ repo="mozilla-central",
+ remote_test_root="/sdcard/test_root/",
+ version=None,
+ retries=RETRIES,
+):
+ """Extract a conditioned profile in the target directory.
+
+ If task_id is provided, will grab the profile from that task. when not
+ provided (default) will grab the latest profile.
+ """
+
+ # XXX assert values
+ if version:
+ version = "-v%s" % version
+ else:
+ version = ""
+
+ # when we bump the Firefox version on trunk, autoland still needs to catch up
+ # in this case we want to download an older profile- 2 days to account for closures/etc.
+ oldday = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=2)
+ params = {
+ "platform": platform,
+ "scenario": scenario,
+ "customization": customization,
+ "task_id": task_id,
+ "repo": repo,
+ "version": version,
+ "date": str(oldday.date()).replace("-", "."),
+ }
+ logger.info("Getting conditioned profile with arguments: %s" % params)
+ filename = ARTIFACT_NAME % params
+ if task_id is None:
+ url = TC_LINK % params + filename
+ _check_service(TC_SERVICE)
+ else:
+ url = DIRECT_LINK % params + filename
+ _check_service(ARTIFACTS_SERVICE)
+
+ logger.info("preparing download dir")
+ if not download_cache:
+ download_dir = tempfile.mkdtemp()
+ else:
+ # using a cache dir in the user home dir
+ download_dir = os.path.expanduser(CONDPROF_CACHE)
+ if not os.path.exists(download_dir):
+ os.makedirs(download_dir)
+
+ downloaded_archive = os.path.join(download_dir, filename)
+ logger.info("Downloaded archive path: %s" % downloaded_archive)
+
+ def _get_profile():
+ logger.info("Getting %s" % url)
+ try:
+ archive = download_file(url, target=downloaded_archive)
+ except ArchiveNotFound:
+ raise ProfileNotFoundError(url)
+ try:
+ with tarfile.open(archive, "r:gz") as tar:
+ logger.info("Extracting the tarball content in %s" % target_dir)
+ size = len(list(tar))
+ with progress.Bar(expected_size=size) as bar:
+
+ def _extract(self, *args, **kw):
+ if not TASK_CLUSTER:
+ bar.show(bar.last_progress + 1)
+ return self.old(*args, **kw)
+
+ tar.old = tar.extract
+ tar.extract = functools.partial(_extract, tar)
+ tar.extractall(target_dir)
+ except (OSError, tarfile.ReadError) as e:
+ logger.info("Failed to extract the tarball")
+ if download_cache and os.path.exists(archive):
+ logger.info("Removing cached file to attempt a new download")
+ os.remove(archive)
+ raise ProfileNotFoundError(str(e))
+ finally:
+ if not download_cache:
+ shutil.rmtree(download_dir)
+
+ _check_profile(target_dir)
+ logger.info("Success, we have a profile to work with")
+ return target_dir
+
+ def onerror(error):
+ logger.info("Failed to get the profile.")
+ if os.path.exists(downloaded_archive):
+ try:
+ os.remove(downloaded_archive)
+ except Exception:
+ logger.error("Could not remove the file")
+
+ try:
+ return _retries(_get_profile, onerror, retries)
+ except RetriesError:
+ # look for older profile 2 days previously
+ filename = ARTIFACT_NAME % params
+ url = TC_LINK_BY_DATE % params + filename
+ try:
+ return _retries(_get_profile, onerror, retries)
+ except RetriesError:
+ raise ProfileNotFoundError(url)
+
+
+def read_changelog(platform, repo="mozilla-central", scenario="settled"):
+ params = {"platform": platform, "repo": repo, "scenario": scenario}
+ changelog_url = CHANGELOG_LINK % params
+ logger.info("Getting %s" % changelog_url)
+ download_dir = tempfile.mkdtemp()
+ downloaded_changelog = os.path.join(download_dir, "changelog.json")
+
+ def _get_changelog():
+ try:
+ download_file(changelog_url, target=downloaded_changelog)
+ except ArchiveNotFound:
+ shutil.rmtree(download_dir)
+ raise ProfileNotFoundError(changelog_url)
+ return Changelog(download_dir)
+
+ try:
+ return _retries(_get_changelog)
+ except Exception:
+ raise ProfileNotFoundError(changelog_url)
diff --git a/testing/condprofile/condprof/creator.py b/testing/condprofile/condprof/creator.py
new file mode 100644
index 0000000000..7e03f6a8da
--- /dev/null
+++ b/testing/condprofile/condprof/creator.py
@@ -0,0 +1,226 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+""" Creates or updates profiles.
+
+The profile creation works as following:
+
+For each scenario:
+
+- The latest indexed profile is picked on TC, if none we create a fresh profile
+- The scenario is done against it
+- The profile is uploaded on TC, replacing the previous one as the freshest
+
+For each platform we keep a changelog file that keep track of each update
+with the Task ID. That offers us the ability to get a profile from a specific
+date in the past.
+
+Artifacts are staying in TaskCluster for 3 months, and then they are removed,
+so the oldest profile we can get is 3 months old. Profiles are being updated
+continuously, so even after 3 months they are still getting "older".
+
+When Firefox changes its version, profiles from the previous version
+should work as expected. Each profile tarball comes with a metadata file
+that keep track of the Firefox version that was used and the profile age.
+"""
+import os
+import tempfile
+import shutil
+
+from arsenic import get_session
+from arsenic.browsers import Firefox
+
+from condprof.util import fresh_profile, logger, obfuscate_file, obfuscate, get_version
+from condprof.helpers import close_extra_windows
+from condprof.scenarii import scenarii
+from condprof.client import get_profile, ProfileNotFoundError
+from condprof.archiver import Archiver
+from condprof.customization import get_customization
+from condprof.metadata import Metadata
+
+
+START, INIT_GECKODRIVER, START_SESSION, START_SCENARIO = range(4)
+
+
+class ProfileCreator:
+ def __init__(
+ self,
+ scenario,
+ customization,
+ archive,
+ changelog,
+ force_new,
+ env,
+ skip_logs=False,
+ remote_test_root="/sdcard/test_root/",
+ ):
+ self.env = env
+ self.scenario = scenario
+ self.customization = customization
+ self.archive = archive
+ self.changelog = changelog
+ self.force_new = force_new
+ self.skip_logs = skip_logs
+ self.remote_test_root = remote_test_root
+ self.customization_data = get_customization(customization)
+ self.tmp_dir = None
+
+ # Make a temporary directory for the logs if an
+ # archive dir is not provided
+ if not self.archive:
+ self.tmp_dir = tempfile.mkdtemp()
+
+ def _log_filename(self, name):
+ filename = "%s-%s-%s.log" % (
+ name,
+ self.scenario,
+ self.customization_data["name"],
+ )
+ return os.path.join(self.archive or self.tmp_dir, filename)
+
+ async def run(self, headless=True):
+ logger.info(
+ "Building %s x %s" % (self.scenario, self.customization_data["name"])
+ )
+
+ if self.scenario in self.customization_data.get("ignore_scenario", []):
+ logger.info("Skipping (ignored scenario in that customization)")
+ return
+
+ filter_by_platform = self.customization_data.get("platforms")
+ if filter_by_platform and self.env.target_platform not in filter_by_platform:
+ logger.info("Skipping (ignored platform in that customization)")
+ return
+
+ with self.env.get_device(
+ 2828, verbose=True, remote_test_root=self.remote_test_root
+ ) as device:
+ try:
+ with self.env.get_browser():
+ metadata = await self.build_profile(device, headless)
+ except Exception:
+ raise
+ finally:
+ if not self.skip_logs:
+ self.env.dump_logs()
+
+ if not self.archive:
+ return
+
+ logger.info("Creating generic archive")
+ names = ["profile-%(platform)s-%(name)s-%(customization)s.tgz"]
+ if metadata["name"] == "full" and metadata["customization"] == "default":
+ names = [
+ "profile-%(platform)s-%(name)s-%(customization)s.tgz",
+ "profile-v%(version)s-%(platform)s-%(name)s-%(customization)s.tgz",
+ ]
+
+ for name in names:
+ # remove `cache` from profile
+ shutil.rmtree(os.path.join(self.env.profile, "cache"), ignore_errors=True)
+ shutil.rmtree(os.path.join(self.env.profile, "cache2"), ignore_errors=True)
+
+ archiver = Archiver(self.scenario, self.env.profile, self.archive)
+ # the archive name is of the form
+ # profile[-vXYZ.x]-<platform>-<scenario>-<customization>.tgz
+ name = name % metadata
+ archive_name = os.path.join(self.archive, name)
+ dir = os.path.dirname(archive_name)
+ if not os.path.exists(dir):
+ os.makedirs(dir)
+ archiver.create_archive(archive_name)
+ logger.info("Archive created at %s" % archive_name)
+ statinfo = os.stat(archive_name)
+ logger.info("Current size is %d" % statinfo.st_size)
+
+ logger.info("Extracting logs")
+ if "logs" in metadata:
+ logs = metadata.pop("logs")
+ for prefix, prefixed_logs in logs.items():
+ for log in prefixed_logs:
+ content = obfuscate(log["content"])[1]
+ with open(os.path.join(dir, prefix + "-" + log["name"]), "wb") as f:
+ f.write(content.encode("utf-8"))
+
+ if metadata.get("result", 0) != 0:
+ logger.info("The scenario returned a bad exit code")
+ raise Exception(metadata.get("result_message", "scenario error"))
+ self.changelog.append("update", **metadata)
+
+ async def build_profile(self, device, headless):
+ scenario = self.scenario
+ profile = self.env.profile
+ customization_data = self.customization_data
+
+ scenario_func = scenarii[scenario]
+ if scenario in customization_data.get("scenario", {}):
+ options = customization_data["scenario"][scenario]
+ logger.info("Loaded options for that scenario %s" % str(options))
+ else:
+ options = {}
+
+ # Adding general options
+ options["platform"] = self.env.target_platform
+
+ if not self.force_new:
+ try:
+ custom_name = customization_data["name"]
+ get_profile(profile, self.env.target_platform, scenario, custom_name)
+ except ProfileNotFoundError:
+ # XXX we'll use a fresh profile for now
+ fresh_profile(profile, customization_data)
+ else:
+ fresh_profile(profile, customization_data)
+
+ logger.info("Updating profile located at %r" % profile)
+ metadata = Metadata(profile)
+
+ logger.info("Starting the Gecko app...")
+ adb_logs = self._log_filename("adb")
+ self.env.prepare(logfile=adb_logs)
+ geckodriver_logs = self._log_filename("geckodriver")
+ logger.info("Writing geckodriver logs in %s" % geckodriver_logs)
+ step = START
+ try:
+ firefox_instance = Firefox(**self.env.get_browser_args(headless))
+ step = INIT_GECKODRIVER
+ with open(geckodriver_logs, "w") as glog:
+ geckodriver = self.env.get_geckodriver(log_file=glog)
+ step = START_SESSION
+ async with get_session(geckodriver, firefox_instance) as session:
+ step = START_SCENARIO
+ self.env.check_session(session)
+ logger.info("Running the %s scenario" % scenario)
+ metadata.update(await scenario_func(session, options))
+ logger.info("%s scenario done." % scenario)
+ await close_extra_windows(session)
+ except Exception:
+ logger.error("%s scenario broke!" % scenario)
+ if step == START:
+ logger.info("Could not initialize the browser")
+ elif step == INIT_GECKODRIVER:
+ logger.info("Could not initialize Geckodriver")
+ elif step == START_SESSION:
+ logger.info(
+ "Could not start the session, check %s first" % geckodriver_logs
+ )
+ else:
+ logger.info("Could not run the scenario, probably a faulty scenario")
+ raise
+ finally:
+ self.env.stop_browser()
+ for logfile in (adb_logs, geckodriver_logs):
+ if os.path.exists(logfile):
+ obfuscate_file(logfile)
+ self.env.collect_profile()
+
+ # writing metadata
+ metadata.write(
+ name=self.scenario,
+ customization=self.customization_data["name"],
+ version=self.env.get_browser_version(),
+ platform=self.env.target_platform,
+ )
+
+ logger.info("Profile at %s.\nDone." % profile)
+ return metadata
diff --git a/testing/condprofile/condprof/customization/__init__.py b/testing/condprofile/condprof/customization/__init__.py
new file mode 100644
index 0000000000..64f09bf4f7
--- /dev/null
+++ b/testing/condprofile/condprof/customization/__init__.py
@@ -0,0 +1,35 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+
+HERE = os.path.dirname(__file__)
+
+
+def get_customizations():
+ for f in os.listdir(HERE):
+ if not f.endswith("json"):
+ continue
+ yield os.path.join(HERE, f)
+
+
+def find_customization(path_or_name):
+ if not path_or_name.endswith(".json"):
+ path_or_name += ".json"
+ if not os.path.exists(path_or_name):
+ # trying relative
+ rpath = os.path.join(HERE, path_or_name)
+ if not os.path.exists(rpath):
+ return None
+ path_or_name = rpath
+ return path_or_name
+
+
+def get_customization(path_or_name):
+ path = find_customization(path_or_name)
+ if path is None:
+ raise IOError("Can't find the customization file %r" % path_or_name)
+ with open(path) as f:
+ return json.loads(f.read())
diff --git a/testing/condprofile/condprof/customization/default.json b/testing/condprofile/condprof/customization/default.json
new file mode 100644
index 0000000000..eb1ca25e91
--- /dev/null
+++ b/testing/condprofile/condprof/customization/default.json
@@ -0,0 +1,10 @@
+{
+ "name": "default",
+ "addons": {},
+ "prefs": {
+ "gfx.webrender.precache-shaders": true
+ },
+ "scenario": {
+ "full": { "max_urls": 150 }
+ }
+}
diff --git a/testing/condprofile/condprof/customization/webext.json b/testing/condprofile/condprof/customization/webext.json
new file mode 100644
index 0000000000..6647765387
--- /dev/null
+++ b/testing/condprofile/condprof/customization/webext.json
@@ -0,0 +1,15 @@
+{
+ "name": "webext",
+ "addons": {
+ "uBlock": "https://addons.mozilla.org/firefox/downloads/file/4171020/ublock_origin-1.52.2.xpi"
+ },
+ "prefs": {
+ "gfx.webrender.precache-shaders": true,
+ "extensions.autoDisableScopes": 0,
+ "extensions.enabledScopes": 1,
+ "extensions.startupScanScopes": 1
+ },
+ "scenario": {
+ "full": { "max_urls": 150 }
+ }
+}
diff --git a/testing/condprofile/condprof/customization/youtube.json b/testing/condprofile/condprof/customization/youtube.json
new file mode 100644
index 0000000000..0d05ea0ed9
--- /dev/null
+++ b/testing/condprofile/condprof/customization/youtube.json
@@ -0,0 +1,12 @@
+{
+ "name": "youtube",
+ "addons": {},
+ "prefs": {
+ "media.eme.enabled": true,
+ "media.gmp-manager.updateEnabled": true,
+ "media.eme.require-app-approval": false
+ },
+ "scenario": {
+ "full": { "max_urls": 150 }
+ }
+}
diff --git a/testing/condprofile/condprof/desktop.py b/testing/condprofile/condprof/desktop.py
new file mode 100644
index 0000000000..7f8fe4e309
--- /dev/null
+++ b/testing/condprofile/condprof/desktop.py
@@ -0,0 +1,90 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import os
+import contextlib
+
+import attr
+from arsenic.services import Geckodriver, free_port, subprocess_based_service
+
+from condprof.util import (
+ BaseEnv,
+ latest_nightly,
+ logger,
+ get_version,
+ get_current_platform,
+ DEFAULT_PREFS,
+)
+
+
+@attr.s
+class DesktopGeckodriver(Geckodriver):
+ async def start(self):
+ port = free_port()
+ await self._check_version()
+ logger.info("Running Webdriver on port %d" % port)
+ logger.info("Running Marionette on port 2828")
+ pargs = [
+ self.binary,
+ "--log",
+ "trace",
+ "--port",
+ str(port),
+ "--marionette-port",
+ "2828",
+ ]
+ try:
+ return await subprocess_based_service(
+ pargs, f"http://localhost:{port}", self.log_file
+ )
+ except ProcessLookupError as e:
+ return await subprocess_based_service(
+ pargs.extend(["--host", "127.0.0.1"]),
+ f"http://localhost:{port}",
+ self.log_file,
+ )
+
+
+@contextlib.contextmanager
+def dummy_device(*args, **kw):
+ yield None
+
+
+class DesktopEnv(BaseEnv):
+ def get_target_platform(self):
+ return get_current_platform()
+
+ def get_device(self, *args, **kw):
+ return dummy_device(*args, **kw)
+
+ @contextlib.contextmanager
+ def get_browser(self):
+ with latest_nightly(self.firefox) as binary:
+ self.firefox = os.path.abspath(binary)
+ if not os.path.exists(self.firefox):
+ raise IOError(self.firefox)
+ yield
+
+ def get_browser_args(self, headless, prefs=None):
+ final_prefs = dict(DEFAULT_PREFS)
+ if prefs is not None:
+ final_prefs.update(prefs)
+ options = ["--allow-downgrade", "-profile", self.profile]
+ if headless:
+ options.append("-headless")
+ args = {"moz:firefoxOptions": {"args": options}}
+ if self.firefox is not None:
+ args["moz:firefoxOptions"]["binary"] = self.firefox
+ args["moz:firefoxOptions"]["prefs"] = final_prefs
+ args["moz:firefoxOptions"]["log"] = {"level": "trace"}
+ return args
+
+ def get_browser_version(self):
+ try:
+ return get_version(self.firefox)
+ except Exception:
+ logger.error("Could not get Firefox version", exc_info=True)
+ return "unknown"
+
+ def get_geckodriver(self, log_file):
+ return DesktopGeckodriver(binary=self.geckodriver, log_file=log_file)
diff --git a/testing/condprofile/condprof/helpers.py b/testing/condprofile/condprof/helpers.py
new file mode 100644
index 0000000000..befe9e0331
--- /dev/null
+++ b/testing/condprofile/condprof/helpers.py
@@ -0,0 +1,110 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+""" Helpers to build scenarii
+"""
+from condprof.util import logger
+
+_SUPPORTED_MOBILE_BROWSERS = "fenix", "gecko", "firefox"
+
+
+def is_mobile(platform):
+ return any(mobile in platform for mobile in _SUPPORTED_MOBILE_BROWSERS)
+
+
+class TabSwitcher:
+ """Helper used to create tabs and circulate in them."""
+
+ def __init__(self, session, options):
+ self.handles = None
+ self.current = 0
+ self.session = session
+ self._max = options.get("max_urls", 10)
+ self.platform = options.get("platform", "")
+ self.num_tabs = self._max >= 100 and 100 or self._max
+ self._mobile = is_mobile(self.platform)
+
+ async def create_windows(self):
+ # on mobile we don't use tabs for now
+ # see https://bugzil.la/1559120
+ if self._mobile:
+ return
+ # creating tabs
+ for i in range(self.num_tabs):
+ # see https://github.com/HDE/arsenic/issues/71
+ await self.session._request(
+ url="/window/new", method="POST", data={"type": "tab"}
+ )
+
+ async def switch(self):
+ if self._mobile:
+ return
+ try:
+ if self.handles is None:
+ self.handles = await self.session.get_window_handles()
+ self.current = 0
+ except Exception:
+ logger.error("Could not get window handles")
+ return
+
+ handle = self.handles[self.current]
+ if self.current == len(self.handles) - 1:
+ self.current = 0
+ else:
+ self.current += 1
+ try:
+ await self.session.switch_to_window(handle)
+ except Exception:
+ logger.error("Could not switch to handle %s" % str(handle))
+
+
+# 10 minutes
+_SCRIPT_TIMEOUT = 10 * 60 * 1000
+
+
+async def execute_async_script(session, script, *args):
+ # switch to the right context if needed
+ current_context = await session._request(url="/moz/context", method="GET")
+ if current_context != "chrome":
+ await session._request(
+ url="/moz/context", method="POST", data={"context": "chrome"}
+ )
+ switch_back = True
+ else:
+ switch_back = False
+ await session._request(
+ url="/timeouts", method="POST", data={"script": _SCRIPT_TIMEOUT}
+ )
+ try:
+ attempts = 0
+ while True:
+ try:
+ return await session._request(
+ url="/execute/async",
+ method="POST",
+ data={"script": script, "args": list(args)},
+ )
+ except Exception as e:
+ attempts += 1
+ logger.error("The script failed.", exc_info=True)
+ if attempts > 2:
+ return {
+ "result": 1,
+ "result_message": str(e),
+ "result_exc": e,
+ "logs": {},
+ }
+ finally:
+ if switch_back:
+ await session._request(
+ url="/moz/context", method="POST", data={"context": current_context}
+ )
+
+
+async def close_extra_windows(session):
+ logger.info("Closing all tabs")
+ handles = await session.get_window_handles()
+ # we're closing all tabs except the last one
+ for handle in handles[:-1]:
+ await session.switch_to_window(handle)
+ await session._request(url="/window", method="DELETE")
diff --git a/testing/condprofile/condprof/main.py b/testing/condprofile/condprof/main.py
new file mode 100644
index 0000000000..b53e199220
--- /dev/null
+++ b/testing/condprofile/condprof/main.py
@@ -0,0 +1,86 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+""" Script that launches profiles creation.
+"""
+import argparse
+import os
+import sys
+
+# easier than setting PYTHONPATH in various platforms
+if __name__ == "__main__":
+ sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
+
+from condprof.check_install import check # NOQA
+
+if "MANUAL_MACH_RUN" not in os.environ:
+ check()
+
+from condprof import patch # noqa
+
+
+def main(args=sys.argv[1:]):
+ parser = argparse.ArgumentParser(description="Profile Creator")
+ parser.add_argument("archive", help="Archives Dir", type=str, default=None)
+ parser.add_argument("--firefox", help="Firefox Binary", type=str, default=None)
+ parser.add_argument("--scenario", help="Scenario to use", type=str, default="all")
+ parser.add_argument(
+ "--profile", help="Existing profile Dir", type=str, default=None
+ )
+ parser.add_argument(
+ "--customization",
+ help="Profile customization to use",
+ type=str,
+ default="default",
+ )
+ parser.add_argument(
+ "--visible", help="Don't use headless mode", action="store_true", default=False
+ )
+ parser.add_argument(
+ "--archives-dir", help="Archives local dir", type=str, default="/tmp/archives"
+ )
+ parser.add_argument(
+ "--force-new", help="Create from scratch", action="store_true", default=False
+ )
+ parser.add_argument(
+ "--strict",
+ help="Errors out immediatly on a scenario failure",
+ action="store_true",
+ default=False,
+ )
+ parser.add_argument(
+ "--geckodriver",
+ help="Path to the geckodriver binary",
+ type=str,
+ default=sys.platform.startswith("win") and "geckodriver.exe" or "geckodriver",
+ )
+
+ parser.add_argument(
+ "--device-name", help="Name of the device", type=str, default=None
+ )
+
+ args = parser.parse_args(args=args)
+ os.environ["CONDPROF_RUNNER"] = "1"
+
+ from condprof.runner import run # NOQA
+
+ try:
+ run(
+ args.archive,
+ args.firefox,
+ args.scenario,
+ args.profile,
+ args.customization,
+ args.visible,
+ args.archives_dir,
+ args.force_new,
+ args.strict,
+ args.geckodriver,
+ args.device_name,
+ )
+ except Exception:
+ sys.exit(4) # TBPL_RETRY
+
+
+if __name__ == "__main__":
+ main()
diff --git a/testing/condprofile/condprof/metadata.py b/testing/condprofile/condprof/metadata.py
new file mode 100644
index 0000000000..4d03e57f6f
--- /dev/null
+++ b/testing/condprofile/condprof/metadata.py
@@ -0,0 +1,83 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+""" Manages a metadata file.
+"""
+import datetime
+import json
+import os
+from collections.abc import MutableMapping
+
+from condprof.util import logger
+
+METADATA_NAME = "condprofile.json"
+
+
+class Metadata(MutableMapping):
+ """dict-like class that holds metadata for a profile."""
+
+ def __init__(self, profile_dir):
+ self.metadata_file = os.path.join(profile_dir, METADATA_NAME)
+ logger.info("Reading existing metadata at %s" % self.metadata_file)
+ if not os.path.exists(self.metadata_file):
+ logger.info("Could not find the metadata file in that profile")
+ self._data = {}
+ else:
+ with open(self.metadata_file) as f:
+ self._data = json.loads(f.read())
+
+ def __getitem__(self, key):
+ return self._data[self.__keytransform__(key)]
+
+ def __setitem__(self, key, value):
+ self._data[self.__keytransform__(key)] = value
+
+ def __delitem__(self, key):
+ del self._data[self.__keytransform__(key)]
+
+ def __iter__(self):
+ return iter(self._data)
+
+ def __len__(self):
+ return len(self._data)
+
+ def __keytransform__(self, key):
+ return key
+
+ def _days2age(self, days):
+ if days < 7:
+ return "days"
+ if days < 30:
+ return "weeks"
+ if days < 30 * 6:
+ return "months"
+ return "old" # :)
+
+ def _delta(self, created, updated):
+ created = created[:26]
+ updated = updated[:26]
+ # tz..
+ format = "%Y-%m-%d %H:%M:%S.%f"
+ created = datetime.datetime.strptime(created, format)
+ updated = datetime.datetime.strptime(updated, format)
+ delta = created - updated
+ return delta.days
+
+ def write(self, **extras):
+ # writing metadata
+ logger.info("Creating metadata...")
+ self._data.update(**extras)
+ ts = str(datetime.datetime.now())
+ if "created" not in self._data:
+ self._data["created"] = ts
+ self._data["updated"] = ts
+ # XXX need android arch version here
+ days = self._delta(self._data["created"], self._data["updated"])
+ self._data["days"] = days
+ self._data["age"] = self._days2age(days)
+ # adding info about the firefox version
+ # XXX build ID ??
+ # XXX android ??
+ logger.info("Saving metadata file in %s" % self.metadata_file)
+ with open(self.metadata_file, "w") as f:
+ f.write(json.dumps(self._data))
diff --git a/testing/condprofile/condprof/patch.py b/testing/condprofile/condprof/patch.py
new file mode 100644
index 0000000000..35e4c978b5
--- /dev/null
+++ b/testing/condprofile/condprof/patch.py
@@ -0,0 +1,47 @@
+# flake8: noqa
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# patch for https://bugzilla.mozilla.org/show_bug.cgi?id=1655869
+# see https://github.com/HDE/arsenic/issues/85
+from arsenic.connection import *
+
+
+@ensure_task
+async def request(self, *, url: str, method: str, data=None) -> Tuple[int, Any]:
+ if data is None:
+ data = {}
+ if method not in {"POST", "PUT"}:
+ data = None
+ headers = {}
+ else:
+ headers = {"Content-Type": "application/json"}
+ body = json.dumps(data) if data is not None else None
+ full_url = self.prefix + url
+ log.info(
+ "request", url=strip_auth(full_url), method=method, body=body, headers=headers
+ )
+
+ async with self.session.request(
+ url=full_url, method=method, data=body, headers=headers
+ ) as response:
+ response_body = await response.read()
+ try:
+ data = json.loads(response_body)
+ except JSONDecodeError as exc:
+ log.error("json-decode", body=response_body)
+ data = {"error": "!internal", "message": str(exc), "stacktrace": ""}
+ wrap_screen(data)
+ log.info(
+ "response",
+ url=strip_auth(full_url),
+ method=method,
+ body=body,
+ response=response,
+ data=data,
+ )
+ return response.status, data
+
+
+Connection.request = request
diff --git a/testing/condprofile/condprof/progress.py b/testing/condprofile/condprof/progress.py
new file mode 100644
index 0000000000..097af53a8c
--- /dev/null
+++ b/testing/condprofile/condprof/progress.py
@@ -0,0 +1,223 @@
+# -*- coding: utf-8 -*-
+"""
+clint.textui.progress
+~~~~~~~~~~~~~~~~~
+This module provides the progressbar functionality.
+
+
+ISC License
+
+Copyright (c) 2011, Kenneth Reitz <me@kennethreitz.com>
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+"""
+
+import sys
+import time
+
+STREAM = sys.stderr
+
+BAR_TEMPLATE = "%s[%s%s] %i/%i - %s\r"
+MILL_TEMPLATE = "%s %s %i/%i\r"
+
+DOTS_CHAR = "."
+BAR_FILLED_CHAR = "#"
+BAR_EMPTY_CHAR = " "
+MILL_CHARS = ["|", "/", "-", "\\"]
+
+# How long to wait before recalculating the ETA
+ETA_INTERVAL = 1
+# How many intervals (excluding the current one) to calculate the simple moving
+# average
+ETA_SMA_WINDOW = 9
+
+
+class Bar(object):
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.done()
+ return False # we're not suppressing exceptions
+
+ def __init__(
+ self,
+ label="",
+ width=32,
+ hide=None,
+ empty_char=BAR_EMPTY_CHAR,
+ filled_char=BAR_FILLED_CHAR,
+ expected_size=None,
+ every=1,
+ ):
+ self.label = label
+ self.width = width
+ self.hide = hide
+ # Only show bar in terminals by default (better for piping, logging etc.)
+ if hide is None:
+ try:
+ self.hide = not STREAM.isatty()
+ except AttributeError: # output does not support isatty()
+ self.hide = True
+ self.empty_char = empty_char
+ self.filled_char = filled_char
+ self.expected_size = expected_size
+ self.every = every
+ self.start = time.time()
+ self.ittimes = []
+ self.eta = 0
+ self.etadelta = time.time()
+ self.etadisp = self.format_time(self.eta)
+ self.last_progress = 0
+ if self.expected_size:
+ self.show(0)
+
+ def show(self, progress, count=None):
+ if count is not None:
+ self.expected_size = count
+ if self.expected_size is None:
+ raise Exception("expected_size not initialized")
+ self.last_progress = progress
+ if (time.time() - self.etadelta) > ETA_INTERVAL:
+ self.etadelta = time.time()
+ # pylint --py3k W1619
+ self.ittimes = self.ittimes[-ETA_SMA_WINDOW:] + [
+ -(self.start - time.time()) / (progress + 1)
+ ]
+ self.eta = (
+ sum(self.ittimes)
+ / float(len(self.ittimes))
+ * (self.expected_size - progress)
+ )
+ self.etadisp = self.format_time(self.eta)
+ # pylint --py3k W1619
+ x = int(self.width * progress / self.expected_size)
+ if not self.hide:
+ if (progress % self.every) == 0 or ( # True every "every" updates
+ progress == self.expected_size
+ ): # And when we're done
+ STREAM.write(
+ BAR_TEMPLATE
+ % (
+ self.label,
+ self.filled_char * x,
+ self.empty_char * (self.width - x),
+ progress,
+ self.expected_size,
+ self.etadisp,
+ )
+ )
+ STREAM.flush()
+
+ def done(self):
+ self.elapsed = time.time() - self.start
+ elapsed_disp = self.format_time(self.elapsed)
+ if not self.hide:
+ # Print completed bar with elapsed time
+ STREAM.write(
+ BAR_TEMPLATE
+ % (
+ self.label,
+ self.filled_char * self.width,
+ self.empty_char * 0,
+ self.last_progress,
+ self.expected_size,
+ elapsed_disp,
+ )
+ )
+ STREAM.write("\n")
+ STREAM.flush()
+
+ def format_time(self, seconds):
+ return time.strftime("%H:%M:%S", time.gmtime(seconds))
+
+
+def bar(
+ it,
+ label="",
+ width=32,
+ hide=None,
+ empty_char=BAR_EMPTY_CHAR,
+ filled_char=BAR_FILLED_CHAR,
+ expected_size=None,
+ every=1,
+):
+ """Progress iterator. Wrap your iterables with it."""
+
+ count = len(it) if expected_size is None else expected_size
+
+ with Bar(
+ label=label,
+ width=width,
+ hide=hide,
+ empty_char=empty_char,
+ filled_char=filled_char,
+ expected_size=count,
+ every=every,
+ ) as bar:
+ for i, item in enumerate(it):
+ yield item
+ bar.show(i + 1)
+
+
+def dots(it, label="", hide=None, every=1):
+ """Progress iterator. Prints a dot for each item being iterated"""
+
+ count = 0
+
+ if not hide:
+ STREAM.write(label)
+
+ for i, item in enumerate(it):
+ if not hide:
+ if i % every == 0: # True every "every" updates
+ STREAM.write(DOTS_CHAR)
+ sys.stderr.flush()
+
+ count += 1
+
+ yield item
+
+ STREAM.write("\n")
+ STREAM.flush()
+
+
+def mill(it, label="", hide=None, expected_size=None, every=1):
+ """Progress iterator. Prints a mill while iterating over the items."""
+
+ def _mill_char(_i):
+ if _i >= count:
+ return " "
+ else:
+ return MILL_CHARS[(_i // every) % len(MILL_CHARS)]
+
+ def _show(_i):
+ if not hide:
+ if (_i % every) == 0 or ( # True every "every" updates
+ _i == count
+ ): # And when we're done
+ STREAM.write(MILL_TEMPLATE % (label, _mill_char(_i), _i, count))
+ STREAM.flush()
+
+ count = len(it) if expected_size is None else expected_size
+
+ if count:
+ _show(0)
+
+ for i, item in enumerate(it):
+ yield item
+ _show(i + 1)
+
+ if not hide:
+ STREAM.write("\n")
+ STREAM.flush()
diff --git a/testing/condprofile/condprof/runner.py b/testing/condprofile/condprof/runner.py
new file mode 100644
index 0000000000..07fb6d3e73
--- /dev/null
+++ b/testing/condprofile/condprof/runner.py
@@ -0,0 +1,211 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+""" Script that launches profiles creation.
+"""
+import os
+import shutil
+import asyncio
+
+import mozversion
+
+from condprof.creator import ProfileCreator
+from condprof.desktop import DesktopEnv
+from condprof.android import AndroidEnv
+from condprof.changelog import Changelog
+from condprof.scenarii import scenarii
+from condprof.util import logger, get_current_platform, extract_from_dmg
+from condprof.customization import get_customizations, find_customization
+from condprof.client import read_changelog, ProfileNotFoundError
+
+
+class Runner:
+ def __init__(
+ self,
+ profile,
+ firefox,
+ geckodriver,
+ archive,
+ device_name,
+ strict,
+ force_new,
+ visible,
+ skip_logs=False,
+ remote_test_root="/sdcard/test_root/",
+ ):
+ self.force_new = force_new
+ self.profile = profile
+ self.geckodriver = geckodriver
+ self.archive = archive
+ self.device_name = device_name
+ self.strict = strict
+ self.visible = visible
+ self.skip_logs = skip_logs
+ self.remote_test_root = remote_test_root
+ self.env = {}
+ # unpacking a dmg
+ # XXX do something similar if we get an apk (but later)
+ # XXX we want to do
+ # adb install -r target.apk
+ # and get the installed app name
+ if firefox is not None and firefox.endswith("dmg"):
+ target = os.path.join(os.path.dirname(firefox), "firefox.app")
+ extract_from_dmg(firefox, target)
+ firefox = os.path.join(target, "Contents", "MacOS", "firefox")
+ self.firefox = firefox
+ self.android = self.firefox is not None and self.firefox.startswith(
+ "org.mozilla"
+ )
+
+ def prepare(self, scenario, customization):
+ self.scenario = scenario
+ self.customization = customization
+
+ # early checks to avoid extra work
+ if self.customization != "all":
+ if find_customization(self.customization) is None:
+ raise IOError("Cannot find customization %r" % self.customization)
+
+ if self.scenario != "all" and self.scenario not in scenarii:
+ raise IOError("Cannot find scenario %r" % self.scenario)
+
+ if not self.android and self.firefox is not None:
+ logger.info("Verifying Desktop Firefox binary")
+ # we want to verify we do have a firefox binary
+ # XXX so lame
+ if not os.path.exists(self.firefox):
+ if "MOZ_FETCHES_DIR" in os.environ:
+ target = os.path.join(os.environ["MOZ_FETCHES_DIR"], self.firefox)
+ if os.path.exists(target):
+ self.firefox = target
+
+ if not os.path.exists(self.firefox):
+ raise IOError("Cannot find %s" % self.firefox)
+
+ mozversion.get_version(self.firefox)
+
+ logger.info(os.environ)
+ if self.archive:
+ self.archive = os.path.abspath(self.archive)
+ logger.info("Archives directory is %s" % self.archive)
+ if not os.path.exists(self.archive):
+ os.makedirs(self.archive, exist_ok=True)
+
+ if shutil.which(self.geckodriver) is None and not os.path.exists(
+ self.geckodriver
+ ):
+ raise IOError("Cannot find %s" % self.geckodriver)
+
+ if not self.skip_logs:
+ try:
+ if self.android:
+ plat = "%s-%s" % (
+ self.device_name,
+ self.firefox.split("org.mozilla.")[-1],
+ )
+ else:
+ plat = get_current_platform()
+ self.changelog = read_changelog(plat)
+ logger.info("Got the changelog from TaskCluster")
+ except ProfileNotFoundError:
+ logger.info("changelog not found on TaskCluster, creating a local one.")
+ self.changelog = Changelog(self.archive)
+ else:
+ self.changelog = []
+
+ def _create_env(self):
+ if self.android:
+ klass = AndroidEnv
+ else:
+ klass = DesktopEnv
+
+ return klass(
+ self.profile, self.firefox, self.geckodriver, self.archive, self.device_name
+ )
+
+ def display_error(self, scenario, customization):
+ logger.error("%s x %s failed." % (scenario, customization), exc_info=True)
+ # TODO: this might avoid the exceptions that slip through in automation
+ # if self.strict:
+ # raise
+
+ async def one_run(self, scenario, customization):
+ """Runs one single conditioned profile.
+
+ Create an instance of the environment and run the ProfileCreator.
+ """
+ self.env = self._create_env()
+ return await ProfileCreator(
+ scenario,
+ customization,
+ self.archive,
+ self.changelog,
+ self.force_new,
+ self.env,
+ skip_logs=self.skip_logs,
+ remote_test_root=self.remote_test_root,
+ ).run(not self.visible)
+
+ async def run_all(self):
+ """Runs the conditioned profile builders"""
+ if self.scenario != "all":
+ selected_scenario = [self.scenario]
+ else:
+ selected_scenario = scenarii.keys()
+
+ # this is the loop that generates all combinations of profile
+ # for the current platform when "all" is selected
+ res = []
+ failures = 0
+ for scenario in selected_scenario:
+ if self.customization != "all":
+ try:
+ res.append(await self.one_run(scenario, self.customization))
+ except Exception:
+ failures += 1
+ self.display_error(scenario, self.customization)
+ else:
+ for customization in get_customizations():
+ logger.info("Customization %s" % customization)
+ try:
+ res.append(await self.one_run(scenario, customization))
+ except Exception:
+ failures += 1
+ self.display_error(scenario, customization)
+
+ return failures, [one_res for one_res in res if one_res]
+
+ def save(self):
+ self.changelog.save(self.archive)
+
+
+def run(
+ archive,
+ firefox=None,
+ scenario="all",
+ profile=None,
+ customization="all",
+ visible=False,
+ archives_dir="/tmp/archives",
+ force_new=False,
+ strict=True,
+ geckodriver="geckodriver",
+ device_name=None,
+):
+ runner = Runner(
+ profile, firefox, geckodriver, archive, device_name, strict, force_new, visible
+ )
+
+ runner.prepare(scenario, customization)
+ loop = asyncio.get_event_loop()
+
+ try:
+ failures, results = loop.run_until_complete(runner.run_all())
+ logger.info("Saving changelog in %s" % archive)
+ runner.save()
+ if failures > 0:
+ raise Exception("At least one scenario failed")
+ except Exception as e:
+ raise e
+ finally:
+ loop.close()
diff --git a/testing/condprofile/condprof/scenarii/__init__.py b/testing/condprofile/condprof/scenarii/__init__.py
new file mode 100644
index 0000000000..b665843b2b
--- /dev/null
+++ b/testing/condprofile/condprof/scenarii/__init__.py
@@ -0,0 +1,15 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from condprof.scenarii.full import full
+from condprof.scenarii.settled import settled
+from condprof.scenarii.settled2 import settled2
+from condprof.scenarii.settled_youtube import settled_youtube
+
+scenarii = {
+ "full": full,
+ "settled": settled,
+ "settled-youtube": settled_youtube,
+ "settled2": settled2,
+}
diff --git a/testing/condprofile/condprof/scenarii/bookmark.js b/testing/condprofile/condprof/scenarii/bookmark.js
new file mode 100644
index 0000000000..70ef028320
--- /dev/null
+++ b/testing/condprofile/condprof/scenarii/bookmark.js
@@ -0,0 +1,52 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+// 'arguments' is defined by the marionette harness.
+/* global arguments */
+
+const { PlacesUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/PlacesUtils.sys.mjs"
+);
+
+let resolve = arguments[3];
+
+try {
+ // Get the number of current bookmarks
+ PlacesUtils.promiseBookmarksTree(PlacesUtils.bookmarks.unfiledGuid, {
+ includeItemIds: true,
+ }).then(root => {
+ let count = root.itemsCount;
+ let maxBookmarks = arguments[2];
+ // making sure we don't exceed the maximum number of bookmarks
+ if (count >= maxBookmarks) {
+ let toRemove = count - maxBookmarks + 1;
+ console.log("We've reached the maximum number of bookmarks");
+ console.log("Removing " + toRemove);
+ let children = root.children;
+ for (let i = 0, p = Promise.resolve(); i < toRemove; i++) {
+ p = p.then(
+ _ =>
+ new Promise(resolve =>
+ PlacesUtils.bookmarks.remove(children[i].guid).then(res => {
+ console.log("removed one bookmark");
+ resolve(res);
+ })
+ )
+ );
+ }
+ }
+ // now adding the bookmark
+ PlacesUtils.bookmarks
+ .insert({
+ parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+ url: arguments[0],
+ title: arguments[1],
+ })
+ .then(res => {
+ resolve(res);
+ });
+ });
+} catch (error) {
+ let res = { logs: {}, result: 1, result_message: error.toString() };
+ resolve(res);
+}
diff --git a/testing/condprofile/condprof/scenarii/full.py b/testing/condprofile/condprof/scenarii/full.py
new file mode 100644
index 0000000000..ec93b48b46
--- /dev/null
+++ b/testing/condprofile/condprof/scenarii/full.py
@@ -0,0 +1,138 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import asyncio
+import os
+import random
+
+from arsenic.errors import UnknownArsenicError, UnknownError
+
+from condprof.helpers import TabSwitcher, execute_async_script, is_mobile
+from condprof.util import get_credentials, logger
+
+BOOKMARK_FREQUENCY = 5
+MAX_URLS = 150
+MAX_BOOKMARKS = 250
+CallErrors = asyncio.TimeoutError, UnknownError, UnknownArsenicError
+
+
+class Builder:
+ def __init__(self, options):
+ self.options = options
+ self.words = self._read_lines("words.txt")
+ self.urls = self._build_url_list(self._read_lines("urls.txt"))
+ self.sync_js = self._load_script("sync")
+ self.max_bookmarks = options.get("max_bookmarks", MAX_BOOKMARKS)
+ self.bookmark_js = self._load_script("bookmark")
+ self.platform = options.get("platform", "")
+ self.mobile = is_mobile(self.platform)
+ self.max_urls = options.get("max_urls", MAX_URLS)
+
+ # see Bug 1608604 & see Bug 1619107 - we have stability issues @ bitbar
+ if self.mobile:
+ self.max_urls = min(self.max_urls, 20)
+
+ logger.info("platform: %s" % self.platform)
+ logger.info("max_urls: %s" % self.max_urls)
+ self.bookmark_frequency = options.get("bookmark_frequency", BOOKMARK_FREQUENCY)
+
+ # we're syncing only on desktop for now
+ self.syncing = not self.mobile
+ if self.syncing:
+ self.username, self.password = get_credentials()
+ if self.username is None:
+ raise ValueError("Sync operations need an FxA username and password")
+ else:
+ self.username, self.password = None, None
+
+ def _load_script(self, name):
+ return "\n".join(self._read_lines("%s.js" % name))
+
+ def _read_lines(self, filename):
+ path = os.path.join(os.path.dirname(__file__), filename)
+ with open(path) as f:
+ return f.readlines()
+
+ def _build_url_list(self, urls):
+ url_list = []
+ for url in urls:
+ url = url.strip()
+ if url.startswith("#"):
+ continue
+ for word in self.words:
+ word = word.strip()
+ if word.startswith("#"):
+ continue
+ url_list.append((url.format(word), word))
+ random.shuffle(url_list)
+ return url_list
+
+ async def add_bookmark(self, session, url, title):
+ logger.info("Adding bookmark to %s" % url)
+ return await execute_async_script(
+ session, self.bookmark_js, url, title, self.max_bookmarks
+ )
+
+ async def sync(self, session, metadata):
+ if not self.syncing:
+ return
+ # now that we've visited all pages, we want to upload to FXSync
+ logger.info("Syncing profile to FxSync")
+ logger.info("Username is %s, password is %s" % (self.username, self.password))
+ script_res = await execute_async_script(
+ session,
+ self.sync_js,
+ self.username,
+ self.password,
+ "https://accounts.stage.mozaws.net",
+ )
+ if script_res is None:
+ script_res = {}
+ metadata["logs"] = script_res.get("logs", {})
+ metadata["result"] = script_res.get("result", 0)
+ metadata["result_message"] = script_res.get("result_message", "SUCCESS")
+ return metadata
+
+ async def _visit_url(self, current, session, url, word):
+ await asyncio.wait_for(session.get(url), 5)
+ if current % self.bookmark_frequency == 0 and not self.mobile:
+ await asyncio.wait_for(self.add_bookmark(session, url, word), 5)
+
+ async def __call__(self, session):
+ metadata = {}
+
+ tabs = TabSwitcher(session, self.options)
+ await tabs.create_windows()
+ visited = 0
+
+ for current, (url, word) in enumerate(self.urls):
+ logger.info("%d/%d %s" % (current + 1, self.max_urls, url))
+ retries = 0
+ while retries < 3:
+ try:
+ await self._visit_url(current, session, url, word)
+ visited += 1
+ break
+ except CallErrors:
+ await asyncio.sleep(1.0)
+ retries += 1
+
+ if current == self.max_urls - 1:
+ break
+
+ # switch to the next tab
+ try:
+ await asyncio.wait_for(tabs.switch(), 5)
+ except CallErrors:
+ # if we can't switch, it's ok
+ pass
+
+ metadata["visited_url"] = visited
+ await self.sync(session, metadata)
+ return metadata
+
+
+async def full(session, options):
+ builder = Builder(options)
+ return await builder(session)
diff --git a/testing/condprofile/condprof/scenarii/settled.py b/testing/condprofile/condprof/scenarii/settled.py
new file mode 100644
index 0000000000..eb54f4638b
--- /dev/null
+++ b/testing/condprofile/condprof/scenarii/settled.py
@@ -0,0 +1,11 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import asyncio
+
+
+async def settled(session, options):
+ # nothing is done, we just settle here for 30 seconds
+ await asyncio.sleep(options.get("sleep", 30))
+ return {}
diff --git a/testing/condprofile/condprof/scenarii/settled2.py b/testing/condprofile/condprof/scenarii/settled2.py
new file mode 100644
index 0000000000..0bc828c4c3
--- /dev/null
+++ b/testing/condprofile/condprof/scenarii/settled2.py
@@ -0,0 +1,19 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import asyncio
+
+
+async def settled2(session, options):
+ retries = 0
+ while retries < 3:
+ try:
+ await asyncio.wait_for(session.get("https://www.mozilla.org/en-US/"), 5)
+ break
+ except asyncio.TimeoutError:
+ retries += 1
+
+ # we just settle here for 5 minutes
+ await asyncio.sleep(options.get("sleep", 5 * 60))
+ return {}
diff --git a/testing/condprofile/condprof/scenarii/settled_youtube.py b/testing/condprofile/condprof/scenarii/settled_youtube.py
new file mode 100644
index 0000000000..b39c0874a9
--- /dev/null
+++ b/testing/condprofile/condprof/scenarii/settled_youtube.py
@@ -0,0 +1,20 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import asyncio
+
+
+async def settled_youtube(session, options):
+ # nothing is done, we just settle here for 30 seconds
+ await asyncio.sleep(options.get("sleep", 5))
+ await asyncio.wait_for(
+ session.get(
+ "https://yttest.prod.mozaws.net/2020/main.html?"
+ "test_type=playbackperf-widevine-sfr-h264-test&"
+ "raptor=true&exclude=1,2&muted=true&command=run"
+ ),
+ timeout=120,
+ )
+ await asyncio.sleep(options.get("sleep", 30))
+ return {}
diff --git a/testing/condprofile/condprof/scenarii/sync.js b/testing/condprofile/condprof/scenarii/sync.js
new file mode 100644
index 0000000000..0f7b7bb90d
--- /dev/null
+++ b/testing/condprofile/condprof/scenarii/sync.js
@@ -0,0 +1,21 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const { triggerSync } = ChromeUtils.importESModule(
+ "resource://gre/modules/services-automation/ServicesAutomation.sys.mjs"
+);
+
+// Arguments is expected to be provided in the scope that this file is loaded
+// into.
+/* global arguments */
+
+let resolve = arguments[3];
+try {
+ triggerSync(arguments[0], arguments[1], arguments[2]).then(res => {
+ resolve(res);
+ });
+} catch (error) {
+ let res = { logs: {}, result: 1, result_message: error.toString() };
+ resolve(res);
+}
diff --git a/testing/condprofile/condprof/scenarii/urls.txt b/testing/condprofile/condprof/scenarii/urls.txt
new file mode 100644
index 0000000000..5dd86b024a
--- /dev/null
+++ b/testing/condprofile/condprof/scenarii/urls.txt
@@ -0,0 +1,7 @@
+# urls combined with the words list
+https://www.google.com/search?q={0}
+https://search.yahoo.com/yhs/search?p={0}
+https://www.bing.com/search?q=create+list+of+nouns{0}
+https://www.amazon.com/s/ref=nb_sb_noss_2?url=search-alias%3Daps&field-keywords={0}
+https://www.youtube.com/results?search_query={0}
+https://www.ebay.com/sch/i.html?_from=R40&_trksid=p2380057.m570.l1313.TR0.TRC0.H0.Xbottle.TRS0&_nkw={0}
diff --git a/testing/condprofile/condprof/scenarii/words.txt b/testing/condprofile/condprof/scenarii/words.txt
new file mode 100644
index 0000000000..107f65180e
--- /dev/null
+++ b/testing/condprofile/condprof/scenarii/words.txt
@@ -0,0 +1,4555 @@
+# word list from http://www.worldclasslearning.com/english/4000-most-common-english-words.html
+aardvark
+abacus
+abbey
+abdomen
+ability
+abolishment
+abroad
+abuse
+accelerant
+accelerator
+access
+accident
+accommodation
+accompanist
+accordion
+account
+accountant
+achiever
+acid
+acknowledgment
+acoustic
+acoustics
+acrylic
+act
+action
+activity
+actor
+actress
+acupuncture
+ad
+adapter
+addiction
+addition
+address
+adjustment
+administration
+adrenalin
+adult
+adulthood
+advance
+advancement
+advantage
+advertisement
+advertising
+advice
+affair
+affect
+aftermath
+afternoon
+aftershave
+aftershock
+afterthought
+age
+agency
+agenda
+agent
+aggression
+aglet
+agreement
+aid
+air
+airbag
+airbus
+airfare
+airforce
+airline
+airmail
+airplane
+airport
+airship
+alarm
+alb
+albatross
+alcohol
+alcove
+alder
+algebra
+alibi
+allergist
+alley
+alligator
+alloy
+almanac
+almond
+alpaca
+alpenglow
+alpenhorn
+alpha
+alphabet
+alternative
+altitude
+alto
+aluminium
+aluminum
+ambassador
+ambition
+ambulance
+amendment
+amount
+amusement
+anagram
+analgesia
+analog
+analogue
+analogy
+analysis
+analyst
+anatomy
+anesthesiology
+anethesiologist
+anger
+angiosperm
+angle
+angora
+angstrom
+anguish
+animal
+anime
+ankle
+anklet
+annual
+anorak
+answer
+ant
+anteater
+antechamber
+antelope
+anthony
+anthropology
+antler
+anxiety
+anybody
+anything
+anywhere
+apartment
+ape
+aperitif
+apology
+apparatus
+apparel
+appeal
+appearance
+appendix
+applause
+apple
+applewood
+appliance
+application
+appointment
+approval
+apron
+apse
+aquifer
+arch
+archaeology
+archeology
+archer
+architect
+architecture
+arch-rival
+area
+argument
+arithmetic
+arm
+armadillo
+armament
+armchair
+armoire
+armor
+arm-rest
+army
+arrival
+arrow
+art
+artichoke
+article
+artificer
+ascot
+ash
+ashram
+ashtray
+aside
+ask
+asparagus
+aspect
+asphalt
+assignment
+assist
+assistance
+assistant
+associate
+association
+assumption
+asterisk
+astrakhan
+astrolabe
+astrologer
+astrology
+astronomy
+atelier
+athlete
+athletics
+atmosphere
+atom
+atrium
+attachment
+attack
+attempt
+attendant
+attention
+attenuation
+attic
+attitude
+attorney
+attraction
+audience
+auditorium
+aunt
+author
+authorisation
+authority
+authorization
+automaton
+avalanche
+avenue
+average
+award
+awareness
+azimuth
+babe
+baboon
+babushka
+baby
+back
+backbone
+backdrop
+background
+backpack
+bacon
+bad
+badge
+badger
+bafflement
+bag
+bagel
+baggage
+bagpipe
+bail
+bait
+bake
+baker
+bakery
+bakeware
+balaclava
+balalaika
+balance
+balcony
+ball
+ballet
+balloon
+ballpark
+bamboo
+banana
+band
+bandana
+bandanna
+bandolier
+bangle
+banjo
+bank
+bankbook
+banker
+banquette
+baobab
+bar
+barbeque
+barber
+barbiturate
+barge
+baritone
+barium
+barn
+barometer
+barracks
+barstool
+base
+baseball
+basement
+basin
+basis
+basket
+basketball
+bass
+bassinet
+bassoon
+bat
+bath
+bather
+bathhouse
+bathrobe
+bathroom
+bathtub
+batter
+battery
+batting
+battle
+battleship
+bay
+bayou
+beach
+bead
+beak
+beam
+bean
+beanie
+beanstalk
+bear
+beard
+beast
+beat
+beautiful
+beauty
+beaver
+bed
+bedroom
+bee
+beech
+beef
+beer
+beet
+beetle
+beggar
+beginner
+beginning
+begonia
+behavior
+beheading
+behest
+being
+belfry
+belief
+believe
+bell
+belligerency
+bellows
+belly
+belt
+bench
+bend
+beneficiary
+benefit
+bengal
+beret
+berry
+bestseller
+best-seller
+bet
+beverage
+beyond
+bibliography
+bicycle
+bid
+bidet
+bifocals
+big
+big-rig
+bijou
+bike
+bikini
+bill
+billboard
+bin
+biology
+biplane
+birch
+bird
+birdbath
+birdcage
+birdhouse
+bird-watcher
+birth
+birthday
+bit
+bite
+bitter
+black
+blackberry
+blackboard
+blackfish
+bladder
+blade
+blame
+blank
+blanket
+blazer
+blight
+blind
+blinker
+blister
+blizzard
+block
+blocker
+blood
+bloodflow
+bloom
+bloomers
+blossom
+blouse
+blow
+blowgun
+blowhole
+blue
+blueberry
+boar
+board
+boat
+boat-building
+boatload
+boatyard
+bobcat
+body
+bog
+bolero
+bolt
+bomb
+bomber
+bondsman
+bone
+bongo
+bonnet
+bonsai
+bonus
+boogeyman
+book
+bookcase
+bookend
+booklet
+booster
+boot
+bootee
+bootie
+boots
+booty
+border
+bore
+bosom
+boss
+botany
+bother
+bottle
+bottling
+bottom
+bottom-line
+boudoir
+bough
+boundary
+bow
+bower
+bowl
+bowler
+bowling
+bowtie
+box
+boxer
+boxspring
+boy
+boyfriend
+bra
+brace
+bracelet
+bracket
+brain
+brake
+branch
+brand
+brandy
+brass
+brassiere
+bratwurst
+brave
+bread
+breadcrumb
+break
+breakfast
+breakpoint
+breast
+breastplate
+breath
+breeze
+bribery
+brick
+bricklaying
+bridge
+brief
+briefs
+brilliant
+british
+broad
+broccoli
+brochure
+broiler
+broker
+brome
+bronchitis
+bronco
+bronze
+brooch
+brood
+brook
+broom
+brother
+brother-in-law
+brow
+brown
+brush
+brushfire
+brushing
+bubble
+bucket
+buckle
+bud
+buddy
+budget
+buffer
+buffet
+bug
+buggy
+bugle
+building
+bulb
+bull
+bulldozer
+bullet
+bull-fighter
+bumper
+bun
+bunch
+bungalow
+bunghole
+bunkhouse
+burglar
+burlesque
+burn
+burn-out
+burst
+bus
+bush
+business
+bust
+bustle
+butane
+butcher
+butter
+button
+buy
+buyer
+buzzard
+cabana
+cabbage
+cabin
+cabinet
+cable
+caboose
+cacao
+cactus
+caddy
+cadet
+cafe
+caftan
+cake
+calcification
+calculation
+calculator
+calculus
+calendar
+calf
+calico
+call
+calm
+camel
+cameo
+camera
+camp
+campaign
+campanile
+can
+canal
+cancel
+cancer
+candelabra
+candidate
+candle
+candy
+cane
+cannon
+canoe
+canon
+canopy
+canteen
+canvas
+cap
+cape
+capital
+capitulation
+capon
+cappelletti
+cappuccino
+captain
+caption
+car
+caravan
+carbon
+card
+cardboard
+cardigan
+care
+career
+cargo
+carload
+carnation
+carol
+carotene
+carp
+carpenter
+carpet
+carport
+carriage
+carrier
+carrot
+carry
+cart
+cartilage
+cartload
+cartoon
+cartridge
+cascade
+case
+casement
+cash
+cashier
+casino
+casserole
+cassock
+cast
+castanet
+castanets
+castle
+cat
+catacomb
+catamaran
+catch
+category
+caterpillar
+cathedral
+catsup
+cattle
+cauliflower
+cause
+caution
+cave
+c-clamp
+cd
+ceiling
+celebration
+celeriac
+celery
+celeste
+cell
+cellar
+cello
+celsius
+cement
+cemetery
+cenotaph
+census
+cent
+center
+centimeter
+centurion
+century
+cephalopod
+ceramic
+cereal
+certification
+cesspool
+chafe
+chain
+chainstay
+chair
+chairlift
+chairman
+chairperson
+chaise
+chalet
+chalice
+chalk
+challenge
+champion
+championship
+chance
+chandelier
+change
+channel
+chaos
+chap
+chapel
+chapter
+character
+chard
+charge
+charity
+charlatan
+charles
+charm
+chart
+chastity
+chasuble
+chateau
+chauffeur
+chauvinist
+check
+checkroom
+cheek
+cheetah
+chef
+chemical
+chemistry
+cheque
+cherries
+cherry
+chess
+chest
+chick
+chicken
+chicory
+chief
+chiffonier
+child
+childhood
+children
+chill
+chime
+chimpanzee
+chin
+chino
+chip
+chipmunk
+chit-chat
+chivalry
+chive
+chocolate
+choice
+choker
+chop
+chopstick
+chord
+chowder
+chrome
+chromolithograph
+chronograph
+chronometer
+chub
+chug
+church
+churn
+cicada
+cigarette
+cinema
+circle
+circulation
+circumference
+cirrus
+citizenship
+city
+civilisation
+claim
+clam
+clank
+clapboard
+clarinet
+clasp
+class
+classic
+classroom
+clause
+clave
+clavicle
+clavier
+cleaner
+cleat
+cleavage
+clef
+cleric
+clerk
+click
+client
+cliff
+climate
+climb
+clip
+clipper
+cloak
+cloakroom
+clock
+clockwork
+clogs
+cloister
+close
+closet
+cloth
+clothes
+clothing
+cloud
+cloudburst
+cloudy
+clove
+clover
+club
+clue
+clutch
+coach
+coal
+coast
+coat
+cob
+cobweb
+cockpit
+cockroach
+cocktail
+cocoa
+cod
+code
+codon
+codpiece
+coevolution
+coffee
+coffin
+coil
+coin
+coinsurance
+coke
+cold
+coliseum
+collar
+collection
+college
+collision
+colloquia
+colon
+colonisation
+colony
+color
+colt
+column
+columnist
+comb
+combat
+combination
+combine
+comfort
+comfortable
+comic
+comma
+command
+comment
+commerce
+commercial
+commission
+committee
+common
+communicant
+communication
+community
+company
+comparison
+compassion
+competition
+competitor
+complaint
+complement
+complex
+component
+comportment
+composer
+composition
+compost
+comprehension
+compulsion
+computer
+comradeship
+concentrate
+concept
+concern
+concert
+conclusion
+concrete
+condition
+condominium
+condor
+conductor
+cone
+confectionery
+conference
+confidence
+confirmation
+conflict
+confusion
+conga
+congo
+congress
+congressman
+congressperson
+conifer
+connection
+consent
+consequence
+consideration
+consist
+console
+consonant
+conspirator
+constant
+constellation
+construction
+consul
+consulate
+contact
+contact lens
+contagion
+content
+contest
+context
+continent
+contract
+contrail
+contrary
+contribution
+control
+convection
+conversation
+convert
+convertible
+cook
+cookie
+cooking
+coonskin
+cope
+cop-out
+copper
+co-producer
+copy
+copyright
+copywriter
+cord
+corduroy
+cork
+cormorant
+corn
+corner
+cornerstone
+cornet
+corral
+correspondent
+corridor
+corruption
+corsage
+cost
+costume
+cot
+cottage
+cotton
+couch
+cougar
+cough
+council
+councilman
+councilor
+councilperson
+count
+counter
+counter-force
+countess
+country
+county
+couple
+courage
+course
+court
+cousin
+covariate
+cover
+coverall
+cow
+cowbell
+cowboy
+crab
+crack
+cracker
+crackers
+cradle
+craft
+craftsman
+crash
+crate
+cravat
+craw
+crawdad
+crayfish
+crayon
+crazy
+cream
+creative
+creator
+creature
+creche
+credenza
+credit
+creditor
+creek
+creme brulee
+crest
+crew
+crib
+cribbage
+cricket
+cricketer
+crime
+criminal
+crinoline
+criteria
+criterion
+criticism
+crocodile
+crocus
+croissant
+crook
+crop
+cross
+cross-contamination
+cross-stitch
+crotch
+croup
+crow
+crowd
+crown
+crude
+crush
+cry
+crystallography
+cub
+cuckoo
+cucumber
+cuff-links
+cultivar
+cultivator
+culture
+culvert
+cummerbund
+cup
+cupboard
+cupcake
+cupola
+curio
+curl
+curler
+currency
+current
+cursor
+curtain
+curve
+cushion
+custard
+customer
+cut
+cuticle
+cutlet
+cutover
+cutting
+cyclamen
+cycle
+cyclone
+cylinder
+cymbal
+cymbals
+cynic
+cyst
+cytoplasm
+dad
+daffodil
+dagger
+dahlia
+daisy
+damage
+dame
+dance
+dancer
+dancing
+danger
+daniel
+dare
+dark
+dart
+dash
+dashboard
+data
+database
+date
+daughter
+david
+day
+daybed
+dead
+deadline
+deal
+dealer
+dear
+death
+deathwatch
+debate
+debt
+debtor
+decade
+decimal
+decision
+deck
+declination
+decongestant
+decrease
+decryption
+dedication
+deep
+deer
+defense
+deficit
+definition
+deformation
+degree
+delay
+delete
+delight
+delivery
+demand
+demur
+den
+denim
+dentist
+deodorant
+department
+departure
+dependent
+deployment
+deposit
+depression
+depressive
+depth
+deputy
+derby
+derrick
+description
+desert
+design
+designer
+desire
+desk
+dessert
+destiny
+destroyer
+destruction
+detail
+detainment
+detective
+detention
+determination
+development
+deviance
+device
+devil
+dew
+dhow
+diadem
+diamond
+diaphragm
+diarist
+dibble
+dickey
+dictaphone
+diction
+dictionary
+diet
+difference
+differential
+difficulty
+dig
+digestion
+digger
+digital
+dignity
+dilapidation
+dill
+dime
+dimension
+dimple
+diner
+dinghy
+dinner
+dinosaur
+diploma
+dipstick
+direction
+director
+dirndl
+dirt
+disadvantage
+disarmament
+disaster
+discipline
+disco
+disconnection
+discount
+discovery
+discrepancy
+discussion
+disease
+disembodiment
+disengagement
+disguise
+disgust
+dish
+dishes
+dishwasher
+disk
+display
+disposer
+distance
+distribution
+distributor
+district
+divan
+diver
+divide
+divider
+diving
+division
+dock
+doctor
+document
+doe
+dog
+dogsled
+dogwood
+doll
+dollar
+dolman
+dolphin
+domain
+donkey
+door
+doorknob
+doorpost
+dory
+dot
+double
+doubling
+doubt
+doubter
+downforce
+downgrade
+downtown
+draft
+drag
+dragon
+dragonfly
+dragster
+drain
+drake
+drama
+dramaturge
+draw
+drawbridge
+drawer
+drawing
+dream
+dredger
+dress
+dresser
+dressing
+drill
+drink
+drive
+driver
+driveway
+driving
+drizzle
+dromedary
+drop
+drug
+drum
+drummer
+drunk
+dry
+dryer
+duck
+duckling
+dud
+due
+duffel
+dugout
+dulcimer
+dumbwaiter
+dump
+dump truck
+dune buggy
+dungarees
+dungeon
+duplexer
+dust
+dust storm
+duster
+duty
+dwarf
+dwelling
+dynamo
+eagle
+ear
+eardrum
+earmuffs
+earplug
+earrings
+earth
+earthquake
+earthworm
+ease
+easel
+east
+eat
+eave
+eavesdropper
+e-book
+ecclesia
+eclipse
+ecliptic
+economics
+economy
+ecumenist
+eddy
+edge
+edger
+editor
+editorial
+education
+edward
+eel
+effacement
+effect
+effective
+efficacy
+efficiency
+effort
+egg
+egghead
+eggnog
+eggplant
+eight
+ejector
+elbow
+election
+electricity
+electrocardiogram
+element
+elephant
+elevator
+elixir
+elk
+ellipse
+elm
+elongation
+embossing
+emergence
+emergency
+emergent
+emery
+emotion
+emphasis
+employ
+employee
+employer
+employment
+empowerment
+emu
+encirclement
+encyclopedia
+end
+endothelium
+enemy
+energy
+engine
+engineer
+engineering
+enigma
+enjoyment
+enquiry
+entertainment
+enthusiasm
+entrance
+entry
+environment
+envy
+epauliere
+epee
+ephemera
+ephemeris
+epoch
+eponym
+epoxy
+equal
+equinox
+equipment
+equivalent
+era
+e-reader
+error
+escape
+ese
+espadrille
+espalier
+essay
+establishment
+estate
+estimate
+estrogen
+estuary
+ethernet
+ethics
+euphonium
+eurocentrism
+europe
+evaluator
+evening
+evening-wear
+event
+eviction
+evidence
+evocation
+evolution
+exam
+examination
+examiner
+example
+exchange
+excitement
+exclamation
+excuse
+executor
+exercise
+exhaust
+ex-husband
+exile
+existence
+exit
+expansion
+expansionism
+experience
+expert
+explanation
+exposition
+expression
+extension
+extent
+external
+extreme
+ex-wife
+eye
+eyeball
+eyebrow
+eyebrows
+eyeglasses
+eyelash
+eyelashes
+eyelid
+eyelids
+eyeliner
+eyestrain
+face
+facelift
+facet
+facilities
+facsimile
+fact
+factor
+factory
+faculty
+fahrenheit
+fail
+failure
+fairies
+fairy
+faith
+fall
+falling-out
+fame
+familiar
+family
+fan
+fang
+fanlight
+fanny
+fanny-pack
+farm
+farmer
+fascia
+fat
+father
+father-in-law
+fatigues
+faucet
+fault
+fawn
+fax
+fear
+feast
+feather
+feature
+fedelini
+fedora
+fee
+feed
+feedback
+feel
+feeling
+feet
+felony
+female
+fen
+fence
+fencing
+fender
+ferry
+ferryboat
+fertilizer
+few
+fiber
+fiberglass
+fibre
+fiction
+fiddle
+field
+fifth
+fight
+fighter
+figure
+figurine
+file
+fill
+filly
+film
+filth
+final
+finance
+find
+finding
+fine
+finger
+fingernail
+finish
+finisher
+fir
+fire
+fireman
+fireplace
+firewall
+fish
+fishbone
+fisherman
+fishery
+fishing
+fishmonger
+fishnet
+fisting
+fix
+fixture
+flag
+flame
+flanker
+flare
+flash
+flat
+flatboat
+flavor
+flax
+fleck
+fleece
+flesh
+flight
+flintlock
+flip-flops
+flock
+flood
+floor
+floozie
+flour
+flow
+flower
+flu
+flugelhorn
+fluke
+flute
+fly
+flytrap
+foam
+fob
+focus
+fog
+fold
+folder
+following
+fondue
+font
+food
+foot
+football
+footnote
+footrest
+foot-rest
+footstool
+foray
+force
+forearm
+forebear
+forecast
+forehead
+forest
+forestry
+forever
+forgery
+fork
+form
+formal
+format
+former
+fort
+fortnight
+fortress
+fortune
+forum
+foundation
+fountain
+fowl
+fox
+foxglove
+fragrance
+frame
+fratricide
+fraudster
+frazzle
+freckle
+freedom
+freeplay
+freeze
+freezer
+freight
+freighter
+freon
+fresco
+friction
+fridge
+friend
+friendship
+frigate
+fringe
+frock
+frog
+front
+frost
+frown
+fruit
+frustration
+fuel
+fulfillment
+full
+fun
+function
+fundraising
+funeral
+funny
+fur
+furnace
+furniture
+fusarium
+futon
+future
+gaffer
+gain
+gaiters
+gale
+gall-bladder
+gallery
+galley
+gallon
+galn
+galoshes
+game
+gamebird
+gamma-ray
+gander
+gap
+garage
+garb
+garbage
+garden
+garlic
+garment
+garter
+gas
+gasoline
+gastropod
+gate
+gateway
+gather
+gauge
+gauntlet
+gazebo
+gazelle
+gear
+gearshift
+geese
+gelding
+gem
+gemsbok
+gender
+gene
+general
+genetics
+geography
+geology
+geometry
+george
+geranium
+gerbil
+geyser
+gherkin
+ghost
+giant
+gift
+gigantism
+ginseng
+giraffe
+girdle
+girl
+girlfriend
+git
+give
+glad
+gladiolus
+gland
+glass
+glasses
+glen
+glider
+gliding
+glockenspiel
+glove
+gloves
+glue
+glut
+go
+goal
+goat
+gobbler
+god
+godmother
+goggles
+go-kart
+gold
+goldfish
+golf
+gondola
+gong
+good
+goodbye
+good-bye
+goodie
+goose
+gopher
+gore-tex
+gorilla
+gosling
+gossip
+governance
+government
+governor
+gown
+grab
+grab-bag
+grade
+grain
+gram
+grammar
+grand
+granddaughter
+grandfather
+grandmom
+grandmother
+grandson
+granny
+grape
+grapefruit
+graph
+graphic
+grass
+grasshopper
+grassland
+gratitude
+gray
+grease
+great
+great-grandfather
+great-grandmother
+greek
+green
+greenhouse
+grenade
+grey
+grief
+grill
+grip
+grit
+grocery
+ground
+group
+grouper
+grouse
+growth
+guarantee
+guard
+guess
+guest
+guestbook
+guidance
+guide
+guilt
+guilty
+guitar
+guitarist
+gum
+gumshoes
+gun
+gutter
+guy
+gym
+gymnast
+gymnastics
+gynaecology
+gyro
+habit
+hacienda
+hacksaw
+hackwork
+hail
+hair
+haircut
+half
+half-brother
+half-sister
+halibut
+hall
+hallway
+hamaki
+hamburger
+hammer
+hammock
+hamster
+hand
+handball
+hand-holding
+handicap
+handle
+handlebar
+handmaiden
+handsaw
+hang
+happiness
+harbor
+harbour
+hardboard
+hardcover
+hardening
+hardhat
+hard-hat
+hardware
+harm
+harmonica
+harmony
+harp
+harpooner
+harpsichord
+hassock
+hat
+hatbox
+hatchet
+hate
+hatred
+haunt
+haversack
+hawk
+hay
+head
+headlight
+headline
+headrest
+health
+hearing
+heart
+heartache
+hearth
+hearthside
+heart-throb
+heartwood
+heat
+heater
+heaven
+heavy
+hedge
+hedgehog
+heel
+height
+heirloom
+helen
+helicopter
+helium
+hell
+hellcat
+hello
+helmet
+helo
+help
+hemp
+hen
+herb
+heron
+herring
+hexagon
+heyday
+hide
+high
+highlight
+high-rise
+highway
+hill
+hip
+hippodrome
+hippopotamus
+hire
+history
+hit
+hive
+hobbies
+hobbit
+hobby
+hockey
+hoe
+hog
+hold
+hole
+holiday
+home
+homework
+homogenate
+homonym
+honesty
+honey
+honeybee
+honoree
+hood
+hoof
+hook
+hope
+hops
+horn
+hornet
+horror
+horse
+hose
+hosiery
+hospice
+hospital
+hospitality
+host
+hostel
+hostess
+hot
+hot-dog
+hotel
+hour
+hourglass
+house
+houseboat
+housework
+housing
+hovel
+hovercraft
+howitzer
+hub
+hubcap
+hugger
+human
+humidity
+humor
+humour
+hunger
+hunt
+hurdler
+hurricane
+hurry
+hurt
+husband
+hut
+hutch
+hyacinth
+hybridisation
+hydrant
+hydraulics
+hydrofoil
+hydrogen
+hyena
+hygienic
+hyphenation
+hypochondria
+hypothermia
+ice
+icebreaker
+icecream
+ice-cream
+icicle
+icon
+idea
+ideal
+if
+igloo
+ikebana
+illegal
+image
+imagination
+impact
+implement
+importance
+impress
+impression
+imprisonment
+improvement
+impudence
+impulse
+inbox
+incandescence
+inch
+incident
+income
+increase
+independence
+independent
+index
+indication
+indigence
+individual
+industry
+inevitable
+infancy
+inflammation
+inflation
+influence
+information
+infusion
+inglenook
+ingrate
+initial
+initiative
+in-joke
+injury
+injustice
+ink
+in-laws
+inlay
+inn
+innervation
+innocence
+innocent
+input
+inquiry
+inscription
+insect
+inside
+insolence
+inspection
+inspector
+instance
+instruction
+instrument
+instrumentalist
+instrumentation
+insulation
+insurance
+insurgence
+intelligence
+intention
+interaction
+interactive
+interest
+interferometer
+interior
+interloper
+internal
+international
+internet
+interpreter
+intervenor
+interview
+interviewer
+intestine
+intestines
+introduction
+invention
+inventor
+inventory
+investment
+invite
+invoice
+iridescence
+iris
+iron
+ironclad
+irony
+island
+issue
+it
+item
+jackal
+jacket
+jaguar
+jail
+jailhouse
+jam
+james
+jar
+jasmine
+jaw
+jealousy
+jeans
+jeep
+jeff
+jelly
+jellyfish
+jet
+jewel
+jewelry
+jiffy
+job
+jockey
+jodhpurs
+joey
+jogging
+join
+joint
+joke
+jot
+journey
+joy
+judge
+judgment
+judo
+juggernaut
+juice
+jumbo
+jump
+jumper
+jumpsuit
+junior
+junk
+junker
+junket
+jury
+justice
+jute
+kale
+kamikaze
+kangaroo
+karate
+karen
+kayak
+kazoo
+keep
+kendo
+ketch
+ketchup
+kettle
+kettledrum
+key
+keyboard
+keyboarding
+keystone
+kick
+kick-off
+kid
+kidney
+kidneys
+kielbasa
+kill
+kilogram
+kilometer
+kilt
+kimono
+kind
+kindness
+king
+kingfish
+kiosk
+kiss
+kitchen
+kite
+kitten
+kitty
+kleenex
+klomps
+knee
+kneejerk
+knickers
+knife
+knife-edge
+knight
+knitting
+knot
+knowledge
+knuckle
+koala
+kohlrabi
+lab
+laborer
+labour
+lace
+lack
+lacquerware
+ladder
+lady
+ladybug
+lake
+lamb
+lamp
+lan
+lanai
+land
+landform
+landmine
+landscape
+language
+lantern
+lap
+laparoscope
+lapdog
+laptop
+larch
+larder
+lark
+laryngitis
+lasagna
+latency
+latex
+lathe
+latte
+laugh
+laughter
+laundry
+lava
+law
+lawn
+lawsuit
+lawyer
+lay
+layer
+lead
+leader
+leadership
+leading
+leaf
+league
+leaker
+learning
+leash
+leather
+leave
+leaver
+lecture
+leek
+leg
+legal
+legging
+legume
+lei
+leisure
+lemon
+lemonade
+lemur
+length
+lentil
+leprosy
+lesson
+let
+letter
+lettuce
+level
+lever
+leverage
+license
+lie
+lier
+life
+lift
+light
+lighting
+lightning
+lilac
+lily
+limit
+limo
+line
+linen
+liner
+linguistics
+link
+linseed
+lion
+lip
+lipstick
+liquid
+liquor
+lisa
+list
+listen
+literature
+litigation
+litter
+liver
+livestock
+living
+lizard
+llama
+load
+loaf
+loafer
+loan
+lobotomy
+lobster
+local
+location
+lock
+locker
+locket
+locomotive
+locust
+loft
+log
+loggia
+logic
+loincloth
+loneliness
+long
+look
+loss
+lot
+lotion
+lounge
+lout
+love
+low
+loyalty
+luck
+luggage
+lumber
+lumberman
+lunch
+luncheonette
+lunchroom
+lung
+lunge
+lute
+luttuce
+lycra
+lye
+lymphocyte
+lynx
+lyocell
+lyre
+lyric
+macadamia
+macaroni
+machine
+machinery
+macrame
+macrofauna
+maelstrom
+maestro
+magazine
+magic
+maid
+maiden
+mail
+mailbox
+mailman
+main
+maintenance
+major
+major-league
+make
+makeup
+male
+mall
+mallet
+mambo
+mammoth
+man
+management
+manager
+mandarin
+mandolin
+mangrove
+manhunt
+maniac
+manicure
+mankind
+manner
+manor
+mansard
+manservant
+mansion
+mantel
+mantle
+mantua
+manufacturer
+manx
+many
+map
+maple
+maraca
+maracas
+marble
+mare
+margin
+mariachi
+marimba
+mark
+market
+marketing
+marksman
+marriage
+marsh
+marshland
+marxism
+mascara
+mask
+mass
+massage
+master
+mastication
+mastoid
+mat
+match
+mate
+material
+math
+mathematics
+matter
+mattock
+mattress
+maximum
+maybe
+mayonnaise
+mayor
+meal
+meaning
+measles
+measure
+measurement
+meat
+mechanic
+media
+medicine
+medium
+meet
+meeting
+megaliac
+melody
+member
+membership
+memory
+men
+menorah
+mention
+menu
+mercury
+mess
+message
+metal
+metallurgist
+meteor
+meteorology
+meter
+methane
+method
+methodology
+metro
+metronome
+mezzanine
+mice
+microlending
+microwave
+mid-course
+middle
+middleman
+midi
+midline
+midnight
+midwife
+might
+migrant
+mile
+milk
+milkshake
+millennium
+millimeter
+millisecond
+mime
+mimosa
+mind
+mine
+mini
+minibus
+minimum
+minion
+mini-skirt
+minister
+minor
+minor-league
+mint
+minute
+mirror
+miscarriage
+miscommunication
+misfit
+misogyny
+misplacement
+misreading
+miss
+missile
+mission
+mist
+mistake
+mister
+miter
+mitten
+mix
+mixer
+mixture
+moat
+mobile
+moccasins
+mocha
+mode
+model
+modem
+mole
+mom
+moment
+monastery
+monasticism
+money
+monger
+monitor
+monkey
+monocle
+monotheism
+monsoon
+monster
+month
+mood
+moon
+moonscape
+moonshine
+mop
+morning
+morsel
+mortgage
+mortise
+mosque
+mosquito
+most
+motel
+moth
+mother
+mother-in-law
+motion
+motor
+motorboat
+motorcar
+motorcycle
+mound
+mountain
+mouse
+mouser
+mousse
+moustache
+mouth
+mouton
+move
+mover
+movie
+mower
+mud
+mug
+mukluk
+mule
+multimedia
+muscle
+musculature
+museum
+music
+music-box
+music-making
+mustache
+mustard
+mutt
+mycoplasma
+n
+nail
+name
+naming
+nanoparticle
+napkin
+nasty
+nation
+national
+native
+natural
+naturalisation
+nature
+neat
+necessary
+neck
+necklace
+necktie
+need
+needle
+negative
+negligee
+negotiation
+neologism
+neon
+nephew
+nerve
+nest
+net
+netball
+netbook
+netsuke
+network
+neurobiologist
+neuropathologist
+neuropsychiatry
+news
+newspaper
+newsprint
+newsstand
+nexus
+nicety
+niche
+nickel
+niece
+night
+nightclub
+nightgown
+nightingale
+nightlight
+nitrogen
+nobody
+node
+noise
+nonbeliever
+nonconformist
+nondisclosure
+nonsense
+noodle
+normal
+norse
+north
+nose
+note
+notebook
+nothing
+notice
+notify
+notoriety
+nougat
+novel
+nudge
+number
+numeracy
+numeric
+numismatist
+nurse
+nursery
+nurture
+nut
+nutrition
+nylon
+oak
+oar
+oasis
+oatmeal
+obedience
+obesity
+obi
+object
+objective
+obligation
+oboe
+observation
+observatory
+occasion
+occupation
+ocean
+ocelot
+octagon
+octave
+octavo
+octet
+octopus
+odometer
+oeuvre
+offence
+offer
+office
+officer
+official
+off-ramp
+oil
+okra
+oldie
+olive
+omega
+omelet
+oncology
+one
+onion
+open
+opening
+opera
+operation
+ophthalmologist
+opinion
+opium
+opossum
+opportunist
+opportunity
+opposite
+option
+orange
+orangutan
+orator
+orchard
+orchestra
+orchid
+order
+ordinary
+ordination
+organ
+organisation
+organization
+original
+ornament
+osmosis
+osprey
+ostrich
+other
+others
+ott
+otter
+ounce
+outback
+outcome
+outfit
+outhouse
+outlay
+output
+outrigger
+outset
+outside
+oval
+ovary
+oven
+overcharge
+overclocking
+overcoat
+overexertion
+overflight
+overnighter
+overshoot
+owl
+owner
+ox
+oxen
+oxford
+oxygen
+oyster
+pace
+pacemaker
+pack
+package
+packet
+pad
+paddle
+paddock
+page
+pagoda
+pail
+pain
+paint
+painter
+painting
+paintwork
+pair
+pajama
+pajamas
+palm
+pamphlet
+pan
+pancake
+pancreas
+panda
+panic
+pannier
+panpipe
+pansy
+panther
+panties
+pantologist
+pantology
+pantry
+pants
+pantsuit
+panty
+pantyhose
+paper
+paperback
+parable
+parachute
+parade
+parallelogram
+paramedic
+parcel
+parchment
+pard
+parent
+parentheses
+park
+parka
+parking
+parrot
+parsnip
+part
+participant
+particle
+particular
+partner
+partridge
+party
+pass
+passage
+passbook
+passenger
+passion
+passive
+past
+pasta
+paste
+pastor
+pastoralist
+pastry
+patch
+path
+patience
+patient
+patina
+patio
+patriarch
+patricia
+patrimony
+patriot
+patrol
+pattern
+pause
+pavement
+pavilion
+paw
+pawnshop
+pay
+payee
+payment
+pea
+peace
+peach
+peacoat
+peacock
+peak
+peanut
+pear
+pearl
+pedal
+peen
+peer
+peer-to-peer
+pegboard
+pelican
+pelt
+pen
+penalty
+pencil
+pendant
+pendulum
+penicillin
+pension
+pentagon
+peony
+people
+pepper
+percentage
+perception
+perch
+performance
+perfume
+period
+periodical
+peripheral
+permafrost
+permission
+permit
+perp
+person
+personal
+personality
+perspective
+pest
+pet
+petal
+petticoat
+pew
+pha
+pharmacist
+pharmacopoeia
+phase
+pheasant
+philosopher
+philosophy
+phone
+photo
+photographer
+phrase
+physical
+physics
+pianist
+piano
+piccolo
+pick
+pickax
+picket
+pickle
+picture
+pie
+piece
+pier
+piety
+pig
+pigeon
+pike
+pile
+pilgrimage
+pillbox
+pillow
+pilot
+pimp
+pimple
+pin
+pinafore
+pince-nez
+pine
+pineapple
+pinecone
+ping
+pink
+pinkie
+pinstripe
+pint
+pinto
+pinworm
+pioneer
+pipe
+piracy
+piss
+pitch
+pitching
+pith
+pizza
+place
+plain
+plan
+plane
+planet
+plant
+plantation
+planter
+plaster
+plasterboard
+plastic
+plate
+platform
+platinum
+platypus
+play
+player
+playground
+playroom
+pleasure
+pleated
+plenty
+plier
+plot
+plough
+plover
+plow
+plowman
+plume
+plunger
+plywood
+pneumonia
+pocket
+pocketbook
+pocket-watch
+poem
+poet
+poetry
+poignance
+point
+poison
+poisoning
+pole
+polenta
+police
+policeman
+policy
+polish
+politics
+pollution
+polo
+polyester
+pompom
+poncho
+pond
+pony
+poof
+pool
+pop
+popcorn
+poppy
+popsicle
+population
+populist
+porch
+porcupine
+port
+porter
+portfolio
+porthole
+position
+positive
+possession
+possibility
+possible
+post
+postage
+postbox
+poster
+pot
+potato
+potential
+potty
+pouch
+poultry
+pound
+pounding
+poverty
+powder
+power
+practice
+precedent
+precipitation
+preface
+preference
+prelude
+premeditation
+premier
+preoccupation
+preparation
+presence
+present
+presentation
+president
+press
+pressroom
+pressure
+pressurisation
+price
+pride
+priest
+priesthood
+primary
+primate
+prince
+princess
+principal
+principle
+print
+printer
+prior
+priority
+prison
+private
+prize
+prizefight
+probation
+problem
+procedure
+process
+processing
+produce
+producer
+product
+production
+profession
+professional
+professor
+profile
+profit
+program
+progress
+project
+promise
+promotion
+prompt
+pronunciation
+proof
+proof-reader
+propane
+property
+proposal
+prose
+prosecution
+protection
+protest
+protocol
+prow
+pruner
+pseudoscience
+psychiatrist
+psychoanalyst
+psychologist
+psychology
+ptarmigan
+public
+publicity
+publisher
+pudding
+puddle
+puffin
+pull
+pulley
+puma
+pump
+pumpkin
+pumpkinseed
+punch
+punctuation
+punishment
+pupa
+pupil
+puppy
+purchase
+puritan
+purple
+purpose
+purse
+push
+pusher
+put
+pvc
+pyjama
+pyramid
+quadrant
+quail
+quality
+quantity
+quart
+quarter
+quartz
+queen
+question
+quicksand
+quiet
+quill
+quilt
+quince
+quit
+quiver
+quotation
+quote
+rabbi
+rabbit
+raccoon
+race
+racer
+racing
+racism
+racist
+rack
+radar
+radiator
+radio
+radiosonde
+radish
+raffle
+raft
+rag
+rage
+rail
+railway
+raiment
+rain
+rainbow
+raincoat
+rainmaker
+rainstorm
+raise
+rake
+ram
+rambler
+ramie
+ranch
+random
+randomisation
+range
+rank
+raspberry
+rat
+rate
+ratio
+raven
+ravioli
+raw
+rawhide
+ray
+rayon
+reach
+reactant
+reaction
+read
+reading
+reality
+reamer
+rear
+reason
+receipt
+reception
+recess
+recipe
+recliner
+recognition
+recommendation
+record
+recorder
+recording
+recover
+recreation
+recruit
+rectangle
+red
+redesign
+rediscovery
+reduction
+reef
+refectory
+reference
+reflection
+refrigerator
+refund
+refuse
+region
+register
+regret
+regular
+regulation
+reindeer
+reinscription
+reject
+relation
+relationship
+relative
+relaxation
+release
+reliability
+relief
+religion
+relish
+reminder
+remote
+remove
+rent
+repair
+reparation
+repeat
+replace
+replacement
+replication
+reply
+report
+representative
+reprocessing
+republic
+reputation
+request
+requirement
+resale
+research
+reserve
+resident
+resist
+resolution
+resolve
+resort
+resource
+respect
+respite
+respond
+response
+responsibility
+rest
+restaurant
+result
+retailer
+rethinking
+retina
+retouch
+return
+reveal
+revenant
+revenge
+revenue
+review
+revolution
+revolve
+revolver
+reward
+rheumatism
+rhinoceros
+rhyme
+rhythm
+rice
+rich
+riddle
+ride
+rider
+ridge
+rifle
+right
+rim
+ring
+ringworm
+rip
+ripple
+rise
+riser
+risk
+river
+riverbed
+rivulet
+road
+roadway
+roast
+robe
+robin
+rock
+rocker
+rocket
+rocket-ship
+rod
+role
+roll
+roller
+roof
+room
+rooster
+root
+rope
+rose
+rostrum
+rotate
+rough
+round
+roundabout
+route
+router
+routine
+row
+rowboat
+royal
+rub
+rubber
+rubbish
+rubric
+ruckus
+ruffle
+rugby
+ruin
+rule
+rum
+run
+runaway
+runner
+rush
+rutabaga
+ruth
+ry
+sabre
+sack
+sad
+saddle
+safe
+safety
+sage
+sail
+sailboat
+sailor
+salad
+salary
+sale
+salesman
+salmon
+salon
+saloon
+salt
+samovar
+sampan
+sample
+samurai
+sand
+sandals
+sandbar
+sandwich
+sardine
+sari
+sarong
+sash
+satellite
+satin
+satire
+satisfaction
+sauce
+sausage
+save
+saving
+savings
+savior
+saviour
+saw
+saxophone
+scale
+scallion
+scanner
+scarecrow
+scarf
+scarification
+scene
+scenery
+scent
+schedule
+scheme
+schizophrenic
+schnitzel
+school
+schoolhouse
+schooner
+science
+scimitar
+scissors
+scooter
+score
+scorn
+scow
+scraper
+scratch
+screamer
+screen
+screenwriting
+screw
+screwdriver
+screw-up
+scrim
+scrip
+script
+sculpting
+sculpture
+sea
+seafood
+seagull
+seal
+seaplane
+search
+seashore
+seaside
+season
+seat
+second
+secret
+secretariat
+secretary
+section
+sectional
+sector
+secure
+security
+seed
+seeder
+segment
+select
+selection
+self
+sell
+semicircle
+semicolon
+senator
+senior
+sense
+sensitive
+sentence
+sepal
+septicaemia
+series
+servant
+serve
+server
+service
+session
+set
+setting
+settler
+sewer
+sex
+shack
+shade
+shadow
+shadowbox
+shake
+shakedown
+shaker
+shallot
+shame
+shampoo
+shanty
+shape
+share
+shark
+sharon
+shawl
+she
+shearling
+shears
+sheath
+shed
+sheep
+sheet
+shelf
+shell
+shelter
+sherry
+shield
+shift
+shin
+shine
+shingle
+ship
+shirt
+shirtdress
+shoat
+shock
+shoe
+shoehorn
+shoe-horn
+shoelace
+shoemaker
+shoes
+shoestring
+shofar
+shoot
+shootdown
+shop
+shopper
+shopping
+shore
+shortage
+shorts
+shortwave
+shot
+shoulder
+shovel
+show
+shower
+show-stopper
+shred
+shrimp
+shrine
+sibling
+sick
+side
+sideboard
+sideburns
+sidecar
+sidestream
+sidewalk
+siding
+sign
+signal
+signature
+signet
+significance
+signup
+silence
+silica
+silk
+silkworm
+sill
+silly
+silo
+silver
+simple
+sing
+singer
+single
+sink
+sir
+sister
+sister-in-law
+sitar
+site
+situation
+size
+skate
+skiing
+skill
+skin
+skirt
+skull
+skullcap
+skullduggery
+skunk
+sky
+skylight
+skyscraper
+skywalk
+slapstick
+slash
+slave
+sled
+sledge
+sleep
+sleet
+sleuth
+slice
+slide
+slider
+slime
+slip
+slipper
+slippers
+slope
+sloth
+smash
+smell
+smelting
+smile
+smock
+smog
+smoke
+smoking
+smuggling
+snail
+snake
+snakebite
+sneakers
+sneeze
+snob
+snorer
+snow
+snowboarding
+snowflake
+snowman
+snowmobiling
+snowplow
+snowstorm
+snowsuit
+snuggle
+soap
+soccer
+society
+sociology
+sock
+socks
+soda
+sofa
+soft
+softball
+softdrink
+softening
+software
+soil
+soldier
+solid
+solitaire
+solution
+sombrero
+somersault
+somewhere
+son
+song
+songbird
+sonnet
+soot
+soprano
+sorbet
+sorrow
+sort
+soulmate
+sound
+soup
+source
+sourwood
+sousaphone
+south
+south america
+south korea
+sow
+soy
+soybean
+space
+spacing
+spade
+spaghetti
+spandex
+spank
+spare
+spark
+sparrow
+spasm
+speaker
+speakerphone
+spear
+special
+specialist
+specific
+spectacle
+spectacles
+spectrograph
+speech
+speed
+speedboat
+spell
+spelling
+spend
+sphere
+sphynx
+spider
+spike
+spinach
+spine
+spiral
+spirit
+spiritual
+spite
+spleen
+split
+sponge
+spoon
+sport
+spot
+spotlight
+spray
+spread
+spring
+sprinter
+sprout
+spruce
+spume
+spur
+spy
+square
+squash
+squatter
+squeegee
+squid
+squirrel
+stable
+stack
+stacking
+stadium
+staff
+stag
+stage
+stain
+stair
+staircase
+stallion
+stamen
+stamina
+stamp
+stance
+stand
+standard
+standoff
+star
+start
+starter
+state
+statement
+station
+station-wagon
+statistic
+status
+stay
+steak
+steal
+steam
+steamroller
+steel
+steeple
+stem
+stencil
+step
+step-aunt
+step-brother
+stepdaughter
+step-daughter
+step-father
+step-grandfather
+step-grandmother
+stepmother
+step-mother
+stepping-stone
+steps
+step-sister
+stepson
+step-son
+step-uncle
+stew
+stick
+stiletto
+still
+stinger
+stitch
+stock
+stocking
+stockings
+stock-in-trade
+stole
+stomach
+stone
+stonework
+stool
+stop
+stopsign
+stopwatch
+storage
+store
+storey
+storm
+story
+storyboard
+story-telling
+stove
+strain
+strait
+stranger
+strap
+strategy
+straw
+strawberry
+stream
+street
+streetcar
+strength
+stress
+stretch
+strike
+string
+strip
+stroke
+structure
+struggle
+stud
+student
+studio
+study
+stuff
+stumbling
+stupid
+stupidity
+sturgeon
+style
+styling
+stylus
+subcomponent
+subconscious
+subject
+submarine
+subroutine
+subsidence
+substance
+suburb
+subway
+success
+suck
+suede
+suffocation
+sugar
+suggestion
+suit
+suitcase
+sultan
+summer
+sun
+sunbeam
+sunbonnet
+sunday
+sundial
+sunflower
+sunglasses
+sunlamp
+sunroom
+sunshine
+supermarket
+supply
+support
+supporter
+suppression
+surface
+surfboard
+surgeon
+surgery
+surname
+surprise
+surround
+survey
+sushi
+suspect
+suspenders
+sustainment
+SUV
+swallow
+swamp
+swan
+swath
+sweat
+sweater
+sweats
+sweatshirt
+sweatshop
+sweatsuit
+swedish
+sweet
+sweets
+swell
+swim
+swimming
+swimsuit
+swing
+swiss
+switch
+switchboard
+swivel
+sword
+swordfish
+sycamore
+symmetry
+sympathy
+syndicate
+synergy
+synod
+syrup
+system
+tabby
+tabernacle
+table
+tablecloth
+tabletop
+tachometer
+tackle
+tadpole
+tail
+tailor
+tailspin
+tale
+talk
+tam
+tambour
+tambourine
+tam-o'-shanter
+tandem
+tangerine
+tank
+tanker
+tankful
+tank-top
+tap
+tard
+target
+task
+tassel
+taste
+tatami
+tattler
+tattoo
+tavern
+tax
+taxi
+taxicab
+tea
+teach
+teacher
+teaching
+team
+tear
+technologist
+technology
+teen
+teeth
+telephone
+telescreen
+teletype
+television
+tell
+teller
+temp
+temper
+temperature
+temple
+tempo
+temporariness
+temporary
+temptress
+tendency
+tenement
+tennis
+tenor
+tension
+tent
+tepee
+term
+terracotta
+terrapin
+territory
+test
+text
+textbook
+texture
+thanks
+thaw
+theater
+theism
+theme
+theory
+therapist
+thermals
+thermometer
+thigh
+thing
+thinking
+thirst
+thistle
+thomas
+thong
+thongs
+thorn
+thought
+thread
+thrill
+throat
+throne
+thrush
+thumb
+thunder
+thunderbolt
+thunderhead
+thunderstorm
+tiara
+tic
+ticket
+tie
+tiger
+tight
+tights
+tile
+till
+timbale
+timber
+time
+timeline
+timeout
+timer
+timpani
+tin
+tinderbox
+tinkle
+tintype
+tip
+tire
+tissue
+titanium
+title
+toad
+toast
+today
+toe
+toenail
+toga
+togs
+toilet
+tolerance
+tom
+tomato
+tomography
+tomorrow
+tom-tom
+ton
+tone
+tongue
+tonight
+tool
+toot
+tooth
+toothbrush
+toothpaste
+toothpick
+top
+top-hat
+topic
+topsail
+toque
+torchiere
+toreador
+tornado
+torso
+tortellini
+tortoise
+tosser
+total
+tote
+touch
+tough
+tough-guy
+tour
+tourist
+towel
+tower
+town
+townhouse
+tow-truck
+toy
+trachoma
+track
+tracksuit
+tractor
+trade
+tradition
+traditionalism
+traffic
+trail
+trailer
+train
+trainer
+training
+tram
+tramp
+transaction
+transition
+translation
+transmission
+transom
+transport
+transportation
+trapdoor
+trapezium
+trapezoid
+trash
+travel
+tray
+treat
+treatment
+tree
+trellis
+tremor
+trench
+trial
+triangle
+tribe
+trick
+trigonometry
+trim
+trinket
+trip
+tripod
+trolley
+trombone
+trooper
+trouble
+trousers
+trout
+trove
+trowel
+truck
+truckit
+trumpet
+trunk
+trust
+truth
+try
+t-shirt
+tsunami
+tub
+tuba
+tube
+tugboat
+tulip
+tummy
+tuna
+tune
+tune-up
+tunic
+tunnel
+turban
+turkish
+turn
+turnip
+turnover
+turnstile
+turret
+turtle
+tussle
+tutu
+tuxedo
+tv
+twig
+twilight
+twine
+twist
+twister
+two
+type
+typewriter
+typhoon
+tyvek
+ukulele
+umbrella
+unblinking
+uncle
+underclothes
+underground
+underneath
+underpants
+underpass
+undershirt
+understanding
+underwear
+underwire
+unemployment
+unibody
+uniform
+union
+unique
+unit
+unity
+university
+upper
+upstairs
+urn
+usage
+use
+user
+usher
+usual
+utensil
+vacation
+vacuum
+vagrant
+valance
+validity
+valley
+valuable
+value
+van
+vane
+vanity
+variation
+variety
+vase
+vast
+vault
+vaulting
+veal
+vegetable
+vegetarianism
+vegetation
+vehicle
+veil
+vein
+veldt
+vellum
+velodrome
+velvet
+vengeance
+venom
+veranda
+verdict
+vermicelli
+verse
+version
+vertigo
+verve
+vessel
+vest
+vestment
+vibe
+vibraphone
+vibration
+video
+view
+villa
+village
+vineyard
+vinyl
+viola
+violence
+violet
+violin
+virginal
+virtue
+virus
+viscose
+vise
+vision
+visit
+visitor
+visor
+visual
+vitality
+vixen
+voice
+volcano
+volleyball
+volume
+voyage
+vulture
+wad
+wafer
+waffle
+waist
+waistband
+wait
+waiter
+waitress
+wake
+walk
+walker
+walkway
+wall
+wallaby
+wallet
+walnut
+walrus
+wampum
+wannabe
+war
+warden
+warlock
+warmth
+warm-up
+warning
+wash
+washbasin
+washcloth
+washer
+washtub
+wasp
+waste
+wastebasket
+watch
+watchmaker
+water
+waterbed
+waterfall
+waterskiing
+waterspout
+wave
+wax
+way
+weakness
+wealth
+weapon
+wear
+weasel
+weather
+web
+wedding
+wedge
+weed
+weeder
+weedkiller
+week
+weekend
+weekender
+weight
+weird
+welcome
+welfare
+well
+west
+western
+wet-bar
+wetsuit
+whale
+wharf
+wheat
+wheel
+whereas
+while
+whip
+whirlpool
+whirlwind
+whisker
+whiskey
+whistle
+white
+whole
+wholesale
+wholesaler
+whorl
+width
+wife
+wilderness
+wildlife
+will
+willow
+win
+wind
+windage
+wind-chime
+window
+windscreen
+windshield
+wine
+wing
+wingman
+wingtip
+winner
+winter
+wire
+wisdom
+wiseguy
+wish
+wisteria
+witch
+witch-hunt
+withdrawal
+witness
+wolf
+wombat
+women
+wonder
+wood
+woodland
+woodshed
+woodwind
+wool
+woolen
+word
+work
+workbench
+worker
+workhorse
+working
+worklife
+workshop
+world
+worm
+worry
+worth
+worthy
+wound
+wrap
+wraparound
+wrecker
+wren
+wrench
+wrestler
+wrinkle
+wrist
+writer
+writing
+wrong
+xylophone
+yacht
+yak
+yam
+yard
+yarmulke
+yarn
+yawl
+year
+yeast
+yellow
+yesterday
+yew
+yin
+yoga
+yogurt
+yoke
+you
+young
+youth
+yurt
+zampone
+zebra
+zebrafish
+zephyr
+ziggurat
+zinc
+zipper
+zither
+zone
+zoo
+zoologist
+zoology
+zoot-suit
+zucchini \ No newline at end of file
diff --git a/testing/condprofile/condprof/tests/__init__.py b/testing/condprofile/condprof/tests/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/testing/condprofile/condprof/tests/__init__.py
diff --git a/testing/condprofile/condprof/tests/fakefirefox.py b/testing/condprofile/condprof/tests/fakefirefox.py
new file mode 100755
index 0000000000..82e24dd081
--- /dev/null
+++ b/testing/condprofile/condprof/tests/fakefirefox.py
@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+VERSION = """\
+Mozilla Firefox 70.0\
+"""
+
+if __name__ == "__main__":
+ print(VERSION)
diff --git a/testing/condprofile/condprof/tests/fakegeckodriver.py b/testing/condprofile/condprof/tests/fakegeckodriver.py
new file mode 100755
index 0000000000..ab40401aaf
--- /dev/null
+++ b/testing/condprofile/condprof/tests/fakegeckodriver.py
@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+import argparse
+import json
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from uuid import uuid4
+
+_SESSIONS = {}
+
+
+class Window:
+ def __init__(self, handle, title="about:blank"):
+ self.handle = handle
+ self.title = title
+
+ def visit_url(self, url):
+ print("Visiting %s" % url)
+ # XXX todo, load the URL for real
+ self.url = url
+
+
+class Session:
+ def __init__(self, uuid):
+ self.session_id = uuid
+ self.autoinc = 0
+ self.windows = {}
+ self.active_handle = self.new_window()
+
+ def visit(self, url):
+ self.windows[self.active_handle].visit_url(url)
+
+ def new_window(self):
+ w = Window(self.autoinc)
+ self.windows[w.handle] = w
+ self.autoinc += 1
+ return w.handle
+
+
+class RequestHandler(BaseHTTPRequestHandler):
+ def _set_headers(self, status=200):
+ self.send_response(status)
+ self.send_header("Content-type", "application/json")
+ self.end_headers()
+
+ def _send_response(self, status=200, data=None):
+ if data is None:
+ data = {}
+ data = json.dumps(data).encode("utf8")
+ self._set_headers(status)
+ self.wfile.write(data)
+
+ def _parse_path(self):
+ path = self.path.lstrip("/")
+ sections = path.split("/")
+ session = None
+ action = []
+ if len(sections) > 1:
+ session_id = sections[1]
+ if session_id in _SESSIONS:
+ session = _SESSIONS[session_id]
+ action = sections[2:]
+ return session, action
+
+ def do_GET(self):
+ print("GET " + self.path)
+ if self.path == "/status":
+ return self._send_response(data={"ready": "OK"})
+
+ session, action = self._parse_path()
+ if action == ["window", "handles"]:
+ data = {"value": list(session.windows.keys())}
+ return self._send_response(data=data)
+
+ if action == ["moz", "context"]:
+ data = {"value": "chrome"}
+ return self._send_response(data=data)
+
+ return self._send_response(status=404)
+
+ def do_POST(self):
+ print("POST " + self.path)
+ content_length = int(self.headers["Content-Length"])
+ post_data = json.loads(self.rfile.read(content_length))
+
+ # new session
+ if self.path == "/session":
+ uuid = str(uuid4())
+ _SESSIONS[uuid] = Session(uuid)
+ return self._send_response(data={"sessionId": uuid})
+
+ session, action = self._parse_path()
+ if action == ["url"]:
+ session.visit(post_data["url"])
+ return self._send_response()
+
+ if action == ["window", "new"]:
+ if session is None:
+ return self._send_response(404)
+ handle = session.new_window()
+ return self._send_response(data={"handle": handle, "type": "tab"})
+
+ if action == ["timeouts"]:
+ return self._send_response()
+
+ if action == ["execute", "async"]:
+ return self._send_response(data={"logs": []})
+
+ # other commands not supported yet, we just return 200s
+ return self._send_response()
+
+ def do_DELETE(self):
+ return self._send_response()
+ session, action = self._parse_path()
+ if session is not None:
+ del _SESSIONS[session.session_id]
+ return self._send_response()
+ return self._send_response(status=404)
+
+
+VERSION = """\
+geckodriver 0.24.0 ( 2019-01-28)
+
+The source code of this program is available from
+testing/geckodriver in https://hg.mozilla.org/mozilla-central.
+
+This program is subject to the terms of the Mozilla Public License 2.0.
+You can obtain a copy of the license at https://mozilla.org/MPL/2.0/.\
+"""
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="FakeGeckodriver")
+ parser.add_argument("--log", type=str, default=None)
+ parser.add_argument("--port", type=int, default=4444)
+ parser.add_argument("--marionette-port", type=int, default=2828)
+ parser.add_argument("--version", action="store_true", default=False)
+ parser.add_argument("--verbose", "-v", action="count")
+ args = parser.parse_args()
+
+ if args.version:
+ print(VERSION)
+ else:
+ HTTPServer.allow_reuse_address = True
+ server = HTTPServer(("127.0.0.1", args.port), RequestHandler)
+ server.serve_forever()
diff --git a/testing/condprofile/condprof/tests/ftp_mozilla.html b/testing/condprofile/condprof/tests/ftp_mozilla.html
new file mode 100644
index 0000000000..c970f8483b
--- /dev/null
+++ b/testing/condprofile/condprof/tests/ftp_mozilla.html
@@ -0,0 +1,1484 @@
+<!DOCTYPE html>
+<html>
+ <head>
+ <meta charset="UTF-8">
+ <title>Directory Listing: /pub/firefox/nightly/latest-mozilla-central/</title>
+ </head>
+ <body>
+ <h1>Index of /pub/firefox/nightly/latest-mozilla-central/</h1>
+ <table>
+ <tr>
+ <th>Type</th>
+ <th>Name</th>
+ <th>Size</th>
+ <th>Last Modified</th>
+ </tr>
+
+ <tr>
+ <td>Dir</td>
+ <td><a href="/pub/firefox/nightly/">..</a></td>
+ <td></td>
+ <td></td>
+ </tr>
+
+
+ <tr>
+ <td>Dir</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/mar-tools/">mar-tools/</a></td>
+ <td></td>
+ <td></td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/Firefox%20Installer.en-US.exe">Firefox Installer.en-US.exe</a></td>
+ <td>294K</td>
+ <td>02-Sep-2019 00:48</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.langpack.xpi">firefox-70.0a1.en-US.langpack.xpi</a></td>
+ <td>453K</td>
+ <td>02-Sep-2019 00:06</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.awsy.tests.tar.gz">firefox-70.0a1.en-US.linux-i686.awsy.tests.tar.gz</a></td>
+ <td>20K</td>
+ <td>01-Sep-2019 23:49</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.buildhub.json">firefox-70.0a1.en-US.linux-i686.buildhub.json</a></td>
+ <td>1K</td>
+ <td>01-Sep-2019 23:49</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.checksums">firefox-70.0a1.en-US.linux-i686.checksums</a></td>
+ <td>8K</td>
+ <td>01-Sep-2019 23:49</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.common.tests.tar.gz">firefox-70.0a1.en-US.linux-i686.common.tests.tar.gz</a></td>
+ <td>45M</td>
+ <td>01-Sep-2019 23:49</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.complete.mar">firefox-70.0a1.en-US.linux-i686.complete.mar</a></td>
+ <td>57M</td>
+ <td>01-Sep-2019 23:49</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.cppunittest.tests.tar.gz">firefox-70.0a1.en-US.linux-i686.cppunittest.tests.tar.gz</a></td>
+ <td>12M</td>
+ <td>01-Sep-2019 23:49</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.crashreporter-symbols.zip">firefox-70.0a1.en-US.linux-i686.crashreporter-symbols.zip</a></td>
+ <td>79M</td>
+ <td>01-Sep-2019 23:49</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.json">firefox-70.0a1.en-US.linux-i686.json</a></td>
+ <td>855</td>
+ <td>01-Sep-2019 23:49</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.mochitest.tests.tar.gz">firefox-70.0a1.en-US.linux-i686.mochitest.tests.tar.gz</a></td>
+ <td>65M</td>
+ <td>01-Sep-2019 23:49</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.mozinfo.json">firefox-70.0a1.en-US.linux-i686.mozinfo.json</a></td>
+ <td>916</td>
+ <td>01-Sep-2019 23:49</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.reftest.tests.tar.gz">firefox-70.0a1.en-US.linux-i686.reftest.tests.tar.gz</a></td>
+ <td>53M</td>
+ <td>01-Sep-2019 23:49</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.talos.tests.tar.gz">firefox-70.0a1.en-US.linux-i686.talos.tests.tar.gz</a></td>
+ <td>18M</td>
+ <td>01-Sep-2019 23:49</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.tar.bz2">firefox-70.0a1.en-US.linux-i686.tar.bz2</a></td>
+ <td>71M</td>
+ <td>01-Sep-2019 23:49</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.tar.bz2.asc">firefox-70.0a1.en-US.linux-i686.tar.bz2.asc</a></td>
+ <td>833</td>
+ <td>01-Sep-2019 23:49</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.test_packages.json">firefox-70.0a1.en-US.linux-i686.test_packages.json</a></td>
+ <td>1K</td>
+ <td>01-Sep-2019 23:49</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.txt">firefox-70.0a1.en-US.linux-i686.txt</a></td>
+ <td>99</td>
+ <td>01-Sep-2019 23:49</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.web-platform.tests.tar.gz">firefox-70.0a1.en-US.linux-i686.web-platform.tests.tar.gz</a></td>
+ <td>53M</td>
+ <td>01-Sep-2019 23:49</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.xpcshell.tests.tar.gz">firefox-70.0a1.en-US.linux-i686.xpcshell.tests.tar.gz</a></td>
+ <td>9M</td>
+ <td>01-Sep-2019 23:49</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686_info.txt">firefox-70.0a1.en-US.linux-i686_info.txt</a></td>
+ <td>23</td>
+ <td>01-Sep-2019 23:49</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.awsy.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.awsy.tests.tar.gz</a></td>
+ <td>20K</td>
+ <td>02-Sep-2019 01:43</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.buildhub.json">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.buildhub.json</a></td>
+ <td>1K</td>
+ <td>02-Sep-2019 01:43</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.checksums">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.checksums</a></td>
+ <td>8K</td>
+ <td>02-Sep-2019 01:44</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.common.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.common.tests.tar.gz</a></td>
+ <td>55M</td>
+ <td>02-Sep-2019 01:43</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.complete.mar">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.complete.mar</a></td>
+ <td>208M</td>
+ <td>02-Sep-2019 01:43</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.cppunittest.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.cppunittest.tests.tar.gz</a></td>
+ <td>118M</td>
+ <td>02-Sep-2019 01:43</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.json">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.json</a></td>
+ <td>860</td>
+ <td>02-Sep-2019 01:43</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.mochitest.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.mochitest.tests.tar.gz</a></td>
+ <td>65M</td>
+ <td>02-Sep-2019 01:43</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.mozinfo.json">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.mozinfo.json</a></td>
+ <td>927</td>
+ <td>02-Sep-2019 01:43</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.reftest.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.reftest.tests.tar.gz</a></td>
+ <td>53M</td>
+ <td>02-Sep-2019 01:43</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.talos.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.talos.tests.tar.gz</a></td>
+ <td>18M</td>
+ <td>02-Sep-2019 01:43</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.tar.bz2">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.tar.bz2</a></td>
+ <td>276M</td>
+ <td>02-Sep-2019 01:43</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.tar.bz2.asc">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.tar.bz2.asc</a></td>
+ <td>833</td>
+ <td>02-Sep-2019 01:43</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.test_packages.json">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.test_packages.json</a></td>
+ <td>1K</td>
+ <td>02-Sep-2019 01:43</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.txt">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.txt</a></td>
+ <td>99</td>
+ <td>02-Sep-2019 01:43</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.web-platform.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.web-platform.tests.tar.gz</a></td>
+ <td>53M</td>
+ <td>02-Sep-2019 01:43</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.xpcshell.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.xpcshell.tests.tar.gz</a></td>
+ <td>10M</td>
+ <td>02-Sep-2019 01:43</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter_info.txt">firefox-70.0a1.en-US.linux-x86_64-asan-reporter_info.txt</a></td>
+ <td>23</td>
+ <td>02-Sep-2019 01:43</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.awsy.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64.awsy.tests.tar.gz</a></td>
+ <td>20K</td>
+ <td>02-Sep-2019 00:06</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.buildhub.json">firefox-70.0a1.en-US.linux-x86_64.buildhub.json</a></td>
+ <td>1K</td>
+ <td>02-Sep-2019 00:06</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.checksums">firefox-70.0a1.en-US.linux-x86_64.checksums</a></td>
+ <td>8K</td>
+ <td>02-Sep-2019 00:07</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.common.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64.common.tests.tar.gz</a></td>
+ <td>45M</td>
+ <td>02-Sep-2019 00:06</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.complete.mar">firefox-70.0a1.en-US.linux-x86_64.complete.mar</a></td>
+ <td>57M</td>
+ <td>02-Sep-2019 00:06</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.cppunittest.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64.cppunittest.tests.tar.gz</a></td>
+ <td>12M</td>
+ <td>02-Sep-2019 00:06</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.crashreporter-symbols.zip">firefox-70.0a1.en-US.linux-x86_64.crashreporter-symbols.zip</a></td>
+ <td>74M</td>
+ <td>02-Sep-2019 00:06</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.json">firefox-70.0a1.en-US.linux-x86_64.json</a></td>
+ <td>846</td>
+ <td>02-Sep-2019 00:06</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.mochitest.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64.mochitest.tests.tar.gz</a></td>
+ <td>65M</td>
+ <td>02-Sep-2019 00:06</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.mozinfo.json">firefox-70.0a1.en-US.linux-x86_64.mozinfo.json</a></td>
+ <td>921</td>
+ <td>02-Sep-2019 00:06</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.reftest.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64.reftest.tests.tar.gz</a></td>
+ <td>53M</td>
+ <td>02-Sep-2019 00:06</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.talos.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64.talos.tests.tar.gz</a></td>
+ <td>18M</td>
+ <td>02-Sep-2019 00:06</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.tar.bz2">firefox-70.0a1.en-US.linux-x86_64.tar.bz2</a></td>
+ <td>71M</td>
+ <td>02-Sep-2019 00:06</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.tar.bz2.asc">firefox-70.0a1.en-US.linux-x86_64.tar.bz2.asc</a></td>
+ <td>833</td>
+ <td>02-Sep-2019 00:06</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.test_packages.json">firefox-70.0a1.en-US.linux-x86_64.test_packages.json</a></td>
+ <td>1K</td>
+ <td>02-Sep-2019 00:06</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.txt">firefox-70.0a1.en-US.linux-x86_64.txt</a></td>
+ <td>99</td>
+ <td>02-Sep-2019 00:06</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.web-platform.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64.web-platform.tests.tar.gz</a></td>
+ <td>53M</td>
+ <td>02-Sep-2019 00:06</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.xpcshell.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64.xpcshell.tests.tar.gz</a></td>
+ <td>9M</td>
+ <td>02-Sep-2019 00:06</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64_info.txt">firefox-70.0a1.en-US.linux-x86_64_info.txt</a></td>
+ <td>23</td>
+ <td>02-Sep-2019 00:06</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.awsy.tests.tar.gz">firefox-70.0a1.en-US.mac.awsy.tests.tar.gz</a></td>
+ <td>20K</td>
+ <td>01-Sep-2019 23:19</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.buildhub.json">firefox-70.0a1.en-US.mac.buildhub.json</a></td>
+ <td>1K</td>
+ <td>01-Sep-2019 23:19</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.checksums">firefox-70.0a1.en-US.mac.checksums</a></td>
+ <td>7K</td>
+ <td>01-Sep-2019 23:19</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.common.tests.tar.gz">firefox-70.0a1.en-US.mac.common.tests.tar.gz</a></td>
+ <td>20M</td>
+ <td>01-Sep-2019 23:19</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.complete.mar">firefox-70.0a1.en-US.mac.complete.mar</a></td>
+ <td>60M</td>
+ <td>01-Sep-2019 23:19</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.cppunittest.tests.tar.gz">firefox-70.0a1.en-US.mac.cppunittest.tests.tar.gz</a></td>
+ <td>10M</td>
+ <td>01-Sep-2019 23:19</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.crashreporter-symbols.zip">firefox-70.0a1.en-US.mac.crashreporter-symbols.zip</a></td>
+ <td>54M</td>
+ <td>01-Sep-2019 23:19</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.dmg">firefox-70.0a1.en-US.mac.dmg</a></td>
+ <td>79M</td>
+ <td>01-Sep-2019 23:19</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.json">firefox-70.0a1.en-US.mac.json</a></td>
+ <td>1K</td>
+ <td>01-Sep-2019 23:19</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.mochitest.tests.tar.gz">firefox-70.0a1.en-US.mac.mochitest.tests.tar.gz</a></td>
+ <td>65M</td>
+ <td>01-Sep-2019 23:19</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.mozinfo.json">firefox-70.0a1.en-US.mac.mozinfo.json</a></td>
+ <td>923</td>
+ <td>01-Sep-2019 23:19</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.pkg">firefox-70.0a1.en-US.mac.pkg</a></td>
+ <td>83M</td>
+ <td>01-Sep-2019 23:19</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.reftest.tests.tar.gz">firefox-70.0a1.en-US.mac.reftest.tests.tar.gz</a></td>
+ <td>53M</td>
+ <td>01-Sep-2019 23:19</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.talos.tests.tar.gz">firefox-70.0a1.en-US.mac.talos.tests.tar.gz</a></td>
+ <td>18M</td>
+ <td>01-Sep-2019 23:19</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.test_packages.json">firefox-70.0a1.en-US.mac.test_packages.json</a></td>
+ <td>1K</td>
+ <td>01-Sep-2019 23:19</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.txt">firefox-70.0a1.en-US.mac.txt</a></td>
+ <td>99</td>
+ <td>01-Sep-2019 23:19</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.web-platform.tests.tar.gz">firefox-70.0a1.en-US.mac.web-platform.tests.tar.gz</a></td>
+ <td>53M</td>
+ <td>01-Sep-2019 23:19</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.xpcshell.tests.tar.gz">firefox-70.0a1.en-US.mac.xpcshell.tests.tar.gz</a></td>
+ <td>9M</td>
+ <td>01-Sep-2019 23:19</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac_info.txt">firefox-70.0a1.en-US.mac_info.txt</a></td>
+ <td>23</td>
+ <td>01-Sep-2019 23:19</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.awsy.tests.tar.gz">firefox-70.0a1.en-US.win32.awsy.tests.tar.gz</a></td>
+ <td>20K</td>
+ <td>02-Sep-2019 00:48</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.buildhub.json">firefox-70.0a1.en-US.win32.buildhub.json</a></td>
+ <td>1K</td>
+ <td>02-Sep-2019 00:48</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.checksums">firefox-70.0a1.en-US.win32.checksums</a></td>
+ <td>8K</td>
+ <td>02-Sep-2019 00:49</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.common.tests.tar.gz">firefox-70.0a1.en-US.win32.common.tests.tar.gz</a></td>
+ <td>22M</td>
+ <td>02-Sep-2019 00:48</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.complete.mar">firefox-70.0a1.en-US.win32.complete.mar</a></td>
+ <td>51M</td>
+ <td>02-Sep-2019 00:48</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.cppunittest.tests.tar.gz">firefox-70.0a1.en-US.win32.cppunittest.tests.tar.gz</a></td>
+ <td>10M</td>
+ <td>02-Sep-2019 00:48</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.crashreporter-symbols.zip">firefox-70.0a1.en-US.win32.crashreporter-symbols.zip</a></td>
+ <td>34M</td>
+ <td>02-Sep-2019 00:48</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.installer.exe">firefox-70.0a1.en-US.win32.installer.exe</a></td>
+ <td>48M</td>
+ <td>02-Sep-2019 00:48</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.installer.msi">firefox-70.0a1.en-US.win32.installer.msi</a></td>
+ <td>48M</td>
+ <td>02-Sep-2019 00:48</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.json">firefox-70.0a1.en-US.win32.json</a></td>
+ <td>884</td>
+ <td>02-Sep-2019 00:48</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.mochitest.tests.tar.gz">firefox-70.0a1.en-US.win32.mochitest.tests.tar.gz</a></td>
+ <td>65M</td>
+ <td>02-Sep-2019 00:48</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.mozinfo.json">firefox-70.0a1.en-US.win32.mozinfo.json</a></td>
+ <td>948</td>
+ <td>02-Sep-2019 00:48</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.reftest.tests.tar.gz">firefox-70.0a1.en-US.win32.reftest.tests.tar.gz</a></td>
+ <td>53M</td>
+ <td>02-Sep-2019 00:48</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.talos.tests.tar.gz">firefox-70.0a1.en-US.win32.talos.tests.tar.gz</a></td>
+ <td>18M</td>
+ <td>02-Sep-2019 00:48</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.test_packages.json">firefox-70.0a1.en-US.win32.test_packages.json</a></td>
+ <td>1K</td>
+ <td>02-Sep-2019 00:48</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.txt">firefox-70.0a1.en-US.win32.txt</a></td>
+ <td>99</td>
+ <td>02-Sep-2019 00:48</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.web-platform.tests.tar.gz">firefox-70.0a1.en-US.win32.web-platform.tests.tar.gz</a></td>
+ <td>53M</td>
+ <td>02-Sep-2019 00:48</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.xpcshell.tests.tar.gz">firefox-70.0a1.en-US.win32.xpcshell.tests.tar.gz</a></td>
+ <td>9M</td>
+ <td>02-Sep-2019 00:48</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.zip">firefox-70.0a1.en-US.win32.zip</a></td>
+ <td>70M</td>
+ <td>02-Sep-2019 00:48</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32_info.txt">firefox-70.0a1.en-US.win32_info.txt</a></td>
+ <td>23</td>
+ <td>02-Sep-2019 00:48</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.awsy.tests.tar.gz">firefox-70.0a1.en-US.win64-aarch64.awsy.tests.tar.gz</a></td>
+ <td>20K</td>
+ <td>02-Sep-2019 01:15</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.buildhub.json">firefox-70.0a1.en-US.win64-aarch64.buildhub.json</a></td>
+ <td>914</td>
+ <td>02-Sep-2019 01:15</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.checksums">firefox-70.0a1.en-US.win64-aarch64.checksums</a></td>
+ <td>8K</td>
+ <td>02-Sep-2019 01:16</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.common.tests.tar.gz">firefox-70.0a1.en-US.win64-aarch64.common.tests.tar.gz</a></td>
+ <td>20M</td>
+ <td>02-Sep-2019 01:15</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.complete.mar">firefox-70.0a1.en-US.win64-aarch64.complete.mar</a></td>
+ <td>78M</td>
+ <td>02-Sep-2019 01:15</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.cppunittest.tests.tar.gz">firefox-70.0a1.en-US.win64-aarch64.cppunittest.tests.tar.gz</a></td>
+ <td>10M</td>
+ <td>02-Sep-2019 01:15</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.crashreporter-symbols.zip">firefox-70.0a1.en-US.win64-aarch64.crashreporter-symbols.zip</a></td>
+ <td>20M</td>
+ <td>02-Sep-2019 01:15</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.installer.exe">firefox-70.0a1.en-US.win64-aarch64.installer.exe</a></td>
+ <td>74M</td>
+ <td>02-Sep-2019 01:15</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.json">firefox-70.0a1.en-US.win64-aarch64.json</a></td>
+ <td>705</td>
+ <td>02-Sep-2019 01:15</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.mochitest.tests.tar.gz">firefox-70.0a1.en-US.win64-aarch64.mochitest.tests.tar.gz</a></td>
+ <td>65M</td>
+ <td>02-Sep-2019 01:15</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.mozinfo.json">firefox-70.0a1.en-US.win64-aarch64.mozinfo.json</a></td>
+ <td>946</td>
+ <td>02-Sep-2019 01:15</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.reftest.tests.tar.gz">firefox-70.0a1.en-US.win64-aarch64.reftest.tests.tar.gz</a></td>
+ <td>53M</td>
+ <td>02-Sep-2019 01:15</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.talos.tests.tar.gz">firefox-70.0a1.en-US.win64-aarch64.talos.tests.tar.gz</a></td>
+ <td>18M</td>
+ <td>02-Sep-2019 01:15</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.test_packages.json">firefox-70.0a1.en-US.win64-aarch64.test_packages.json</a></td>
+ <td>1K</td>
+ <td>02-Sep-2019 01:15</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.txt">firefox-70.0a1.en-US.win64-aarch64.txt</a></td>
+ <td>99</td>
+ <td>02-Sep-2019 01:15</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.web-platform.tests.tar.gz">firefox-70.0a1.en-US.win64-aarch64.web-platform.tests.tar.gz</a></td>
+ <td>53M</td>
+ <td>02-Sep-2019 01:15</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.xpcshell.tests.tar.gz">firefox-70.0a1.en-US.win64-aarch64.xpcshell.tests.tar.gz</a></td>
+ <td>9M</td>
+ <td>02-Sep-2019 01:15</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.zip">firefox-70.0a1.en-US.win64-aarch64.zip</a></td>
+ <td>110M</td>
+ <td>02-Sep-2019 01:15</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64_info.txt">firefox-70.0a1.en-US.win64-aarch64_info.txt</a></td>
+ <td>23</td>
+ <td>02-Sep-2019 01:15</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.awsy.tests.tar.gz">firefox-70.0a1.en-US.win64-asan-reporter.awsy.tests.tar.gz</a></td>
+ <td>20K</td>
+ <td>02-Sep-2019 00:16</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.buildhub.json">firefox-70.0a1.en-US.win64-asan-reporter.buildhub.json</a></td>
+ <td>1K</td>
+ <td>02-Sep-2019 00:16</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.checksums">firefox-70.0a1.en-US.win64-asan-reporter.checksums</a></td>
+ <td>7K</td>
+ <td>02-Sep-2019 00:17</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.common.tests.tar.gz">firefox-70.0a1.en-US.win64-asan-reporter.common.tests.tar.gz</a></td>
+ <td>22M</td>
+ <td>02-Sep-2019 00:16</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.complete.mar">firefox-70.0a1.en-US.win64-asan-reporter.complete.mar</a></td>
+ <td>201M</td>
+ <td>02-Sep-2019 00:16</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.cppunittest.tests.tar.gz">firefox-70.0a1.en-US.win64-asan-reporter.cppunittest.tests.tar.gz</a></td>
+ <td>56M</td>
+ <td>02-Sep-2019 00:16</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.installer.exe">firefox-70.0a1.en-US.win64-asan-reporter.installer.exe</a></td>
+ <td>191M</td>
+ <td>02-Sep-2019 00:16</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.json">firefox-70.0a1.en-US.win64-asan-reporter.json</a></td>
+ <td>894</td>
+ <td>02-Sep-2019 00:16</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.mochitest.tests.tar.gz">firefox-70.0a1.en-US.win64-asan-reporter.mochitest.tests.tar.gz</a></td>
+ <td>65M</td>
+ <td>02-Sep-2019 00:16</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.mozinfo.json">firefox-70.0a1.en-US.win64-asan-reporter.mozinfo.json</a></td>
+ <td>957</td>
+ <td>02-Sep-2019 00:16</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.reftest.tests.tar.gz">firefox-70.0a1.en-US.win64-asan-reporter.reftest.tests.tar.gz</a></td>
+ <td>53M</td>
+ <td>02-Sep-2019 00:16</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.talos.tests.tar.gz">firefox-70.0a1.en-US.win64-asan-reporter.talos.tests.tar.gz</a></td>
+ <td>18M</td>
+ <td>02-Sep-2019 00:16</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.test_packages.json">firefox-70.0a1.en-US.win64-asan-reporter.test_packages.json</a></td>
+ <td>1K</td>
+ <td>02-Sep-2019 00:16</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.txt">firefox-70.0a1.en-US.win64-asan-reporter.txt</a></td>
+ <td>99</td>
+ <td>02-Sep-2019 00:16</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.web-platform.tests.tar.gz">firefox-70.0a1.en-US.win64-asan-reporter.web-platform.tests.tar.gz</a></td>
+ <td>53M</td>
+ <td>02-Sep-2019 00:16</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.xpcshell.tests.tar.gz">firefox-70.0a1.en-US.win64-asan-reporter.xpcshell.tests.tar.gz</a></td>
+ <td>9M</td>
+ <td>02-Sep-2019 00:16</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.zip">firefox-70.0a1.en-US.win64-asan-reporter.zip</a></td>
+ <td>307M</td>
+ <td>02-Sep-2019 00:16</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter_info.txt">firefox-70.0a1.en-US.win64-asan-reporter_info.txt</a></td>
+ <td>23</td>
+ <td>02-Sep-2019 00:16</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.awsy.tests.tar.gz">firefox-70.0a1.en-US.win64.awsy.tests.tar.gz</a></td>
+ <td>20K</td>
+ <td>02-Sep-2019 00:44</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.buildhub.json">firefox-70.0a1.en-US.win64.buildhub.json</a></td>
+ <td>1K</td>
+ <td>02-Sep-2019 00:44</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.checksums">firefox-70.0a1.en-US.win64.checksums</a></td>
+ <td>8K</td>
+ <td>02-Sep-2019 00:45</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.common.tests.tar.gz">firefox-70.0a1.en-US.win64.common.tests.tar.gz</a></td>
+ <td>22M</td>
+ <td>02-Sep-2019 00:44</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.complete.mar">firefox-70.0a1.en-US.win64.complete.mar</a></td>
+ <td>53M</td>
+ <td>02-Sep-2019 00:44</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.cppunittest.tests.tar.gz">firefox-70.0a1.en-US.win64.cppunittest.tests.tar.gz</a></td>
+ <td>11M</td>
+ <td>02-Sep-2019 00:44</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.crashreporter-symbols.zip">firefox-70.0a1.en-US.win64.crashreporter-symbols.zip</a></td>
+ <td>25M</td>
+ <td>02-Sep-2019 00:44</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.installer.exe">firefox-70.0a1.en-US.win64.installer.exe</a></td>
+ <td>50M</td>
+ <td>02-Sep-2019 00:44</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.installer.msi">firefox-70.0a1.en-US.win64.installer.msi</a></td>
+ <td>50M</td>
+ <td>02-Sep-2019 00:44</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.json">firefox-70.0a1.en-US.win64.json</a></td>
+ <td>880</td>
+ <td>02-Sep-2019 00:44</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.mochitest.tests.tar.gz">firefox-70.0a1.en-US.win64.mochitest.tests.tar.gz</a></td>
+ <td>65M</td>
+ <td>02-Sep-2019 00:44</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.mozinfo.json">firefox-70.0a1.en-US.win64.mozinfo.json</a></td>
+ <td>951</td>
+ <td>02-Sep-2019 00:44</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.reftest.tests.tar.gz">firefox-70.0a1.en-US.win64.reftest.tests.tar.gz</a></td>
+ <td>53M</td>
+ <td>02-Sep-2019 00:44</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.talos.tests.tar.gz">firefox-70.0a1.en-US.win64.talos.tests.tar.gz</a></td>
+ <td>18M</td>
+ <td>02-Sep-2019 00:44</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.test_packages.json">firefox-70.0a1.en-US.win64.test_packages.json</a></td>
+ <td>1K</td>
+ <td>02-Sep-2019 00:44</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.txt">firefox-70.0a1.en-US.win64.txt</a></td>
+ <td>99</td>
+ <td>02-Sep-2019 00:44</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.web-platform.tests.tar.gz">firefox-70.0a1.en-US.win64.web-platform.tests.tar.gz</a></td>
+ <td>53M</td>
+ <td>02-Sep-2019 00:44</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.xpcshell.tests.tar.gz">firefox-70.0a1.en-US.win64.xpcshell.tests.tar.gz</a></td>
+ <td>9M</td>
+ <td>02-Sep-2019 00:44</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.zip">firefox-70.0a1.en-US.win64.zip</a></td>
+ <td>73M</td>
+ <td>02-Sep-2019 00:44</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64_info.txt">firefox-70.0a1.en-US.win64_info.txt</a></td>
+ <td>23</td>
+ <td>02-Sep-2019 00:44</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/jsshell-linux-i686.zip">jsshell-linux-i686.zip</a></td>
+ <td>11M</td>
+ <td>01-Sep-2019 23:49</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/jsshell-linux-x86_64.zip">jsshell-linux-x86_64.zip</a></td>
+ <td>11M</td>
+ <td>02-Sep-2019 00:06</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/jsshell-mac.zip">jsshell-mac.zip</a></td>
+ <td>11M</td>
+ <td>01-Sep-2019 23:19</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/jsshell-win32.zip">jsshell-win32.zip</a></td>
+ <td>10M</td>
+ <td>02-Sep-2019 00:48</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/jsshell-win64-aarch64.zip">jsshell-win64-aarch64.zip</a></td>
+ <td>1M</td>
+ <td>02-Sep-2019 01:15</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/jsshell-win64.zip">jsshell-win64.zip</a></td>
+ <td>11M</td>
+ <td>02-Sep-2019 00:44</td>
+ </tr>
+
+
+
+ <tr>
+ <td>File</td>
+ <td><a href="/pub/firefox/nightly/latest-mozilla-central/mozharness.zip">mozharness.zip</a></td>
+ <td>2M</td>
+ <td>02-Sep-2019 01:43</td>
+ </tr>
+
+
+ </table>
+ </body>
+</html>
diff --git a/testing/condprofile/condprof/tests/profile/prefs.js b/testing/condprofile/condprof/tests/profile/prefs.js
new file mode 100644
index 0000000000..abbe0fb70b
--- /dev/null
+++ b/testing/condprofile/condprof/tests/profile/prefs.js
@@ -0,0 +1 @@
+user_pref("gfx.blacklist.direct2d", 1);
diff --git a/testing/condprofile/condprof/tests/profile/user.js b/testing/condprofile/condprof/tests/profile/user.js
new file mode 100644
index 0000000000..c1efc79c10
--- /dev/null
+++ b/testing/condprofile/condprof/tests/profile/user.js
@@ -0,0 +1,9 @@
+
+#Prefs used for the unit test
+user_pref("focusmanager.testmode", true);
+user_pref("marionette.defaultPrefs.port", 2828);
+user_pref("marionette.port", 2828);
+user_pref("remote.log.level", "Trace");
+user_pref("marionette.log.truncate", false);
+user_pref("extensions.autoDisableScopes", 0);
+user_pref("devtools.debugger.remote-enabled", true);
diff --git a/testing/condprofile/condprof/tests/python.toml b/testing/condprofile/condprof/tests/python.toml
new file mode 100644
index 0000000000..14cffb2f5c
--- /dev/null
+++ b/testing/condprofile/condprof/tests/python.toml
@@ -0,0 +1,4 @@
+[DEFAULT]
+subsuite = "condprof"
+
+["test_client.py"]
diff --git a/testing/condprofile/condprof/tests/test_client.py b/testing/condprofile/condprof/tests/test_client.py
new file mode 100644
index 0000000000..8410133e16
--- /dev/null
+++ b/testing/condprofile/condprof/tests/test_client.py
@@ -0,0 +1,126 @@
+import json
+import os
+import re
+import shutil
+import tarfile
+import tempfile
+import unittest
+
+import responses
+from mozprofile.prefs import Preferences
+
+from condprof.client import ROOT_URL, TC_SERVICE, get_profile
+from condprof.util import _DEFAULT_SERVER
+
+PROFILE = re.compile(ROOT_URL + "/.*/.*tgz")
+PROFILE_FOR_TESTS = os.path.join(os.path.dirname(__file__), "profile")
+SECRETS = re.compile(_DEFAULT_SERVER + "/.*")
+SECRETS_PROXY = re.compile("http://taskcluster/secrets/.*")
+
+
+class TestClient(unittest.TestCase):
+ def setUp(self):
+ self.profile_dir = tempfile.mkdtemp()
+
+ # creating profile.tgz on the fly for serving it
+ profile_tgz = os.path.join(self.profile_dir, "profile.tgz")
+ with tarfile.open(profile_tgz, "w:gz") as tar:
+ tar.add(PROFILE_FOR_TESTS, arcname=".")
+
+ # self.profile_data is the tarball we're sending back via HTTP
+ with open(profile_tgz, "rb") as f:
+ self.profile_data = f.read()
+
+ self.target = tempfile.mkdtemp()
+ self.download_dir = os.path.expanduser("~/.condprof-cache")
+ if os.path.exists(self.download_dir):
+ shutil.rmtree(self.download_dir)
+
+ responses.add(
+ responses.GET,
+ PROFILE,
+ body=self.profile_data,
+ headers={"content-length": str(len(self.profile_data)), "ETag": "'12345'"},
+ status=200,
+ )
+
+ responses.add(
+ responses.HEAD,
+ PROFILE,
+ body="",
+ headers={"content-length": str(len(self.profile_data)), "ETag": "'12345'"},
+ status=200,
+ )
+
+ responses.add(responses.HEAD, TC_SERVICE, body="", status=200)
+
+ secret = {"secret": {"username": "user", "password": "pass"}}
+ secret = json.dumps(secret)
+ for pattern in (SECRETS, SECRETS_PROXY):
+ responses.add(
+ responses.GET,
+ pattern,
+ body=secret,
+ headers={"content-length": str(len(secret))},
+ status=200,
+ )
+
+ def tearDown(self):
+ shutil.rmtree(self.target)
+ shutil.rmtree(self.download_dir)
+ shutil.rmtree(self.profile_dir)
+
+ @responses.activate
+ def test_cache(self):
+ download_dir = os.path.expanduser("~/.condprof-cache")
+ if os.path.exists(download_dir):
+ num_elmts = len(os.listdir(download_dir))
+ else:
+ num_elmts = 0
+
+ get_profile(self.target, "win64", "settled", "default")
+
+ # grabbing a profile should generate two files
+ self.assertEqual(len(os.listdir(download_dir)), num_elmts + 2)
+
+ # we do at least two network calls when getting a file,
+ # a HEAD and a GET and possibly a TC secret
+ self.assertTrue(len(responses.calls) >= 2)
+
+ # reseting the response counters
+ responses.calls.reset()
+
+ # and we should reuse them without downloading the file again
+ get_profile(self.target, "win64", "settled", "default")
+
+ # grabbing a profile should not download new stuff
+ self.assertEqual(len(os.listdir(download_dir)), num_elmts + 2)
+
+ # and do a single extra HEAD call, everything else is cached,
+ # even the TC secret
+ self.assertEqual(len(responses.calls), 2)
+
+ prefs_js = os.path.join(self.target, "prefs.js")
+ prefs = Preferences.read_prefs(prefs_js)
+
+ # check that the gfx.blacklist prefs where cleaned out
+ for name, value in prefs:
+ self.assertFalse(name.startswith("gfx.blacklist"))
+
+ # check that we have the startupScanScopes option forced
+ prefs = dict(prefs)
+ self.assertEqual(prefs["extensions.startupScanScopes"], 1)
+
+ # make sure we don't have any marionette option set
+ user_js = os.path.join(self.target, "user.js")
+ for name, value in Preferences.read_prefs(user_js):
+ self.assertFalse(name.startswith("marionette."))
+
+
+if __name__ == "__main__":
+ try:
+ import mozunit
+ except ImportError:
+ pass
+ else:
+ mozunit.main(runwith="unittest")
diff --git a/testing/condprofile/condprof/tests/test_runner.py b/testing/condprofile/condprof/tests/test_runner.py
new file mode 100644
index 0000000000..e200a537e1
--- /dev/null
+++ b/testing/condprofile/condprof/tests/test_runner.py
@@ -0,0 +1,123 @@
+import asyncio
+import os
+import re
+import shutil
+import tarfile
+import tempfile
+import unittest
+
+import responses
+
+from condprof import client
+from condprof.client import ROOT_URL, TC_SERVICE
+from condprof.main import main
+
+client.RETRIES = 1
+client.RETRY_PAUSE = 0
+GECKODRIVER = os.path.join(os.path.dirname(__file__), "fakegeckodriver.py")
+FIREFOX = os.path.join(os.path.dirname(__file__), "fakefirefox.py")
+CHANGELOG = re.compile(ROOT_URL + "/.*/changelog.json")
+FTP = "https://ftp.mozilla.org/pub/firefox/nightly/latest-mozilla-central/"
+PROFILE = re.compile(ROOT_URL + "/.*/.*tgz")
+
+
+with open(os.path.join(os.path.dirname(__file__), "ftp_mozilla.html")) as f:
+ FTP_PAGE = f.read()
+
+FTP_ARCHIVE = re.compile(
+ "https://ftp.mozilla.org/pub/firefox/nightly/" "latest-mozilla-central/firefox.*"
+)
+
+ADDON = re.compile("https://addons.mozilla.org/.*/.*xpi")
+
+
+async def fakesleep(duration):
+ if duration > 1:
+ duration = 1
+ await asyncio.realsleep(duration)
+
+
+asyncio.realsleep = asyncio.sleep
+asyncio.sleep = fakesleep
+
+
+class TestRunner(unittest.TestCase):
+ def setUp(self):
+ self.archive_dir = tempfile.mkdtemp()
+ responses.add(responses.GET, CHANGELOG, json={"error": "not found"}, status=404)
+ responses.add(
+ responses.GET, FTP, content_type="text/html", body=FTP_PAGE, status=200
+ )
+
+ profile_tgz = os.path.join(os.path.dirname(__file__), "profile.tgz")
+ profile = os.path.join(os.path.dirname(__file__), "profile")
+
+ with tarfile.open(profile_tgz, "w:gz") as tar:
+ tar.add(profile, arcname=".")
+
+ with open(profile_tgz, "rb") as f:
+ PROFILE_DATA = f.read()
+
+ os.remove(profile_tgz)
+
+ responses.add(
+ responses.GET,
+ FTP_ARCHIVE,
+ body="1",
+ headers={"content-length": "1"},
+ status=200,
+ )
+
+ responses.add(
+ responses.GET,
+ PROFILE,
+ body=PROFILE_DATA,
+ headers={"content-length": str(len(PROFILE_DATA))},
+ status=200,
+ )
+
+ responses.add(
+ responses.HEAD,
+ PROFILE,
+ body="",
+ headers={"content-length": str(len(PROFILE_DATA))},
+ status=200,
+ )
+
+ responses.add(responses.HEAD, FTP_ARCHIVE, body="", status=200)
+
+ responses.add(
+ responses.GET, ADDON, body="1", headers={"content-length": "1"}, status=200
+ )
+
+ responses.add(
+ responses.HEAD, ADDON, body="", headers={"content-length": "1"}, status=200
+ )
+
+ responses.add(responses.HEAD, TC_SERVICE, body="", status=200)
+
+ def tearDown(self):
+ shutil.rmtree(self.archive_dir)
+
+ @responses.activate
+ def test_runner(self):
+ if "FXA_USERNAME" not in os.environ:
+ os.environ["FXA_USERNAME"] = "me"
+ if "FXA_PASSWORD" not in os.environ:
+ os.environ["FXA_PASSWORD"] = "password"
+ try:
+ args = [
+ "--geckodriver",
+ GECKODRIVER,
+ "--firefox",
+ FIREFOX,
+ self.archive_dir,
+ ]
+ main(args)
+ # XXX we want a bunch of assertions here to check
+ # that the archives dir gets filled correctly
+ finally:
+ if os.environ["FXA_USERNAME"] == "me":
+ del os.environ["FXA_USERNAME"]
+ if os.environ["FXA_PASSWORD"] == "password":
+ del os.environ["FXA_PASSWORD"]
diff --git a/testing/condprofile/condprof/util.py b/testing/condprofile/condprof/util.py
new file mode 100644
index 0000000000..444398b9f4
--- /dev/null
+++ b/testing/condprofile/condprof/util.py
@@ -0,0 +1,484 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This module needs to stay Python 2 and 3 compatible
+#
+import contextlib
+import os
+import platform
+import shutil
+import sys
+import tempfile
+import time
+from subprocess import PIPE, Popen
+
+import mozlog
+import requests
+import yaml
+from requests.exceptions import ConnectionError
+from requests.packages.urllib3.util.retry import Retry
+
+from condprof import progress
+
+TASK_CLUSTER = "MOZ_AUTOMATION" in os.environ.keys()
+DOWNLOAD_TIMEOUT = 30
+
+
+class ArchiveNotFound(Exception):
+ pass
+
+
+DEFAULT_PREFS = {
+ "focusmanager.testmode": True,
+ "marionette.defaultPrefs.port": 2828,
+ "marionette.port": 2828,
+ "remote.log.level": "Trace",
+ "marionette.log.truncate": False,
+ "extensions.autoDisableScopes": 10,
+ "devtools.debugger.remote-enabled": True,
+ "devtools.console.stdout.content": True,
+ "devtools.console.stdout.chrome": True,
+}
+
+DEFAULT_CUSTOMIZATION = os.path.join(
+ os.path.dirname(__file__), "customization", "default.json"
+)
+STRUCTLOG_PAD_SIZE = 20
+
+
+class BridgeLogger:
+ def __init__(self, logger):
+ self.logger = logger
+
+ def _find(self, text, *names):
+ # structlog's ConsoleRenderer pads values
+ for name in names:
+ if name + " " * STRUCTLOG_PAD_SIZE in text:
+ return True
+ return False
+
+ def _convert(self, message):
+ return obfuscate(message)[1]
+
+ def info(self, message, *args, **kw):
+ if not isinstance(message, str):
+ message = str(message)
+ # converting Arsenic request/response struct log
+ if self._find(message, "request", "response"):
+ self.logger.debug(self._convert(message), *args, **kw)
+ else:
+ self.logger.info(self._convert(message), *args, **kw)
+
+ def error(self, message, *args, **kw):
+ self.logger.error(self._convert(message), *args, **kw)
+
+ def warning(self, message, *args, **kw):
+ self.logger.warning(self._convert(message), *args, **kw)
+
+
+logger = None
+
+
+def get_logger():
+ global logger
+ if logger is not None:
+ return logger
+ new_logger = mozlog.get_default_logger("condprof")
+ if new_logger is None:
+ new_logger = mozlog.unstructured.getLogger("condprof")
+
+ # wrap the logger into the BridgeLogger
+ new_logger = BridgeLogger(new_logger)
+
+ # bridge for Arsenic
+ if sys.version_info.major == 3:
+ try:
+ from arsenic import connection
+ from structlog import wrap_logger
+
+ connection.log = wrap_logger(new_logger)
+ except ImportError:
+ # Arsenic is not installed for client-only usage
+ pass
+ logger = new_logger
+ return logger
+
+
+# initializing the logger right away
+get_logger()
+
+
+def fresh_profile(profile, customization_data):
+ from mozprofile import create_profile # NOQA
+
+ # XXX on android we mgiht need to run it on the device?
+ logger.info("Creating a fresh profile")
+ new_profile = create_profile(app="firefox")
+ prefs = customization_data["prefs"]
+ prefs.update(DEFAULT_PREFS)
+ logger.info("Setting prefs %s" % str(prefs.items()))
+ new_profile.set_preferences(prefs)
+ extensions = []
+ for name, url in customization_data["addons"].items():
+ logger.info("Downloading addon %s" % name)
+ # When running on the CI, we expect the xpi files to have been
+ # fetched by the firefox-addons fetch task dependency (see
+ # taskcluster/ci/fetch/browsertime.yml) and the condprof-addons
+ # linter enforces the content of the archive to be unpacked into
+ # a subdirectory named "firefox-addons".
+ extension = download_file(url, mozfetches_subdir="firefox-addons")
+ extensions.append(extension)
+ logger.info("Installing addons")
+ new_profile.addons.install(extensions)
+ shutil.copytree(new_profile.profile, profile)
+ return profile
+
+
+link = "https://ftp.mozilla.org/pub/firefox/nightly/latest-mozilla-central/"
+
+
+def get_firefox_download_link():
+ try:
+ from bs4 import BeautifulSoup
+ except ImportError:
+ raise ImportError("You need to run pip install beautifulsoup4")
+ if platform.system() == "Darwin":
+ extension = ".dmg"
+ elif platform.system() == "Linux":
+ arch = platform.machine()
+ extension = ".linux-%s.tar.bz2" % arch
+ else:
+ raise NotImplementedError(platform.system())
+
+ page = requests.get(link).text
+ soup = BeautifulSoup(page, "html.parser")
+ for node in soup.find_all("a", href=True):
+ href = node["href"]
+ if href.endswith(extension):
+ return "https://ftp.mozilla.org" + href
+
+ raise Exception()
+
+
+def check_exists(archive, server=None, all_types=False):
+ if server is not None:
+ archive = server + "/" + archive
+ try:
+ logger.info("Getting headers at %s" % archive)
+ resp = requests.head(archive, timeout=DOWNLOAD_TIMEOUT)
+ except ConnectionError:
+ return False, {}
+
+ if resp.status_code in (302, 303):
+ logger.info("Redirected")
+ return check_exists(resp.headers["Location"])
+
+ # see Bug 1574854
+ if (
+ not all_types
+ and resp.status_code == 200
+ and "text/html" in resp.headers["Content-Type"]
+ ):
+ logger.info("Got an html page back")
+ exists = False
+ else:
+ logger.info("Response code is %d" % resp.status_code)
+ exists = resp.status_code
+
+ return exists, resp.headers
+
+
+def check_mozfetches_dir(target, mozfetches_subdir):
+ logger.info("Checking for existence of: %s in MOZ_FETCHES_DIR" % target)
+ fetches = os.environ.get("MOZ_FETCHES_DIR")
+ if fetches is None:
+ return None
+ fetches_target = os.path.join(fetches, mozfetches_subdir, target)
+ if not os.path.exists(fetches_target):
+ return None
+ logger.info("Already fetched and available in MOZ_FETCHES_DIR: %s" % fetches_target)
+ return fetches_target
+
+
+def download_file(url, target=None, mozfetches_subdir=None):
+ if target is None:
+ target = url.split("/")[-1]
+
+ # check if the assets has been fetched through a taskgraph fetch task dependency
+ # and already available in the MOZ_FETCHES_DIR passed as an additional parameter.
+ if mozfetches_subdir is not None:
+ filepath = check_mozfetches_dir(target, mozfetches_subdir)
+ if filepath is not None:
+ return filepath
+
+ present, headers = check_exists(url)
+ if not present:
+ logger.info("Cannot find %r" % url)
+ raise ArchiveNotFound(url)
+
+ etag = headers.get("ETag")
+
+ logger.info("Checking for existence of: %s" % target)
+ if os.path.exists(target):
+ # XXX for now, reusing downloads without checking them
+ # when we don't have an .etag file
+ if etag is None or not os.path.exists(target + ".etag"):
+ logger.info("No existing etag downloads.")
+ return target
+ with open(target + ".etag") as f:
+ current_etag = f.read()
+ if etag == current_etag:
+ logger.info("Already Downloaded.")
+ # should at least check the size?
+ return target
+ else:
+ logger.info("Changed!")
+ else:
+ logger.info("Could not find an existing archive.")
+ # Add some debugging logs for the directory content
+ try:
+ archivedir = os.path.dirname(target)
+ logger.info(
+ "Content in cache directory %s: %s"
+ % (archivedir, os.listdir(archivedir))
+ )
+ except Exception:
+ logger.info("Failed to list cache directory contents")
+
+ logger.info("Downloading %s" % url)
+ req = requests.get(url, stream=True, timeout=DOWNLOAD_TIMEOUT)
+ total_length = int(req.headers.get("content-length"))
+ target_dir = os.path.dirname(target)
+ if target_dir != "" and not os.path.exists(target_dir):
+ logger.info("Creating dir %s" % target_dir)
+ os.makedirs(target_dir)
+
+ with open(target, "wb") as f:
+ if TASK_CLUSTER:
+ for chunk in req.iter_content(chunk_size=1024):
+ if chunk:
+ f.write(chunk)
+ f.flush()
+ else:
+ iter = req.iter_content(chunk_size=1024)
+ # pylint --py3k W1619
+ size = total_length / 1024 + 1
+ for chunk in progress.bar(iter, expected_size=size):
+ if chunk:
+ f.write(chunk)
+ f.flush()
+
+ if etag is not None:
+ with open(target + ".etag", "w") as f:
+ f.write(etag)
+
+ return target
+
+
+def extract_from_dmg(dmg, target):
+ mount = tempfile.mkdtemp()
+ cmd = "hdiutil attach -nobrowse -mountpoint %s %s"
+ os.system(cmd % (mount, dmg))
+ try:
+ found = False
+ for f in os.listdir(mount):
+ if not f.endswith(".app"):
+ continue
+ app = os.path.join(mount, f)
+ shutil.copytree(app, target)
+ found = True
+ break
+ finally:
+ os.system("hdiutil detach " + mount)
+ shutil.rmtree(mount)
+ if not found:
+ raise IOError("No app file found in %s" % dmg)
+
+
+@contextlib.contextmanager
+def latest_nightly(binary=None):
+ if binary is None:
+ # we want to use the latest nightly
+ nightly_archive = get_firefox_download_link()
+ logger.info("Downloading %s" % nightly_archive)
+ target = download_file(nightly_archive)
+ # on macOs we just mount the DMG
+ # XXX replace with extract_from_dmg
+ if platform.system() == "Darwin":
+ cmd = "hdiutil attach -mountpoint /Volumes/Nightly %s"
+ os.system(cmd % target)
+ binary = "/Volumes/Nightly/Firefox Nightly.app/Contents/MacOS/firefox"
+ # on linux we unpack it
+ elif platform.system() == "Linux":
+ cmd = "bunzip2 %s" % target
+ os.system(cmd)
+ cmd = "tar -xvf %s" % target[: -len(".bz2")]
+ os.system(cmd)
+ binary = "firefox/firefox"
+
+ mounted = True
+ else:
+ mounted = False
+ try:
+ yield binary
+ finally:
+ # XXX replace with extract_from_dmg
+ if mounted:
+ if platform.system() == "Darwin":
+ logger.info("Unmounting Firefox")
+ time.sleep(10)
+ os.system("hdiutil detach /Volumes/Nightly")
+ elif platform.system() == "Linux":
+ # XXX we should keep it for next time
+ shutil.rmtree("firefox")
+
+
+def write_yml_file(yml_file, yml_data):
+ logger.info("writing %s to %s" % (yml_data, yml_file))
+ try:
+ with open(yml_file, "w") as outfile:
+ yaml.dump(yml_data, outfile, default_flow_style=False)
+ except Exception:
+ logger.error("failed to write yaml file", exc_info=True)
+
+
+def get_version(firefox):
+ p = Popen([firefox, "--version"], stdin=PIPE, stdout=PIPE, stderr=PIPE)
+ output, __ = p.communicate()
+ first_line = output.strip().split(b"\n")[0]
+ res = first_line.split()[-1]
+ return res.decode("utf-8")
+
+
+def get_current_platform():
+ """Returns a combination of system and arch info that matches TC standards.
+
+ e.g. macosx64, win32, linux64, etc..
+ """
+ arch = sys.maxsize == 2**63 - 1 and "64" or "32"
+ plat = platform.system().lower()
+ if plat == "windows":
+ plat = "win"
+ elif plat == "darwin":
+ plat = "macosx"
+ return plat + arch
+
+
+class BaseEnv:
+ def __init__(self, profile, firefox, geckodriver, archive, device_name):
+ self.profile = profile
+ self.firefox = firefox
+ self.geckodriver = geckodriver
+ if profile is None:
+ self.profile = os.path.join(tempfile.mkdtemp(), "profile")
+ else:
+ self.profile = profile
+ self.archive = archive
+ self.device_name = device_name
+
+ @property
+ def target_platform(self):
+ return self.get_target_platform()
+
+ def get_target_platform(self):
+ raise NotImplementedError()
+
+ def get_device(self, *args, **kw):
+ raise NotImplementedError()
+
+ @contextlib.contextmanager
+ def get_browser(self, path):
+ raise NotImplementedError()
+
+ def get_browser_args(self, headless):
+ raise NotImplementedError()
+
+ def prepare(self, logfile):
+ pass
+
+ def check_session(self, session):
+ pass
+
+ def dump_logs(self):
+ pass
+
+ def get_browser_version(self):
+ raise NotImplementedError()
+
+ def get_geckodriver(self, log_file):
+ raise NotImplementedError()
+
+ def collect_profile(self):
+ pass
+
+ def stop_browser(self):
+ pass
+
+
+_URL = (
+ "{0}/secrets/v1/secret/project"
+ "{1}releng{1}gecko{1}build{1}level-{2}{1}conditioned-profiles"
+)
+_DEFAULT_SERVER = "https://firefox-ci-tc.services.mozilla.com"
+
+
+def get_tc_secret():
+ if not TASK_CLUSTER:
+ raise OSError("Not running in Taskcluster")
+ session = requests.Session()
+ retry = Retry(total=5, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504])
+ http_adapter = requests.adapters.HTTPAdapter(max_retries=retry)
+ session.mount("https://", http_adapter)
+ session.mount("http://", http_adapter)
+ secrets_url = _URL.format(
+ os.environ.get("TASKCLUSTER_PROXY_URL", _DEFAULT_SERVER),
+ "%2F",
+ os.environ.get("MOZ_SCM_LEVEL", "1"),
+ )
+ res = session.get(secrets_url, timeout=DOWNLOAD_TIMEOUT)
+ res.raise_for_status()
+ return res.json()["secret"]
+
+
+_CACHED = {}
+
+
+def obfuscate(text):
+ if "CONDPROF_RUNNER" not in os.environ:
+ return True, text
+ username, password = get_credentials()
+ if username is None:
+ return False, text
+ if username not in text and password not in text:
+ return False, text
+ text = text.replace(password, "<PASSWORD>")
+ text = text.replace(username, "<USERNAME>")
+ return True, text
+
+
+def obfuscate_file(path):
+ if "CONDPROF_RUNNER" not in os.environ:
+ return
+ with open(path) as f:
+ data = f.read()
+ hit, data = obfuscate(data)
+ if not hit:
+ return
+ with open(path, "w") as f:
+ f.write(data)
+
+
+def get_credentials():
+ if "creds" in _CACHED:
+ return _CACHED["creds"]
+ password = os.environ.get("FXA_PASSWORD")
+ username = os.environ.get("FXA_USERNAME")
+ if username is None or password is None:
+ if not TASK_CLUSTER:
+ return None, None
+ secret = get_tc_secret()
+ password = secret["password"]
+ username = secret["username"]
+ _CACHED["creds"] = username, password
+ return username, password
diff --git a/testing/condprofile/mach_commands.py b/testing/condprofile/mach_commands.py
new file mode 100644
index 0000000000..e9a0d54867
--- /dev/null
+++ b/testing/condprofile/mach_commands.py
@@ -0,0 +1,121 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, # You can obtain one at http://mozilla.org/MPL/2.0/.
+import logging
+import os
+import sys
+import tempfile
+
+from mach.decorators import Command, CommandArgument
+from mozbuild.base import BinaryNotFoundException
+
+requirements = os.path.join(os.path.dirname(__file__), "requirements", "base.txt")
+
+
+def _init(command_context):
+ command_context.activate_virtualenv()
+ command_context.virtualenv_manager.install_pip_requirements(
+ requirements, require_hashes=False
+ )
+
+
+@Command("fetch-condprofile", category="testing")
+@CommandArgument("--target-dir", default=None, help="Target directory")
+@CommandArgument("--platform", default=None, help="Platform")
+@CommandArgument("--scenario", default="full", help="Scenario") # grab choices
+@CommandArgument("--customization", default="default", help="Customization") # same
+@CommandArgument("--task-id", default=None, help="Task ID")
+@CommandArgument("--download-cache", action="store_true", default=True)
+@CommandArgument(
+ "--repo",
+ default="mozilla-central",
+ choices=["mozilla-central", "try"],
+ help="Repository",
+)
+def fetch(
+ command_context,
+ target_dir,
+ platform,
+ scenario,
+ customization,
+ task_id,
+ download_cache,
+ repo,
+):
+ _init(command_context)
+ from condprof.client import get_profile
+ from condprof.util import get_current_platform, get_version
+
+ if platform is None:
+ platform = get_current_platform()
+
+ if target_dir is None:
+ target_dir = tempfile.mkdtemp()
+
+ version = get_version(command_context.get_binary_path())
+
+ get_profile(
+ target_dir,
+ platform,
+ scenario,
+ customization,
+ task_id,
+ download_cache,
+ repo,
+ version,
+ )
+ print("Downloaded conditioned profile can be found at: %s" % target_dir)
+
+
+@Command("run-condprofile", category="testing")
+@CommandArgument("archive", help="Archives Dir", type=str, default=None)
+@CommandArgument("--firefox", help="Firefox Binary", type=str, default=None)
+@CommandArgument("--scenario", help="Scenario to use", type=str, default="all")
+@CommandArgument("--profile", help="Existing profile Dir", type=str, default=None)
+@CommandArgument(
+ "--customization", help="Profile customization to use", type=str, default="all"
+)
+@CommandArgument(
+ "--visible", help="Don't use headless mode", action="store_true", default=False
+)
+@CommandArgument(
+ "--archives-dir", help="Archives local dir", type=str, default="/tmp/archives"
+)
+@CommandArgument(
+ "--force-new", help="Create from scratch", action="store_true", default=False
+)
+@CommandArgument(
+ "--strict",
+ help="Errors out immediatly on a scenario failure",
+ action="store_true",
+ default=True,
+)
+@CommandArgument(
+ "--geckodriver",
+ help="Path to the geckodriver binary",
+ type=str,
+ default=sys.platform.startswith("win") and "geckodriver.exe" or "geckodriver",
+)
+@CommandArgument("--device-name", help="Name of the device", type=str, default=None)
+def run(command_context, **kw):
+ os.environ["MANUAL_MACH_RUN"] = "1"
+ _init(command_context)
+
+ if kw["firefox"] is None:
+ try:
+ kw["firefox"] = command_context.get_binary_path()
+ except BinaryNotFoundException as e:
+ command_context.log(
+ logging.ERROR,
+ "run-condprofile",
+ {"error": str(e)},
+ "ERROR: {error}",
+ )
+ command_context.log(
+ logging.INFO, "run-condprofile", {"help": e.help()}, "{help}"
+ )
+ return 1
+
+ from condprof.runner import run
+
+ run(**kw)
diff --git a/testing/condprofile/moz.build b/testing/condprofile/moz.build
new file mode 100644
index 0000000000..94cd2f1555
--- /dev/null
+++ b/testing/condprofile/moz.build
@@ -0,0 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+with Files("**"):
+ BUG_COMPONENT = ("Testing", "Condprofile")
+ SCHEDULES.exclusive = ["condprofile"]
diff --git a/testing/condprofile/requirements/base.txt b/testing/condprofile/requirements/base.txt
new file mode 100644
index 0000000000..fc7d70013a
--- /dev/null
+++ b/testing/condprofile/requirements/base.txt
@@ -0,0 +1,5 @@
+aiohttp==3.7.4.post0
+https://pypi.pub.build.mozilla.org/pub/arsenic-19.1-py3-none-any.whl
+requests==2.22.0
+pyyaml==5.1.2
+mozversion>=1.0
diff --git a/testing/condprofile/requirements/ci-client.txt b/testing/condprofile/requirements/ci-client.txt
new file mode 100644
index 0000000000..f04776863f
--- /dev/null
+++ b/testing/condprofile/requirements/ci-client.txt
@@ -0,0 +1,10 @@
+# pulled when running in TaskCluster for python 2 only
+# we just pull dependencies required by condprof.client
+requests==2.22.0
+pyyaml==5.1.2
+
+# the target.condprof.tests.tar.gz archive pulls those dependencies
+# directly into the condprof project root
+./mozfile
+./mozlog
+./mozprofile
diff --git a/testing/condprofile/requirements/ci.txt b/testing/condprofile/requirements/ci.txt
new file mode 100644
index 0000000000..d1e4f62bef
--- /dev/null
+++ b/testing/condprofile/requirements/ci.txt
@@ -0,0 +1,7 @@
+# pulled when running in TaskCluster
+# the target.condprof.tests.tar.gz archive pulls those dependencies
+# directly into the condprof project root.
+./mozfile
+./mozlog
+./mozprofile
+./mozdevice
diff --git a/testing/condprofile/requirements/local-client.txt b/testing/condprofile/requirements/local-client.txt
new file mode 100644
index 0000000000..8a5e4da1e2
--- /dev/null
+++ b/testing/condprofile/requirements/local-client.txt
@@ -0,0 +1,6 @@
+./mozbase/mozfile
+./mozbase/mozlog
+./mozbase/mozprofile
+
+requests==2.22.0
+pyyaml==5.1.2
diff --git a/testing/condprofile/requirements/local.txt b/testing/condprofile/requirements/local.txt
new file mode 100644
index 0000000000..b487db7274
--- /dev/null
+++ b/testing/condprofile/requirements/local.txt
@@ -0,0 +1,5 @@
+# pulled when running locally
+../mozbase/mozfile
+../mozbase/mozlog
+../mozbase/mozprofile
+../mozbase/mozdevice
diff --git a/testing/condprofile/requirements/tox.txt b/testing/condprofile/requirements/tox.txt
new file mode 100644
index 0000000000..772a4a4818
--- /dev/null
+++ b/testing/condprofile/requirements/tox.txt
@@ -0,0 +1,8 @@
+# pulled when running tox locally
+-r requirements/local.txt
+
+pytest
+pytest-cov
+pytest-random-order
+coveralls
+responses
diff --git a/testing/condprofile/setup.py b/testing/condprofile/setup.py
new file mode 100644
index 0000000000..0648bff232
--- /dev/null
+++ b/testing/condprofile/setup.py
@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from setuptools import find_packages, setup
+
+entry_points = """
+ [console_scripts]
+ cp-creator = condprof.main:main
+ cp-client = condprof.client:main
+ """
+
+setup(
+ name="conditioned-profile",
+ version="0.2",
+ packages=find_packages(),
+ description="Firefox Heavy Profile creator",
+ include_package_data=True,
+ data_files=[
+ (
+ "condprof",
+ [
+ "condprof/customization/default.json",
+ "condprof/customization/youtube.json",
+ "condprof/customization/webext.json",
+ ],
+ )
+ ],
+ zip_safe=False,
+ install_requires=[], # use requirements files
+ entry_points=entry_points,
+)
diff --git a/testing/condprofile/tox.ini b/testing/condprofile/tox.ini
new file mode 100644
index 0000000000..ddaca5911a
--- /dev/null
+++ b/testing/condprofile/tox.ini
@@ -0,0 +1,16 @@
+[tox]
+downloadcache = {toxworkdir}/cache/
+envlist = py36,flake8
+
+[testenv]
+passenv = TRAVIS TRAVIS_JOB_ID TRAVIS_BRANCH
+deps = -rrequirements/tox.txt
+commands =
+ pytest --random-order-bucket=global -sv --cov-report= --cov-config .coveragerc --cov condprof condprof/tests
+ - coverage report -m
+ - coveralls
+
+[testenv:flake8]
+commands = flake8 condprof
+deps =
+ flake8