diff options
Diffstat (limited to 'testing/condprofile')
49 files changed, 9477 insertions, 0 deletions
diff --git a/testing/condprofile/Makefile b/testing/condprofile/Makefile new file mode 100644 index 0000000000..b626f5c7e9 --- /dev/null +++ b/testing/condprofile/Makefile @@ -0,0 +1,23 @@ +HERE = $(shell pwd) +BIN = $(HERE)/bin +PYTHON = $(BIN)/python +INSTALL = $(BIN)/pip install --no-deps +BUILD_DIRS = bin build include lib lib64 man share +VIRTUALENV = virtualenv + +.PHONY: all test build docs + +all: build + +$(PYTHON): + $(VIRTUALENV) $(VTENV_OPTS) . + +build: $(PYTHON) + $(PYTHON) setup.py develop + $(BIN)/pip install tox + +test: build + $(BIN)/tox + +docs: build + $(BIN)/tox -e docs diff --git a/testing/condprofile/README.rst b/testing/condprofile/README.rst new file mode 100644 index 0000000000..01b7fb73be --- /dev/null +++ b/testing/condprofile/README.rst @@ -0,0 +1,114 @@ +Conditioned Profile +=================== + +This project provides a command-line tool that is used to generate and maintain +a collection of Gecko profiles. + +Unlike testing/profiles, the **conditioned profiles** are a collection of full +Gecko profiles that are dynamically updated every day. + +Each profile is created or updated using a **scenario** and a +**customization**, and eventually uploaded as an artifact in TaskCluster. + +The goal of the project is to build a collection of profiles that we can use in +our performance or functional tests instead of the empty profile that we +usually create on the fly with **mozprofile**. + +Having a collection of realistic profiles we can use when running some tests +gives us the ability to check the impact of user profiles on page loads or +other tests. + +A full cycle of how this tool is used in Taskcluster looks like this: + +For each combination of scenario, customization and platform: + +- grabs an existing profile in Taskcluster +- browses the web using the scenario, via the WebDriver client +- recreates a tarball with the updated profile +- uploads it as an index artifact into TaskCluster - maintains a changelog of each change + +It's based on the Arsenic webdriver client https://github.com/HDE/arsenic + +The project provides two **Mach** commands to interact with the conditioned +profile: + +- **fetch-condprofile**: downloads a conditioned profile and deecompress it +- **run-condprofile**: runs on or all conditioned profiles scenarii locally + +How to download a conditioned profile +===================================== + +From your mozilla-central root, run: + +:: + + $ ./mach fetch-condprofile + +This will grab the latest conditioned profile for your platform. But +you can also grab a specific profile built from any scenario or platform. + +You can look at all the options with --help + +How to run a conditioned profile +================================ + +If you want to play a scenario locally to modify it, run for example: + +:: + + $ ./mach run-condprofile --scenario settled --visible /path/to/generated/profile + +The project will run a webdriver session against Firefox and generate the profile. +You can look at all the options with --help + +Architecture +============ + +The conditioned profile project is organized into webdriver **scenarii** and +**customization** files. + +Scenarii +-------- + +Scenarii are coroutines registered under a unique name in condprof/scenarii/__init__.py. + +They get a **session** object and some **options**. + +The scenario can do whatever it wants with the browser, through the webdriver session +instance. + +See Arsenic's `API documentation <https://arsenic.readthedocs.io/en/latest/reference/session.html>`_ for the session class. + +Adding a new scenario is done by adding a module in condprof/scenarii/ +and register it in condprof/scenarii/__init__.py + + +Customization +------------- + +A customization is a configuration file that can be used to set some +prefs in the browser and install some webextensions. + +Customizations are JSON files registered into condprof/customizations, +and they provide four keys: + +- **name**: the name of the customization +- **addons**: a mapping of add-ons to install. +- **prefs**: a mapping of prefs to set +- **scenario**: a mapping of options to pass to a specific scenario + +In the example below, we install uBlock, set a pref, and pass the +**max_urls** option to the **heavy** scenario. + + { + "name": "intermediate", + "addons":{ + "uBlock":"https://addons.mozilla.org/firefox/downloads/file/3361355/ublock_origin-1.21.2-an+fx.xpi" + }, + "prefs":{ + "accessibility.tabfocus": 9 + }, + "scenario": { + "heavy": {"max_urls": 10} + } + } diff --git a/testing/condprofile/condprof/__init__.py b/testing/condprofile/condprof/__init__.py new file mode 100644 index 0000000000..6fbe8159b2 --- /dev/null +++ b/testing/condprofile/condprof/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/testing/condprofile/condprof/android.py b/testing/condprofile/condprof/android.py new file mode 100644 index 0000000000..b56d0b906c --- /dev/null +++ b/testing/condprofile/condprof/android.py @@ -0,0 +1,294 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" Drives an android device. +""" +import os +import posixpath +import tempfile +import contextlib +import time +import logging + +import attr +from arsenic.services import Geckodriver, free_port, subprocess_based_service +from mozdevice import ADBDeviceFactory, ADBError + +from condprof.util import write_yml_file, logger, DEFAULT_PREFS, BaseEnv + + +# XXX most of this code should migrate into mozdevice - see Bug 1574849 +class AndroidDevice: + def __init__( + self, + app_name, + marionette_port=2828, + verbose=False, + remote_test_root="/sdcard/test_root/", + ): + self.app_name = app_name + + # XXX make that an option + if "fenix" in app_name: + self.activity = "org.mozilla.fenix.IntentReceiverActivity" + else: + self.activity = "org.mozilla.geckoview_example.GeckoViewActivity" + self.verbose = verbose + self.device = None + self.marionette_port = marionette_port + self.profile = None + self.remote_profile = None + self.log_file = None + self.remote_test_root = remote_test_root + self._adb_fh = None + + def _set_adb_logger(self, log_file): + self.log_file = log_file + if self.log_file is None: + return + logger.info("Setting ADB log file to %s" % self.log_file) + adb_logger = logging.getLogger("adb") + adb_logger.setLevel(logging.DEBUG) + self._adb_fh = logging.FileHandler(self.log_file) + self._adb_fh.setLevel(logging.DEBUG) + adb_logger.addHandler(self._adb_fh) + + def _unset_adb_logger(self): + if self._adb_fh is None: + return + logging.getLogger("adb").removeHandler(self._adb_fh) + self._adb_fh = None + + def clear_logcat(self, timeout=None, buffers=[]): + if not self.device: + return + self.device.clear_logcat(timeout, buffers) + + def get_logcat(self): + if not self.device: + return None + # we don't want to have ADBCommand dump the command + # in the debug stream so we reduce its verbosity here + # temporarely + old_verbose = self.device._verbose + self.device._verbose = False + try: + return self.device.get_logcat() + finally: + self.device._verbose = old_verbose + + def prepare(self, profile, logfile): + self._set_adb_logger(logfile) + try: + # See android_emulator_pgo.py run_tests for more + # details on why test_root must be /sdcard/test_root + # for android pgo due to Android 4.3. + self.device = ADBDeviceFactory( + verbose=self.verbose, logger_name="adb", test_root=self.remote_test_root + ) + except Exception: + logger.error("Cannot initialize device") + raise + device = self.device + self.profile = profile + + # checking that the app is installed + if not device.is_app_installed(self.app_name): + raise Exception("%s is not installed" % self.app_name) + + # debug flag + logger.info("Setting %s as the debug app on the phone" % self.app_name) + device.shell( + "am set-debug-app --persistent %s" % self.app_name, + stdout_callback=logger.info, + ) + + # creating the profile on the device + logger.info("Creating the profile on the device") + + remote_profile = posixpath.join(self.remote_test_root, "profile") + logger.info("The profile on the phone will be at %s" % remote_profile) + + device.rm(remote_profile, force=True, recursive=True) + logger.info("Pushing %s on the phone" % self.profile) + device.push(profile, remote_profile) + device.chmod(remote_profile, recursive=True) + self.profile = profile + self.remote_profile = remote_profile + + # creating the yml file + yml_data = { + "args": ["-marionette", "-profile", self.remote_profile], + "prefs": DEFAULT_PREFS, + "env": {"LOG_VERBOSE": 1, "R_LOG_LEVEL": 6, "MOZ_LOG": ""}, + } + + yml_name = "%s-geckoview-config.yaml" % self.app_name + yml_on_host = posixpath.join(tempfile.mkdtemp(), yml_name) + write_yml_file(yml_on_host, yml_data) + tmp_on_device = posixpath.join("/data", "local", "tmp") + if not device.exists(tmp_on_device): + raise IOError("%s does not exists on the device" % tmp_on_device) + yml_on_device = posixpath.join(tmp_on_device, yml_name) + try: + device.rm(yml_on_device, force=True, recursive=True) + device.push(yml_on_host, yml_on_device) + device.chmod(yml_on_device, recursive=True) + except Exception: + logger.info("could not create the yaml file on device. Permission issue?") + raise + + # command line 'extra' args not used with geckoview apps; instead we use + # an on-device config.yml file + intent = "android.intent.action.VIEW" + device.stop_application(self.app_name) + device.launch_application( + self.app_name, self.activity, intent, extras=None, url="about:blank" + ) + if not device.process_exist(self.app_name): + raise Exception("Could not start %s" % self.app_name) + + logger.info("Creating socket forwarding on port %d" % self.marionette_port) + device.forward( + local="tcp:%d" % self.marionette_port, + remote="tcp:%d" % self.marionette_port, + ) + + # we don't have a clean way for now to check that GV or Fenix + # is ready to handle our tests. So here we just wait 30s + logger.info("Sleeping for 30s") + time.sleep(30) + + def stop_browser(self): + logger.info("Stopping %s" % self.app_name) + try: + self.device.stop_application(self.app_name) + except ADBError: + logger.info("Could not stop the application using force-stop") + + time.sleep(5) + if self.device.process_exist(self.app_name): + logger.info("%s still running, trying SIGKILL" % self.app_name) + num_tries = 0 + while self.device.process_exist(self.app_name) and num_tries < 5: + try: + self.device.pkill(self.app_name) + except ADBError: + pass + num_tries += 1 + time.sleep(1) + logger.info("%s stopped" % self.app_name) + + def collect_profile(self): + logger.info("Collecting profile from %s" % self.remote_profile) + self.device.pull(self.remote_profile, self.profile) + + def close(self): + self._unset_adb_logger() + if self.device is None: + return + try: + self.device.remove_forwards("tcp:%d" % self.marionette_port) + except ADBError: + logger.warning("Could not remove forward port") + + +# XXX redundant, remove +@contextlib.contextmanager +def device( + app_name, marionette_port=2828, verbose=True, remote_test_root="/sdcard/test_root/" +): + device_ = AndroidDevice( + app_name, marionette_port, verbose, remote_test_root=remote_test_root + ) + try: + yield device_ + finally: + device_.close() + + +@attr.s +class AndroidGeckodriver(Geckodriver): + async def start(self): + port = free_port() + await self._check_version() + logger.info("Running Webdriver on port %d" % port) + logger.info("Running Marionette on port 2828") + pargs = [ + self.binary, + "--log", + "trace", + "--port", + str(port), + "--marionette-port", + "2828", + ] + logger.info("Connecting on Android device") + pargs.append("--connect-existing") + return await subprocess_based_service( + pargs, f"http://localhost:{port}", self.log_file + ) + + +class AndroidEnv(BaseEnv): + @contextlib.contextmanager + def get_device(self, *args, **kw): + with device(self.firefox, *args, **kw) as d: + self.device = d + yield self.device + + def get_target_platform(self): + app = self.firefox.split("org.mozilla.")[-1] + if self.device_name is None: + return app + return "%s-%s" % (self.device_name, app) + + def dump_logs(self): + logger.info("Dumping Android logs") + try: + logcat = self.device.get_logcat() + if logcat: + # local path, not using posixpath + logfile = os.path.join(self.archive, "logcat.log") + logger.info("Writing logcat at %s" % logfile) + with open(logfile, "wb") as f: + for line in logcat: + f.write(line.encode("utf8", errors="replace") + b"\n") + else: + logger.info("logcat came back empty") + except Exception: + logger.error("Could not extract the logcat", exc_info=True) + + @contextlib.contextmanager + def get_browser(self): + yield + + def get_browser_args(self, headless, prefs=None): + # XXX merge with DesktopEnv.get_browser_args + options = ["--allow-downgrade"] + if headless: + options.append("-headless") + if prefs is None: + prefs = {} + return {"moz:firefoxOptions": {"args": options, "prefs": prefs}} + + def prepare(self, logfile): + self.device.prepare(self.profile, logfile) + + def get_browser_version(self): + return self.target_platform + "-XXXneedtograbversion" + + def get_geckodriver(self, log_file): + return AndroidGeckodriver(binary=self.geckodriver, log_file=log_file) + + def check_session(self, session): + async def fake_close(*args): + pass + + session.close = fake_close + + def collect_profile(self): + self.device.collect_profile() + + def stop_browser(self): + self.device.stop_browser() diff --git a/testing/condprofile/condprof/archiver.py b/testing/condprofile/condprof/archiver.py new file mode 100644 index 0000000000..db7fc94347 --- /dev/null +++ b/testing/condprofile/condprof/archiver.py @@ -0,0 +1,76 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +"""Helper to create tarballs. +""" +import copy +import glob +import os +import tarfile + +from condprof import progress +from condprof.util import TASK_CLUSTER + + +def _tarinfo2mem(tar, tarinfo): + metadata = copy.copy(tarinfo) + try: + data = tar.extractfile(tarinfo) + if data is not None: + data = data.read() + except Exception: + data = None + + return metadata, data + + +class Archiver(object): + def __init__(self, scenario, profile_dir, archives_dir): + self.profile_dir = profile_dir + self.archives_dir = archives_dir + self.scenario = scenario + + def _strftime(self, date, template="-%Y-%m-%d-hp.tar.gz"): + return date.strftime(self.scenario + template) + + def _get_archive_path(self, when): + archive = self._strftime(when) + return os.path.join(self.archives_dir, archive), archive + + def create_archive(self, when, iterator=None): + if iterator is None: + + def _files(tar): + files = glob.glob(os.path.join(self.profile_dir, "*")) + yield len(files) + for filename in files: + try: + tar.add(filename, os.path.basename(filename)) + yield filename + except FileNotFoundError: # NOQA + # locks and such + pass + + iterator = _files + + if isinstance(when, str): + archive = when + else: + archive, __ = self._get_archive_path(when) + + with tarfile.open(archive, "w:gz", dereference=True) as tar: + it = iterator(tar) + size = next(it) + with progress.Bar(expected_size=size) as bar: + for filename in it: + if not TASK_CLUSTER: + bar.show(bar.last_progress + 1) + + return archive + + def _read_tar(self, filename): + files = {} + with tarfile.open(filename, "r:gz") as tar: + for tarinfo in tar: + files[tarinfo.name] = _tarinfo2mem(tar, tarinfo) + return files diff --git a/testing/condprofile/condprof/changelog.py b/testing/condprofile/condprof/changelog.py new file mode 100644 index 0000000000..72c8e7e8ea --- /dev/null +++ b/testing/condprofile/condprof/changelog.py @@ -0,0 +1,56 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This module needs to stay Python 2 and 3 compatible +# +""" +Maintains a unique file that lists all artifacts operations. +""" +import json +import os +import sys +from datetime import datetime + + +# XXX we should do one per platform and use platform-changelog.json as a name +class Changelog: + def __init__(self, archives_dir): + self.archives_dir = archives_dir + self.location = os.path.join(archives_dir, "changelog.json") + if os.path.exists(self.location): + with open(self.location) as f: + self._data = json.loads(f.read()) + if "changes" not in self._data: + self._data["changes"] = [] + else: + self._data = {"changes": []} + + def append(self, action, platform=sys.platform, **metadata): + now = datetime.timestamp(datetime.now()) + log = {"action": action, "platform": platform, "when": now} + log.update(metadata) + # adding taskcluster specific info if we see it in the env + for key in ( + "TC_SCHEDULER_ID", + "TASK_ID", + "TC_OWNER", + "TC_SOURCE", + "TC_PROJECT", + ): + if key in os.environ: + log[key] = os.environ[key] + self._data["changes"].append(log) + + def save(self, archives_dir=None): + if archives_dir is not None and archives_dir != self.archives_dir: + self.location = os.path.join(archives_dir, "changelog.json") + # we need to resolve potential r/w conflicts on TC here + with open(self.location, "w") as f: + f.write(json.dumps(self._data)) + + def history(self): + """From older to newer""" + return sorted( + self._data["changes"], key=lambda entry: entry["when"], reverse=True + ) diff --git a/testing/condprofile/condprof/check_install.py b/testing/condprofile/condprof/check_install.py new file mode 100644 index 0000000000..0209db3a55 --- /dev/null +++ b/testing/condprofile/condprof/check_install.py @@ -0,0 +1,71 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" Installs dependencies at runtime to simplify deployment. + +This module tries to make sure we have all dependencies installed on +all our environments. +""" +import os +import subprocess +import sys + +PY3 = sys.version_info.major == 3 +TOPDIR = os.path.join(os.path.dirname(__file__), "..") + + +def install_reqs(): + """We install requirements one by one, with no cache, and in isolated mode.""" + try: + import yaml # NOQA + + return False + except Exception: + # we're detecting here that this is running in Taskcluster + # by checking for the presence of the mozfile directory + # that was decompressed from target.condprof.tests.tar.gz + run_in_ci = os.path.exists(os.path.join(TOPDIR, "mozfile")) + + # On Python 2 we only install what's required for condprof.client + # On Python 3 it's the full thing + if not run_in_ci: + req_files = PY3 and ["base.txt", "local.txt"] or ["local-client.txt"] + else: + req_files = PY3 and ["base.txt", "ci.txt"] or ["ci-client.txt"] + + for req_file in req_files: + req_file = os.path.join(TOPDIR, "requirements", req_file) + + with open(req_file) as f: + reqs = [ + req + for req in f.read().split("\n") + if req.strip() != "" and not req.startswith("#") + ] + for req in reqs: + subprocess.check_call( + [ + sys.executable, + "-m", + "pip", + "install", + "--no-cache-dir", + "--isolated", + "--find-links", + "https://pypi.pub.build.mozilla.org/pub/", + req, + ] + ) + + return True + + +def check(): + """Called by the runner. + + The check function will restart the app after + all deps have been installed. + """ + if install_reqs(): + os.execl(sys.executable, sys.executable, *sys.argv) + os._exit(0) diff --git a/testing/condprofile/condprof/client.py b/testing/condprofile/condprof/client.py new file mode 100644 index 0000000000..186e8cf4c4 --- /dev/null +++ b/testing/condprofile/condprof/client.py @@ -0,0 +1,255 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This module needs to stay Python 2 and 3 compatible +# +import datetime +import functools +import os +import shutil +import tarfile +import tempfile +import time + +from mozprofile.prefs import Preferences + +from condprof import progress +from condprof.changelog import Changelog +from condprof.util import ( + TASK_CLUSTER, + ArchiveNotFound, + check_exists, + download_file, + logger, +) + +TC_SERVICE = "https://firefox-ci-tc.services.mozilla.com" +ROOT_URL = TC_SERVICE + "/api/index" +INDEX_PATH = "gecko.v2.%(repo)s.latest.firefox.condprof-%(platform)s-%(scenario)s" +INDEX_BY_DATE_PATH = "gecko.v2.%(repo)s.pushdate.%(date)s.latest.firefox.condprof-%(platform)s-%(scenario)s" +PUBLIC_DIR = "artifacts/public/condprof" +TC_LINK = ROOT_URL + "/v1/task/" + INDEX_PATH + "/" + PUBLIC_DIR + "/" +TC_LINK_BY_DATE = ROOT_URL + "/v1/task/" + INDEX_BY_DATE_PATH + "/" + PUBLIC_DIR + "/" +ARTIFACT_NAME = "profile%(version)s-%(platform)s-%(scenario)s-%(customization)s.tgz" +CHANGELOG_LINK = ( + ROOT_URL + "/v1/task/" + INDEX_PATH + "/" + PUBLIC_DIR + "/changelog.json" +) +ARTIFACTS_SERVICE = "https://taskcluster-artifacts.net" +DIRECT_LINK = ARTIFACTS_SERVICE + "/%(task_id)s/0/public/condprof/" +CONDPROF_CACHE = "~/.condprof-cache" +RETRIES = 3 +RETRY_PAUSE = 45 + + +class ServiceUnreachableError(Exception): + pass + + +class ProfileNotFoundError(Exception): + pass + + +class RetriesError(Exception): + pass + + +def _check_service(url): + """Sanity check to see if we can reach the service root url.""" + + def _check(): + exists, _ = check_exists(url, all_types=True) + if not exists: + raise ServiceUnreachableError(url) + + try: + return _retries(_check) + except RetriesError: + raise ServiceUnreachableError(url) + + +def _check_profile(profile_dir): + """Checks for prefs we need to remove or set.""" + to_remove = ("gfx.blacklist.", "marionette.") + + def _keep_pref(name, value): + for item in to_remove: + if not name.startswith(item): + continue + logger.info("Removing pref %s: %s" % (name, value)) + return False + return True + + def _clean_pref_file(name): + js_file = os.path.join(profile_dir, name) + prefs = Preferences.read_prefs(js_file) + cleaned_prefs = dict([pref for pref in prefs if _keep_pref(*pref)]) + if name == "prefs.js": + # When we start Firefox, forces startupScanScopes to SCOPE_PROFILE (1) + # otherwise, side loading will be deactivated and the + # Raptor web extension won't be able to run. + cleaned_prefs["extensions.startupScanScopes"] = 1 + + # adding a marker so we know it's a conditioned profile + cleaned_prefs["profile.conditioned"] = True + + with open(js_file, "w") as f: + Preferences.write(f, cleaned_prefs) + + _clean_pref_file("prefs.js") + _clean_pref_file("user.js") + + +def _retries(callable, onerror=None, retries=RETRIES): + _retry_count = 0 + pause = RETRY_PAUSE + + while _retry_count < retries: + try: + return callable() + except Exception as e: + if onerror is not None: + onerror(e) + logger.info("Failed, retrying") + _retry_count += 1 + time.sleep(pause) + pause *= 1.5 + + # If we reach that point, it means all attempts failed + if _retry_count >= RETRIES: + logger.error("All attempt failed") + else: + logger.info("Retried %s attempts and failed" % _retry_count) + raise RetriesError() + + +def get_profile( + target_dir, + platform, + scenario, + customization="default", + task_id=None, + download_cache=True, + repo="mozilla-central", + remote_test_root="/sdcard/test_root/", + version=None, + retries=RETRIES, +): + """Extract a conditioned profile in the target directory. + + If task_id is provided, will grab the profile from that task. when not + provided (default) will grab the latest profile. + """ + + # XXX assert values + if version: + version = "-v%s" % version + else: + version = "" + + # when we bump the Firefox version on trunk, autoland still needs to catch up + # in this case we want to download an older profile- 2 days to account for closures/etc. + oldday = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=2) + params = { + "platform": platform, + "scenario": scenario, + "customization": customization, + "task_id": task_id, + "repo": repo, + "version": version, + "date": str(oldday.date()).replace("-", "."), + } + logger.info("Getting conditioned profile with arguments: %s" % params) + filename = ARTIFACT_NAME % params + if task_id is None: + url = TC_LINK % params + filename + _check_service(TC_SERVICE) + else: + url = DIRECT_LINK % params + filename + _check_service(ARTIFACTS_SERVICE) + + logger.info("preparing download dir") + if not download_cache: + download_dir = tempfile.mkdtemp() + else: + # using a cache dir in the user home dir + download_dir = os.path.expanduser(CONDPROF_CACHE) + if not os.path.exists(download_dir): + os.makedirs(download_dir) + + downloaded_archive = os.path.join(download_dir, filename) + logger.info("Downloaded archive path: %s" % downloaded_archive) + + def _get_profile(): + logger.info("Getting %s" % url) + try: + archive = download_file(url, target=downloaded_archive) + except ArchiveNotFound: + raise ProfileNotFoundError(url) + try: + with tarfile.open(archive, "r:gz") as tar: + logger.info("Extracting the tarball content in %s" % target_dir) + size = len(list(tar)) + with progress.Bar(expected_size=size) as bar: + + def _extract(self, *args, **kw): + if not TASK_CLUSTER: + bar.show(bar.last_progress + 1) + return self.old(*args, **kw) + + tar.old = tar.extract + tar.extract = functools.partial(_extract, tar) + tar.extractall(target_dir) + except (OSError, tarfile.ReadError) as e: + logger.info("Failed to extract the tarball") + if download_cache and os.path.exists(archive): + logger.info("Removing cached file to attempt a new download") + os.remove(archive) + raise ProfileNotFoundError(str(e)) + finally: + if not download_cache: + shutil.rmtree(download_dir) + + _check_profile(target_dir) + logger.info("Success, we have a profile to work with") + return target_dir + + def onerror(error): + logger.info("Failed to get the profile.") + if os.path.exists(downloaded_archive): + try: + os.remove(downloaded_archive) + except Exception: + logger.error("Could not remove the file") + + try: + return _retries(_get_profile, onerror, retries) + except RetriesError: + # look for older profile 2 days previously + filename = ARTIFACT_NAME % params + url = TC_LINK_BY_DATE % params + filename + try: + return _retries(_get_profile, onerror, retries) + except RetriesError: + raise ProfileNotFoundError(url) + + +def read_changelog(platform, repo="mozilla-central", scenario="settled"): + params = {"platform": platform, "repo": repo, "scenario": scenario} + changelog_url = CHANGELOG_LINK % params + logger.info("Getting %s" % changelog_url) + download_dir = tempfile.mkdtemp() + downloaded_changelog = os.path.join(download_dir, "changelog.json") + + def _get_changelog(): + try: + download_file(changelog_url, target=downloaded_changelog) + except ArchiveNotFound: + shutil.rmtree(download_dir) + raise ProfileNotFoundError(changelog_url) + return Changelog(download_dir) + + try: + return _retries(_get_changelog) + except Exception: + raise ProfileNotFoundError(changelog_url) diff --git a/testing/condprofile/condprof/creator.py b/testing/condprofile/condprof/creator.py new file mode 100644 index 0000000000..7e03f6a8da --- /dev/null +++ b/testing/condprofile/condprof/creator.py @@ -0,0 +1,226 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" Creates or updates profiles. + +The profile creation works as following: + +For each scenario: + +- The latest indexed profile is picked on TC, if none we create a fresh profile +- The scenario is done against it +- The profile is uploaded on TC, replacing the previous one as the freshest + +For each platform we keep a changelog file that keep track of each update +with the Task ID. That offers us the ability to get a profile from a specific +date in the past. + +Artifacts are staying in TaskCluster for 3 months, and then they are removed, +so the oldest profile we can get is 3 months old. Profiles are being updated +continuously, so even after 3 months they are still getting "older". + +When Firefox changes its version, profiles from the previous version +should work as expected. Each profile tarball comes with a metadata file +that keep track of the Firefox version that was used and the profile age. +""" +import os +import tempfile +import shutil + +from arsenic import get_session +from arsenic.browsers import Firefox + +from condprof.util import fresh_profile, logger, obfuscate_file, obfuscate, get_version +from condprof.helpers import close_extra_windows +from condprof.scenarii import scenarii +from condprof.client import get_profile, ProfileNotFoundError +from condprof.archiver import Archiver +from condprof.customization import get_customization +from condprof.metadata import Metadata + + +START, INIT_GECKODRIVER, START_SESSION, START_SCENARIO = range(4) + + +class ProfileCreator: + def __init__( + self, + scenario, + customization, + archive, + changelog, + force_new, + env, + skip_logs=False, + remote_test_root="/sdcard/test_root/", + ): + self.env = env + self.scenario = scenario + self.customization = customization + self.archive = archive + self.changelog = changelog + self.force_new = force_new + self.skip_logs = skip_logs + self.remote_test_root = remote_test_root + self.customization_data = get_customization(customization) + self.tmp_dir = None + + # Make a temporary directory for the logs if an + # archive dir is not provided + if not self.archive: + self.tmp_dir = tempfile.mkdtemp() + + def _log_filename(self, name): + filename = "%s-%s-%s.log" % ( + name, + self.scenario, + self.customization_data["name"], + ) + return os.path.join(self.archive or self.tmp_dir, filename) + + async def run(self, headless=True): + logger.info( + "Building %s x %s" % (self.scenario, self.customization_data["name"]) + ) + + if self.scenario in self.customization_data.get("ignore_scenario", []): + logger.info("Skipping (ignored scenario in that customization)") + return + + filter_by_platform = self.customization_data.get("platforms") + if filter_by_platform and self.env.target_platform not in filter_by_platform: + logger.info("Skipping (ignored platform in that customization)") + return + + with self.env.get_device( + 2828, verbose=True, remote_test_root=self.remote_test_root + ) as device: + try: + with self.env.get_browser(): + metadata = await self.build_profile(device, headless) + except Exception: + raise + finally: + if not self.skip_logs: + self.env.dump_logs() + + if not self.archive: + return + + logger.info("Creating generic archive") + names = ["profile-%(platform)s-%(name)s-%(customization)s.tgz"] + if metadata["name"] == "full" and metadata["customization"] == "default": + names = [ + "profile-%(platform)s-%(name)s-%(customization)s.tgz", + "profile-v%(version)s-%(platform)s-%(name)s-%(customization)s.tgz", + ] + + for name in names: + # remove `cache` from profile + shutil.rmtree(os.path.join(self.env.profile, "cache"), ignore_errors=True) + shutil.rmtree(os.path.join(self.env.profile, "cache2"), ignore_errors=True) + + archiver = Archiver(self.scenario, self.env.profile, self.archive) + # the archive name is of the form + # profile[-vXYZ.x]-<platform>-<scenario>-<customization>.tgz + name = name % metadata + archive_name = os.path.join(self.archive, name) + dir = os.path.dirname(archive_name) + if not os.path.exists(dir): + os.makedirs(dir) + archiver.create_archive(archive_name) + logger.info("Archive created at %s" % archive_name) + statinfo = os.stat(archive_name) + logger.info("Current size is %d" % statinfo.st_size) + + logger.info("Extracting logs") + if "logs" in metadata: + logs = metadata.pop("logs") + for prefix, prefixed_logs in logs.items(): + for log in prefixed_logs: + content = obfuscate(log["content"])[1] + with open(os.path.join(dir, prefix + "-" + log["name"]), "wb") as f: + f.write(content.encode("utf-8")) + + if metadata.get("result", 0) != 0: + logger.info("The scenario returned a bad exit code") + raise Exception(metadata.get("result_message", "scenario error")) + self.changelog.append("update", **metadata) + + async def build_profile(self, device, headless): + scenario = self.scenario + profile = self.env.profile + customization_data = self.customization_data + + scenario_func = scenarii[scenario] + if scenario in customization_data.get("scenario", {}): + options = customization_data["scenario"][scenario] + logger.info("Loaded options for that scenario %s" % str(options)) + else: + options = {} + + # Adding general options + options["platform"] = self.env.target_platform + + if not self.force_new: + try: + custom_name = customization_data["name"] + get_profile(profile, self.env.target_platform, scenario, custom_name) + except ProfileNotFoundError: + # XXX we'll use a fresh profile for now + fresh_profile(profile, customization_data) + else: + fresh_profile(profile, customization_data) + + logger.info("Updating profile located at %r" % profile) + metadata = Metadata(profile) + + logger.info("Starting the Gecko app...") + adb_logs = self._log_filename("adb") + self.env.prepare(logfile=adb_logs) + geckodriver_logs = self._log_filename("geckodriver") + logger.info("Writing geckodriver logs in %s" % geckodriver_logs) + step = START + try: + firefox_instance = Firefox(**self.env.get_browser_args(headless)) + step = INIT_GECKODRIVER + with open(geckodriver_logs, "w") as glog: + geckodriver = self.env.get_geckodriver(log_file=glog) + step = START_SESSION + async with get_session(geckodriver, firefox_instance) as session: + step = START_SCENARIO + self.env.check_session(session) + logger.info("Running the %s scenario" % scenario) + metadata.update(await scenario_func(session, options)) + logger.info("%s scenario done." % scenario) + await close_extra_windows(session) + except Exception: + logger.error("%s scenario broke!" % scenario) + if step == START: + logger.info("Could not initialize the browser") + elif step == INIT_GECKODRIVER: + logger.info("Could not initialize Geckodriver") + elif step == START_SESSION: + logger.info( + "Could not start the session, check %s first" % geckodriver_logs + ) + else: + logger.info("Could not run the scenario, probably a faulty scenario") + raise + finally: + self.env.stop_browser() + for logfile in (adb_logs, geckodriver_logs): + if os.path.exists(logfile): + obfuscate_file(logfile) + self.env.collect_profile() + + # writing metadata + metadata.write( + name=self.scenario, + customization=self.customization_data["name"], + version=self.env.get_browser_version(), + platform=self.env.target_platform, + ) + + logger.info("Profile at %s.\nDone." % profile) + return metadata diff --git a/testing/condprofile/condprof/customization/__init__.py b/testing/condprofile/condprof/customization/__init__.py new file mode 100644 index 0000000000..64f09bf4f7 --- /dev/null +++ b/testing/condprofile/condprof/customization/__init__.py @@ -0,0 +1,35 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os + +HERE = os.path.dirname(__file__) + + +def get_customizations(): + for f in os.listdir(HERE): + if not f.endswith("json"): + continue + yield os.path.join(HERE, f) + + +def find_customization(path_or_name): + if not path_or_name.endswith(".json"): + path_or_name += ".json" + if not os.path.exists(path_or_name): + # trying relative + rpath = os.path.join(HERE, path_or_name) + if not os.path.exists(rpath): + return None + path_or_name = rpath + return path_or_name + + +def get_customization(path_or_name): + path = find_customization(path_or_name) + if path is None: + raise IOError("Can't find the customization file %r" % path_or_name) + with open(path) as f: + return json.loads(f.read()) diff --git a/testing/condprofile/condprof/customization/default.json b/testing/condprofile/condprof/customization/default.json new file mode 100644 index 0000000000..eb1ca25e91 --- /dev/null +++ b/testing/condprofile/condprof/customization/default.json @@ -0,0 +1,10 @@ +{ + "name": "default", + "addons": {}, + "prefs": { + "gfx.webrender.precache-shaders": true + }, + "scenario": { + "full": { "max_urls": 150 } + } +} diff --git a/testing/condprofile/condprof/customization/webext.json b/testing/condprofile/condprof/customization/webext.json new file mode 100644 index 0000000000..6647765387 --- /dev/null +++ b/testing/condprofile/condprof/customization/webext.json @@ -0,0 +1,15 @@ +{ + "name": "webext", + "addons": { + "uBlock": "https://addons.mozilla.org/firefox/downloads/file/4171020/ublock_origin-1.52.2.xpi" + }, + "prefs": { + "gfx.webrender.precache-shaders": true, + "extensions.autoDisableScopes": 0, + "extensions.enabledScopes": 1, + "extensions.startupScanScopes": 1 + }, + "scenario": { + "full": { "max_urls": 150 } + } +} diff --git a/testing/condprofile/condprof/customization/youtube.json b/testing/condprofile/condprof/customization/youtube.json new file mode 100644 index 0000000000..0d05ea0ed9 --- /dev/null +++ b/testing/condprofile/condprof/customization/youtube.json @@ -0,0 +1,12 @@ +{ + "name": "youtube", + "addons": {}, + "prefs": { + "media.eme.enabled": true, + "media.gmp-manager.updateEnabled": true, + "media.eme.require-app-approval": false + }, + "scenario": { + "full": { "max_urls": 150 } + } +} diff --git a/testing/condprofile/condprof/desktop.py b/testing/condprofile/condprof/desktop.py new file mode 100644 index 0000000000..7f8fe4e309 --- /dev/null +++ b/testing/condprofile/condprof/desktop.py @@ -0,0 +1,90 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import os +import contextlib + +import attr +from arsenic.services import Geckodriver, free_port, subprocess_based_service + +from condprof.util import ( + BaseEnv, + latest_nightly, + logger, + get_version, + get_current_platform, + DEFAULT_PREFS, +) + + +@attr.s +class DesktopGeckodriver(Geckodriver): + async def start(self): + port = free_port() + await self._check_version() + logger.info("Running Webdriver on port %d" % port) + logger.info("Running Marionette on port 2828") + pargs = [ + self.binary, + "--log", + "trace", + "--port", + str(port), + "--marionette-port", + "2828", + ] + try: + return await subprocess_based_service( + pargs, f"http://localhost:{port}", self.log_file + ) + except ProcessLookupError as e: + return await subprocess_based_service( + pargs.extend(["--host", "127.0.0.1"]), + f"http://localhost:{port}", + self.log_file, + ) + + +@contextlib.contextmanager +def dummy_device(*args, **kw): + yield None + + +class DesktopEnv(BaseEnv): + def get_target_platform(self): + return get_current_platform() + + def get_device(self, *args, **kw): + return dummy_device(*args, **kw) + + @contextlib.contextmanager + def get_browser(self): + with latest_nightly(self.firefox) as binary: + self.firefox = os.path.abspath(binary) + if not os.path.exists(self.firefox): + raise IOError(self.firefox) + yield + + def get_browser_args(self, headless, prefs=None): + final_prefs = dict(DEFAULT_PREFS) + if prefs is not None: + final_prefs.update(prefs) + options = ["--allow-downgrade", "-profile", self.profile] + if headless: + options.append("-headless") + args = {"moz:firefoxOptions": {"args": options}} + if self.firefox is not None: + args["moz:firefoxOptions"]["binary"] = self.firefox + args["moz:firefoxOptions"]["prefs"] = final_prefs + args["moz:firefoxOptions"]["log"] = {"level": "trace"} + return args + + def get_browser_version(self): + try: + return get_version(self.firefox) + except Exception: + logger.error("Could not get Firefox version", exc_info=True) + return "unknown" + + def get_geckodriver(self, log_file): + return DesktopGeckodriver(binary=self.geckodriver, log_file=log_file) diff --git a/testing/condprofile/condprof/helpers.py b/testing/condprofile/condprof/helpers.py new file mode 100644 index 0000000000..befe9e0331 --- /dev/null +++ b/testing/condprofile/condprof/helpers.py @@ -0,0 +1,110 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" Helpers to build scenarii +""" +from condprof.util import logger + +_SUPPORTED_MOBILE_BROWSERS = "fenix", "gecko", "firefox" + + +def is_mobile(platform): + return any(mobile in platform for mobile in _SUPPORTED_MOBILE_BROWSERS) + + +class TabSwitcher: + """Helper used to create tabs and circulate in them.""" + + def __init__(self, session, options): + self.handles = None + self.current = 0 + self.session = session + self._max = options.get("max_urls", 10) + self.platform = options.get("platform", "") + self.num_tabs = self._max >= 100 and 100 or self._max + self._mobile = is_mobile(self.platform) + + async def create_windows(self): + # on mobile we don't use tabs for now + # see https://bugzil.la/1559120 + if self._mobile: + return + # creating tabs + for i in range(self.num_tabs): + # see https://github.com/HDE/arsenic/issues/71 + await self.session._request( + url="/window/new", method="POST", data={"type": "tab"} + ) + + async def switch(self): + if self._mobile: + return + try: + if self.handles is None: + self.handles = await self.session.get_window_handles() + self.current = 0 + except Exception: + logger.error("Could not get window handles") + return + + handle = self.handles[self.current] + if self.current == len(self.handles) - 1: + self.current = 0 + else: + self.current += 1 + try: + await self.session.switch_to_window(handle) + except Exception: + logger.error("Could not switch to handle %s" % str(handle)) + + +# 10 minutes +_SCRIPT_TIMEOUT = 10 * 60 * 1000 + + +async def execute_async_script(session, script, *args): + # switch to the right context if needed + current_context = await session._request(url="/moz/context", method="GET") + if current_context != "chrome": + await session._request( + url="/moz/context", method="POST", data={"context": "chrome"} + ) + switch_back = True + else: + switch_back = False + await session._request( + url="/timeouts", method="POST", data={"script": _SCRIPT_TIMEOUT} + ) + try: + attempts = 0 + while True: + try: + return await session._request( + url="/execute/async", + method="POST", + data={"script": script, "args": list(args)}, + ) + except Exception as e: + attempts += 1 + logger.error("The script failed.", exc_info=True) + if attempts > 2: + return { + "result": 1, + "result_message": str(e), + "result_exc": e, + "logs": {}, + } + finally: + if switch_back: + await session._request( + url="/moz/context", method="POST", data={"context": current_context} + ) + + +async def close_extra_windows(session): + logger.info("Closing all tabs") + handles = await session.get_window_handles() + # we're closing all tabs except the last one + for handle in handles[:-1]: + await session.switch_to_window(handle) + await session._request(url="/window", method="DELETE") diff --git a/testing/condprofile/condprof/main.py b/testing/condprofile/condprof/main.py new file mode 100644 index 0000000000..b53e199220 --- /dev/null +++ b/testing/condprofile/condprof/main.py @@ -0,0 +1,86 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" Script that launches profiles creation. +""" +import argparse +import os +import sys + +# easier than setting PYTHONPATH in various platforms +if __name__ == "__main__": + sys.path.append(os.path.join(os.path.dirname(__file__), "..")) + +from condprof.check_install import check # NOQA + +if "MANUAL_MACH_RUN" not in os.environ: + check() + +from condprof import patch # noqa + + +def main(args=sys.argv[1:]): + parser = argparse.ArgumentParser(description="Profile Creator") + parser.add_argument("archive", help="Archives Dir", type=str, default=None) + parser.add_argument("--firefox", help="Firefox Binary", type=str, default=None) + parser.add_argument("--scenario", help="Scenario to use", type=str, default="all") + parser.add_argument( + "--profile", help="Existing profile Dir", type=str, default=None + ) + parser.add_argument( + "--customization", + help="Profile customization to use", + type=str, + default="default", + ) + parser.add_argument( + "--visible", help="Don't use headless mode", action="store_true", default=False + ) + parser.add_argument( + "--archives-dir", help="Archives local dir", type=str, default="/tmp/archives" + ) + parser.add_argument( + "--force-new", help="Create from scratch", action="store_true", default=False + ) + parser.add_argument( + "--strict", + help="Errors out immediatly on a scenario failure", + action="store_true", + default=False, + ) + parser.add_argument( + "--geckodriver", + help="Path to the geckodriver binary", + type=str, + default=sys.platform.startswith("win") and "geckodriver.exe" or "geckodriver", + ) + + parser.add_argument( + "--device-name", help="Name of the device", type=str, default=None + ) + + args = parser.parse_args(args=args) + os.environ["CONDPROF_RUNNER"] = "1" + + from condprof.runner import run # NOQA + + try: + run( + args.archive, + args.firefox, + args.scenario, + args.profile, + args.customization, + args.visible, + args.archives_dir, + args.force_new, + args.strict, + args.geckodriver, + args.device_name, + ) + except Exception: + sys.exit(4) # TBPL_RETRY + + +if __name__ == "__main__": + main() diff --git a/testing/condprofile/condprof/metadata.py b/testing/condprofile/condprof/metadata.py new file mode 100644 index 0000000000..4d03e57f6f --- /dev/null +++ b/testing/condprofile/condprof/metadata.py @@ -0,0 +1,83 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" Manages a metadata file. +""" +import datetime +import json +import os +from collections.abc import MutableMapping + +from condprof.util import logger + +METADATA_NAME = "condprofile.json" + + +class Metadata(MutableMapping): + """dict-like class that holds metadata for a profile.""" + + def __init__(self, profile_dir): + self.metadata_file = os.path.join(profile_dir, METADATA_NAME) + logger.info("Reading existing metadata at %s" % self.metadata_file) + if not os.path.exists(self.metadata_file): + logger.info("Could not find the metadata file in that profile") + self._data = {} + else: + with open(self.metadata_file) as f: + self._data = json.loads(f.read()) + + def __getitem__(self, key): + return self._data[self.__keytransform__(key)] + + def __setitem__(self, key, value): + self._data[self.__keytransform__(key)] = value + + def __delitem__(self, key): + del self._data[self.__keytransform__(key)] + + def __iter__(self): + return iter(self._data) + + def __len__(self): + return len(self._data) + + def __keytransform__(self, key): + return key + + def _days2age(self, days): + if days < 7: + return "days" + if days < 30: + return "weeks" + if days < 30 * 6: + return "months" + return "old" # :) + + def _delta(self, created, updated): + created = created[:26] + updated = updated[:26] + # tz.. + format = "%Y-%m-%d %H:%M:%S.%f" + created = datetime.datetime.strptime(created, format) + updated = datetime.datetime.strptime(updated, format) + delta = created - updated + return delta.days + + def write(self, **extras): + # writing metadata + logger.info("Creating metadata...") + self._data.update(**extras) + ts = str(datetime.datetime.now()) + if "created" not in self._data: + self._data["created"] = ts + self._data["updated"] = ts + # XXX need android arch version here + days = self._delta(self._data["created"], self._data["updated"]) + self._data["days"] = days + self._data["age"] = self._days2age(days) + # adding info about the firefox version + # XXX build ID ?? + # XXX android ?? + logger.info("Saving metadata file in %s" % self.metadata_file) + with open(self.metadata_file, "w") as f: + f.write(json.dumps(self._data)) diff --git a/testing/condprofile/condprof/patch.py b/testing/condprofile/condprof/patch.py new file mode 100644 index 0000000000..35e4c978b5 --- /dev/null +++ b/testing/condprofile/condprof/patch.py @@ -0,0 +1,47 @@ +# flake8: noqa +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# patch for https://bugzilla.mozilla.org/show_bug.cgi?id=1655869 +# see https://github.com/HDE/arsenic/issues/85 +from arsenic.connection import * + + +@ensure_task +async def request(self, *, url: str, method: str, data=None) -> Tuple[int, Any]: + if data is None: + data = {} + if method not in {"POST", "PUT"}: + data = None + headers = {} + else: + headers = {"Content-Type": "application/json"} + body = json.dumps(data) if data is not None else None + full_url = self.prefix + url + log.info( + "request", url=strip_auth(full_url), method=method, body=body, headers=headers + ) + + async with self.session.request( + url=full_url, method=method, data=body, headers=headers + ) as response: + response_body = await response.read() + try: + data = json.loads(response_body) + except JSONDecodeError as exc: + log.error("json-decode", body=response_body) + data = {"error": "!internal", "message": str(exc), "stacktrace": ""} + wrap_screen(data) + log.info( + "response", + url=strip_auth(full_url), + method=method, + body=body, + response=response, + data=data, + ) + return response.status, data + + +Connection.request = request diff --git a/testing/condprofile/condprof/progress.py b/testing/condprofile/condprof/progress.py new file mode 100644 index 0000000000..097af53a8c --- /dev/null +++ b/testing/condprofile/condprof/progress.py @@ -0,0 +1,223 @@ +# -*- coding: utf-8 -*- +""" +clint.textui.progress +~~~~~~~~~~~~~~~~~ +This module provides the progressbar functionality. + + +ISC License + +Copyright (c) 2011, Kenneth Reitz <me@kennethreitz.com> + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +""" + +import sys +import time + +STREAM = sys.stderr + +BAR_TEMPLATE = "%s[%s%s] %i/%i - %s\r" +MILL_TEMPLATE = "%s %s %i/%i\r" + +DOTS_CHAR = "." +BAR_FILLED_CHAR = "#" +BAR_EMPTY_CHAR = " " +MILL_CHARS = ["|", "/", "-", "\\"] + +# How long to wait before recalculating the ETA +ETA_INTERVAL = 1 +# How many intervals (excluding the current one) to calculate the simple moving +# average +ETA_SMA_WINDOW = 9 + + +class Bar(object): + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.done() + return False # we're not suppressing exceptions + + def __init__( + self, + label="", + width=32, + hide=None, + empty_char=BAR_EMPTY_CHAR, + filled_char=BAR_FILLED_CHAR, + expected_size=None, + every=1, + ): + self.label = label + self.width = width + self.hide = hide + # Only show bar in terminals by default (better for piping, logging etc.) + if hide is None: + try: + self.hide = not STREAM.isatty() + except AttributeError: # output does not support isatty() + self.hide = True + self.empty_char = empty_char + self.filled_char = filled_char + self.expected_size = expected_size + self.every = every + self.start = time.time() + self.ittimes = [] + self.eta = 0 + self.etadelta = time.time() + self.etadisp = self.format_time(self.eta) + self.last_progress = 0 + if self.expected_size: + self.show(0) + + def show(self, progress, count=None): + if count is not None: + self.expected_size = count + if self.expected_size is None: + raise Exception("expected_size not initialized") + self.last_progress = progress + if (time.time() - self.etadelta) > ETA_INTERVAL: + self.etadelta = time.time() + # pylint --py3k W1619 + self.ittimes = self.ittimes[-ETA_SMA_WINDOW:] + [ + -(self.start - time.time()) / (progress + 1) + ] + self.eta = ( + sum(self.ittimes) + / float(len(self.ittimes)) + * (self.expected_size - progress) + ) + self.etadisp = self.format_time(self.eta) + # pylint --py3k W1619 + x = int(self.width * progress / self.expected_size) + if not self.hide: + if (progress % self.every) == 0 or ( # True every "every" updates + progress == self.expected_size + ): # And when we're done + STREAM.write( + BAR_TEMPLATE + % ( + self.label, + self.filled_char * x, + self.empty_char * (self.width - x), + progress, + self.expected_size, + self.etadisp, + ) + ) + STREAM.flush() + + def done(self): + self.elapsed = time.time() - self.start + elapsed_disp = self.format_time(self.elapsed) + if not self.hide: + # Print completed bar with elapsed time + STREAM.write( + BAR_TEMPLATE + % ( + self.label, + self.filled_char * self.width, + self.empty_char * 0, + self.last_progress, + self.expected_size, + elapsed_disp, + ) + ) + STREAM.write("\n") + STREAM.flush() + + def format_time(self, seconds): + return time.strftime("%H:%M:%S", time.gmtime(seconds)) + + +def bar( + it, + label="", + width=32, + hide=None, + empty_char=BAR_EMPTY_CHAR, + filled_char=BAR_FILLED_CHAR, + expected_size=None, + every=1, +): + """Progress iterator. Wrap your iterables with it.""" + + count = len(it) if expected_size is None else expected_size + + with Bar( + label=label, + width=width, + hide=hide, + empty_char=empty_char, + filled_char=filled_char, + expected_size=count, + every=every, + ) as bar: + for i, item in enumerate(it): + yield item + bar.show(i + 1) + + +def dots(it, label="", hide=None, every=1): + """Progress iterator. Prints a dot for each item being iterated""" + + count = 0 + + if not hide: + STREAM.write(label) + + for i, item in enumerate(it): + if not hide: + if i % every == 0: # True every "every" updates + STREAM.write(DOTS_CHAR) + sys.stderr.flush() + + count += 1 + + yield item + + STREAM.write("\n") + STREAM.flush() + + +def mill(it, label="", hide=None, expected_size=None, every=1): + """Progress iterator. Prints a mill while iterating over the items.""" + + def _mill_char(_i): + if _i >= count: + return " " + else: + return MILL_CHARS[(_i // every) % len(MILL_CHARS)] + + def _show(_i): + if not hide: + if (_i % every) == 0 or ( # True every "every" updates + _i == count + ): # And when we're done + STREAM.write(MILL_TEMPLATE % (label, _mill_char(_i), _i, count)) + STREAM.flush() + + count = len(it) if expected_size is None else expected_size + + if count: + _show(0) + + for i, item in enumerate(it): + yield item + _show(i + 1) + + if not hide: + STREAM.write("\n") + STREAM.flush() diff --git a/testing/condprofile/condprof/runner.py b/testing/condprofile/condprof/runner.py new file mode 100644 index 0000000000..07fb6d3e73 --- /dev/null +++ b/testing/condprofile/condprof/runner.py @@ -0,0 +1,211 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" Script that launches profiles creation. +""" +import os +import shutil +import asyncio + +import mozversion + +from condprof.creator import ProfileCreator +from condprof.desktop import DesktopEnv +from condprof.android import AndroidEnv +from condprof.changelog import Changelog +from condprof.scenarii import scenarii +from condprof.util import logger, get_current_platform, extract_from_dmg +from condprof.customization import get_customizations, find_customization +from condprof.client import read_changelog, ProfileNotFoundError + + +class Runner: + def __init__( + self, + profile, + firefox, + geckodriver, + archive, + device_name, + strict, + force_new, + visible, + skip_logs=False, + remote_test_root="/sdcard/test_root/", + ): + self.force_new = force_new + self.profile = profile + self.geckodriver = geckodriver + self.archive = archive + self.device_name = device_name + self.strict = strict + self.visible = visible + self.skip_logs = skip_logs + self.remote_test_root = remote_test_root + self.env = {} + # unpacking a dmg + # XXX do something similar if we get an apk (but later) + # XXX we want to do + # adb install -r target.apk + # and get the installed app name + if firefox is not None and firefox.endswith("dmg"): + target = os.path.join(os.path.dirname(firefox), "firefox.app") + extract_from_dmg(firefox, target) + firefox = os.path.join(target, "Contents", "MacOS", "firefox") + self.firefox = firefox + self.android = self.firefox is not None and self.firefox.startswith( + "org.mozilla" + ) + + def prepare(self, scenario, customization): + self.scenario = scenario + self.customization = customization + + # early checks to avoid extra work + if self.customization != "all": + if find_customization(self.customization) is None: + raise IOError("Cannot find customization %r" % self.customization) + + if self.scenario != "all" and self.scenario not in scenarii: + raise IOError("Cannot find scenario %r" % self.scenario) + + if not self.android and self.firefox is not None: + logger.info("Verifying Desktop Firefox binary") + # we want to verify we do have a firefox binary + # XXX so lame + if not os.path.exists(self.firefox): + if "MOZ_FETCHES_DIR" in os.environ: + target = os.path.join(os.environ["MOZ_FETCHES_DIR"], self.firefox) + if os.path.exists(target): + self.firefox = target + + if not os.path.exists(self.firefox): + raise IOError("Cannot find %s" % self.firefox) + + mozversion.get_version(self.firefox) + + logger.info(os.environ) + if self.archive: + self.archive = os.path.abspath(self.archive) + logger.info("Archives directory is %s" % self.archive) + if not os.path.exists(self.archive): + os.makedirs(self.archive, exist_ok=True) + + if shutil.which(self.geckodriver) is None and not os.path.exists( + self.geckodriver + ): + raise IOError("Cannot find %s" % self.geckodriver) + + if not self.skip_logs: + try: + if self.android: + plat = "%s-%s" % ( + self.device_name, + self.firefox.split("org.mozilla.")[-1], + ) + else: + plat = get_current_platform() + self.changelog = read_changelog(plat) + logger.info("Got the changelog from TaskCluster") + except ProfileNotFoundError: + logger.info("changelog not found on TaskCluster, creating a local one.") + self.changelog = Changelog(self.archive) + else: + self.changelog = [] + + def _create_env(self): + if self.android: + klass = AndroidEnv + else: + klass = DesktopEnv + + return klass( + self.profile, self.firefox, self.geckodriver, self.archive, self.device_name + ) + + def display_error(self, scenario, customization): + logger.error("%s x %s failed." % (scenario, customization), exc_info=True) + # TODO: this might avoid the exceptions that slip through in automation + # if self.strict: + # raise + + async def one_run(self, scenario, customization): + """Runs one single conditioned profile. + + Create an instance of the environment and run the ProfileCreator. + """ + self.env = self._create_env() + return await ProfileCreator( + scenario, + customization, + self.archive, + self.changelog, + self.force_new, + self.env, + skip_logs=self.skip_logs, + remote_test_root=self.remote_test_root, + ).run(not self.visible) + + async def run_all(self): + """Runs the conditioned profile builders""" + if self.scenario != "all": + selected_scenario = [self.scenario] + else: + selected_scenario = scenarii.keys() + + # this is the loop that generates all combinations of profile + # for the current platform when "all" is selected + res = [] + failures = 0 + for scenario in selected_scenario: + if self.customization != "all": + try: + res.append(await self.one_run(scenario, self.customization)) + except Exception: + failures += 1 + self.display_error(scenario, self.customization) + else: + for customization in get_customizations(): + logger.info("Customization %s" % customization) + try: + res.append(await self.one_run(scenario, customization)) + except Exception: + failures += 1 + self.display_error(scenario, customization) + + return failures, [one_res for one_res in res if one_res] + + def save(self): + self.changelog.save(self.archive) + + +def run( + archive, + firefox=None, + scenario="all", + profile=None, + customization="all", + visible=False, + archives_dir="/tmp/archives", + force_new=False, + strict=True, + geckodriver="geckodriver", + device_name=None, +): + runner = Runner( + profile, firefox, geckodriver, archive, device_name, strict, force_new, visible + ) + + runner.prepare(scenario, customization) + loop = asyncio.get_event_loop() + + try: + failures, results = loop.run_until_complete(runner.run_all()) + logger.info("Saving changelog in %s" % archive) + runner.save() + if failures > 0: + raise Exception("At least one scenario failed") + except Exception as e: + raise e + finally: + loop.close() diff --git a/testing/condprofile/condprof/scenarii/__init__.py b/testing/condprofile/condprof/scenarii/__init__.py new file mode 100644 index 0000000000..b665843b2b --- /dev/null +++ b/testing/condprofile/condprof/scenarii/__init__.py @@ -0,0 +1,15 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from condprof.scenarii.full import full +from condprof.scenarii.settled import settled +from condprof.scenarii.settled2 import settled2 +from condprof.scenarii.settled_youtube import settled_youtube + +scenarii = { + "full": full, + "settled": settled, + "settled-youtube": settled_youtube, + "settled2": settled2, +} diff --git a/testing/condprofile/condprof/scenarii/bookmark.js b/testing/condprofile/condprof/scenarii/bookmark.js new file mode 100644 index 0000000000..70ef028320 --- /dev/null +++ b/testing/condprofile/condprof/scenarii/bookmark.js @@ -0,0 +1,52 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +// 'arguments' is defined by the marionette harness. +/* global arguments */ + +const { PlacesUtils } = ChromeUtils.importESModule( + "resource://gre/modules/PlacesUtils.sys.mjs" +); + +let resolve = arguments[3]; + +try { + // Get the number of current bookmarks + PlacesUtils.promiseBookmarksTree(PlacesUtils.bookmarks.unfiledGuid, { + includeItemIds: true, + }).then(root => { + let count = root.itemsCount; + let maxBookmarks = arguments[2]; + // making sure we don't exceed the maximum number of bookmarks + if (count >= maxBookmarks) { + let toRemove = count - maxBookmarks + 1; + console.log("We've reached the maximum number of bookmarks"); + console.log("Removing " + toRemove); + let children = root.children; + for (let i = 0, p = Promise.resolve(); i < toRemove; i++) { + p = p.then( + _ => + new Promise(resolve => + PlacesUtils.bookmarks.remove(children[i].guid).then(res => { + console.log("removed one bookmark"); + resolve(res); + }) + ) + ); + } + } + // now adding the bookmark + PlacesUtils.bookmarks + .insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: arguments[0], + title: arguments[1], + }) + .then(res => { + resolve(res); + }); + }); +} catch (error) { + let res = { logs: {}, result: 1, result_message: error.toString() }; + resolve(res); +} diff --git a/testing/condprofile/condprof/scenarii/full.py b/testing/condprofile/condprof/scenarii/full.py new file mode 100644 index 0000000000..ec93b48b46 --- /dev/null +++ b/testing/condprofile/condprof/scenarii/full.py @@ -0,0 +1,138 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import asyncio +import os +import random + +from arsenic.errors import UnknownArsenicError, UnknownError + +from condprof.helpers import TabSwitcher, execute_async_script, is_mobile +from condprof.util import get_credentials, logger + +BOOKMARK_FREQUENCY = 5 +MAX_URLS = 150 +MAX_BOOKMARKS = 250 +CallErrors = asyncio.TimeoutError, UnknownError, UnknownArsenicError + + +class Builder: + def __init__(self, options): + self.options = options + self.words = self._read_lines("words.txt") + self.urls = self._build_url_list(self._read_lines("urls.txt")) + self.sync_js = self._load_script("sync") + self.max_bookmarks = options.get("max_bookmarks", MAX_BOOKMARKS) + self.bookmark_js = self._load_script("bookmark") + self.platform = options.get("platform", "") + self.mobile = is_mobile(self.platform) + self.max_urls = options.get("max_urls", MAX_URLS) + + # see Bug 1608604 & see Bug 1619107 - we have stability issues @ bitbar + if self.mobile: + self.max_urls = min(self.max_urls, 20) + + logger.info("platform: %s" % self.platform) + logger.info("max_urls: %s" % self.max_urls) + self.bookmark_frequency = options.get("bookmark_frequency", BOOKMARK_FREQUENCY) + + # we're syncing only on desktop for now + self.syncing = not self.mobile + if self.syncing: + self.username, self.password = get_credentials() + if self.username is None: + raise ValueError("Sync operations need an FxA username and password") + else: + self.username, self.password = None, None + + def _load_script(self, name): + return "\n".join(self._read_lines("%s.js" % name)) + + def _read_lines(self, filename): + path = os.path.join(os.path.dirname(__file__), filename) + with open(path) as f: + return f.readlines() + + def _build_url_list(self, urls): + url_list = [] + for url in urls: + url = url.strip() + if url.startswith("#"): + continue + for word in self.words: + word = word.strip() + if word.startswith("#"): + continue + url_list.append((url.format(word), word)) + random.shuffle(url_list) + return url_list + + async def add_bookmark(self, session, url, title): + logger.info("Adding bookmark to %s" % url) + return await execute_async_script( + session, self.bookmark_js, url, title, self.max_bookmarks + ) + + async def sync(self, session, metadata): + if not self.syncing: + return + # now that we've visited all pages, we want to upload to FXSync + logger.info("Syncing profile to FxSync") + logger.info("Username is %s, password is %s" % (self.username, self.password)) + script_res = await execute_async_script( + session, + self.sync_js, + self.username, + self.password, + "https://accounts.stage.mozaws.net", + ) + if script_res is None: + script_res = {} + metadata["logs"] = script_res.get("logs", {}) + metadata["result"] = script_res.get("result", 0) + metadata["result_message"] = script_res.get("result_message", "SUCCESS") + return metadata + + async def _visit_url(self, current, session, url, word): + await asyncio.wait_for(session.get(url), 5) + if current % self.bookmark_frequency == 0 and not self.mobile: + await asyncio.wait_for(self.add_bookmark(session, url, word), 5) + + async def __call__(self, session): + metadata = {} + + tabs = TabSwitcher(session, self.options) + await tabs.create_windows() + visited = 0 + + for current, (url, word) in enumerate(self.urls): + logger.info("%d/%d %s" % (current + 1, self.max_urls, url)) + retries = 0 + while retries < 3: + try: + await self._visit_url(current, session, url, word) + visited += 1 + break + except CallErrors: + await asyncio.sleep(1.0) + retries += 1 + + if current == self.max_urls - 1: + break + + # switch to the next tab + try: + await asyncio.wait_for(tabs.switch(), 5) + except CallErrors: + # if we can't switch, it's ok + pass + + metadata["visited_url"] = visited + await self.sync(session, metadata) + return metadata + + +async def full(session, options): + builder = Builder(options) + return await builder(session) diff --git a/testing/condprofile/condprof/scenarii/settled.py b/testing/condprofile/condprof/scenarii/settled.py new file mode 100644 index 0000000000..eb54f4638b --- /dev/null +++ b/testing/condprofile/condprof/scenarii/settled.py @@ -0,0 +1,11 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import asyncio + + +async def settled(session, options): + # nothing is done, we just settle here for 30 seconds + await asyncio.sleep(options.get("sleep", 30)) + return {} diff --git a/testing/condprofile/condprof/scenarii/settled2.py b/testing/condprofile/condprof/scenarii/settled2.py new file mode 100644 index 0000000000..0bc828c4c3 --- /dev/null +++ b/testing/condprofile/condprof/scenarii/settled2.py @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import asyncio + + +async def settled2(session, options): + retries = 0 + while retries < 3: + try: + await asyncio.wait_for(session.get("https://www.mozilla.org/en-US/"), 5) + break + except asyncio.TimeoutError: + retries += 1 + + # we just settle here for 5 minutes + await asyncio.sleep(options.get("sleep", 5 * 60)) + return {} diff --git a/testing/condprofile/condprof/scenarii/settled_youtube.py b/testing/condprofile/condprof/scenarii/settled_youtube.py new file mode 100644 index 0000000000..b39c0874a9 --- /dev/null +++ b/testing/condprofile/condprof/scenarii/settled_youtube.py @@ -0,0 +1,20 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import asyncio + + +async def settled_youtube(session, options): + # nothing is done, we just settle here for 30 seconds + await asyncio.sleep(options.get("sleep", 5)) + await asyncio.wait_for( + session.get( + "https://yttest.prod.mozaws.net/2020/main.html?" + "test_type=playbackperf-widevine-sfr-h264-test&" + "raptor=true&exclude=1,2&muted=true&command=run" + ), + timeout=120, + ) + await asyncio.sleep(options.get("sleep", 30)) + return {} diff --git a/testing/condprofile/condprof/scenarii/sync.js b/testing/condprofile/condprof/scenarii/sync.js new file mode 100644 index 0000000000..0f7b7bb90d --- /dev/null +++ b/testing/condprofile/condprof/scenarii/sync.js @@ -0,0 +1,21 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const { triggerSync } = ChromeUtils.importESModule( + "resource://gre/modules/services-automation/ServicesAutomation.sys.mjs" +); + +// Arguments is expected to be provided in the scope that this file is loaded +// into. +/* global arguments */ + +let resolve = arguments[3]; +try { + triggerSync(arguments[0], arguments[1], arguments[2]).then(res => { + resolve(res); + }); +} catch (error) { + let res = { logs: {}, result: 1, result_message: error.toString() }; + resolve(res); +} diff --git a/testing/condprofile/condprof/scenarii/urls.txt b/testing/condprofile/condprof/scenarii/urls.txt new file mode 100644 index 0000000000..5dd86b024a --- /dev/null +++ b/testing/condprofile/condprof/scenarii/urls.txt @@ -0,0 +1,7 @@ +# urls combined with the words list +https://www.google.com/search?q={0} +https://search.yahoo.com/yhs/search?p={0} +https://www.bing.com/search?q=create+list+of+nouns{0} +https://www.amazon.com/s/ref=nb_sb_noss_2?url=search-alias%3Daps&field-keywords={0} +https://www.youtube.com/results?search_query={0} +https://www.ebay.com/sch/i.html?_from=R40&_trksid=p2380057.m570.l1313.TR0.TRC0.H0.Xbottle.TRS0&_nkw={0} diff --git a/testing/condprofile/condprof/scenarii/words.txt b/testing/condprofile/condprof/scenarii/words.txt new file mode 100644 index 0000000000..107f65180e --- /dev/null +++ b/testing/condprofile/condprof/scenarii/words.txt @@ -0,0 +1,4555 @@ +# word list from http://www.worldclasslearning.com/english/4000-most-common-english-words.html +aardvark +abacus +abbey +abdomen +ability +abolishment +abroad +abuse +accelerant +accelerator +access +accident +accommodation +accompanist +accordion +account +accountant +achiever +acid +acknowledgment +acoustic +acoustics +acrylic +act +action +activity +actor +actress +acupuncture +ad +adapter +addiction +addition +address +adjustment +administration +adrenalin +adult +adulthood +advance +advancement +advantage +advertisement +advertising +advice +affair +affect +aftermath +afternoon +aftershave +aftershock +afterthought +age +agency +agenda +agent +aggression +aglet +agreement +aid +air +airbag +airbus +airfare +airforce +airline +airmail +airplane +airport +airship +alarm +alb +albatross +alcohol +alcove +alder +algebra +alibi +allergist +alley +alligator +alloy +almanac +almond +alpaca +alpenglow +alpenhorn +alpha +alphabet +alternative +altitude +alto +aluminium +aluminum +ambassador +ambition +ambulance +amendment +amount +amusement +anagram +analgesia +analog +analogue +analogy +analysis +analyst +anatomy +anesthesiology +anethesiologist +anger +angiosperm +angle +angora +angstrom +anguish +animal +anime +ankle +anklet +annual +anorak +answer +ant +anteater +antechamber +antelope +anthony +anthropology +antler +anxiety +anybody +anything +anywhere +apartment +ape +aperitif +apology +apparatus +apparel +appeal +appearance +appendix +applause +apple +applewood +appliance +application +appointment +approval +apron +apse +aquifer +arch +archaeology +archeology +archer +architect +architecture +arch-rival +area +argument +arithmetic +arm +armadillo +armament +armchair +armoire +armor +arm-rest +army +arrival +arrow +art +artichoke +article +artificer +ascot +ash +ashram +ashtray +aside +ask +asparagus +aspect +asphalt +assignment +assist +assistance +assistant +associate +association +assumption +asterisk +astrakhan +astrolabe +astrologer +astrology +astronomy +atelier +athlete +athletics +atmosphere +atom +atrium +attachment +attack +attempt +attendant +attention +attenuation +attic +attitude +attorney +attraction +audience +auditorium +aunt +author +authorisation +authority +authorization +automaton +avalanche +avenue +average +award +awareness +azimuth +babe +baboon +babushka +baby +back +backbone +backdrop +background +backpack +bacon +bad +badge +badger +bafflement +bag +bagel +baggage +bagpipe +bail +bait +bake +baker +bakery +bakeware +balaclava +balalaika +balance +balcony +ball +ballet +balloon +ballpark +bamboo +banana +band +bandana +bandanna +bandolier +bangle +banjo +bank +bankbook +banker +banquette +baobab +bar +barbeque +barber +barbiturate +barge +baritone +barium +barn +barometer +barracks +barstool +base +baseball +basement +basin +basis +basket +basketball +bass +bassinet +bassoon +bat +bath +bather +bathhouse +bathrobe +bathroom +bathtub +batter +battery +batting +battle +battleship +bay +bayou +beach +bead +beak +beam +bean +beanie +beanstalk +bear +beard +beast +beat +beautiful +beauty +beaver +bed +bedroom +bee +beech +beef +beer +beet +beetle +beggar +beginner +beginning +begonia +behavior +beheading +behest +being +belfry +belief +believe +bell +belligerency +bellows +belly +belt +bench +bend +beneficiary +benefit +bengal +beret +berry +bestseller +best-seller +bet +beverage +beyond +bibliography +bicycle +bid +bidet +bifocals +big +big-rig +bijou +bike +bikini +bill +billboard +bin +biology +biplane +birch +bird +birdbath +birdcage +birdhouse +bird-watcher +birth +birthday +bit +bite +bitter +black +blackberry +blackboard +blackfish +bladder +blade +blame +blank +blanket +blazer +blight +blind +blinker +blister +blizzard +block +blocker +blood +bloodflow +bloom +bloomers +blossom +blouse +blow +blowgun +blowhole +blue +blueberry +boar +board +boat +boat-building +boatload +boatyard +bobcat +body +bog +bolero +bolt +bomb +bomber +bondsman +bone +bongo +bonnet +bonsai +bonus +boogeyman +book +bookcase +bookend +booklet +booster +boot +bootee +bootie +boots +booty +border +bore +bosom +boss +botany +bother +bottle +bottling +bottom +bottom-line +boudoir +bough +boundary +bow +bower +bowl +bowler +bowling +bowtie +box +boxer +boxspring +boy +boyfriend +bra +brace +bracelet +bracket +brain +brake +branch +brand +brandy +brass +brassiere +bratwurst +brave +bread +breadcrumb +break +breakfast +breakpoint +breast +breastplate +breath +breeze +bribery +brick +bricklaying +bridge +brief +briefs +brilliant +british +broad +broccoli +brochure +broiler +broker +brome +bronchitis +bronco +bronze +brooch +brood +brook +broom +brother +brother-in-law +brow +brown +brush +brushfire +brushing +bubble +bucket +buckle +bud +buddy +budget +buffer +buffet +bug +buggy +bugle +building +bulb +bull +bulldozer +bullet +bull-fighter +bumper +bun +bunch +bungalow +bunghole +bunkhouse +burglar +burlesque +burn +burn-out +burst +bus +bush +business +bust +bustle +butane +butcher +butter +button +buy +buyer +buzzard +cabana +cabbage +cabin +cabinet +cable +caboose +cacao +cactus +caddy +cadet +cafe +caftan +cake +calcification +calculation +calculator +calculus +calendar +calf +calico +call +calm +camel +cameo +camera +camp +campaign +campanile +can +canal +cancel +cancer +candelabra +candidate +candle +candy +cane +cannon +canoe +canon +canopy +canteen +canvas +cap +cape +capital +capitulation +capon +cappelletti +cappuccino +captain +caption +car +caravan +carbon +card +cardboard +cardigan +care +career +cargo +carload +carnation +carol +carotene +carp +carpenter +carpet +carport +carriage +carrier +carrot +carry +cart +cartilage +cartload +cartoon +cartridge +cascade +case +casement +cash +cashier +casino +casserole +cassock +cast +castanet +castanets +castle +cat +catacomb +catamaran +catch +category +caterpillar +cathedral +catsup +cattle +cauliflower +cause +caution +cave +c-clamp +cd +ceiling +celebration +celeriac +celery +celeste +cell +cellar +cello +celsius +cement +cemetery +cenotaph +census +cent +center +centimeter +centurion +century +cephalopod +ceramic +cereal +certification +cesspool +chafe +chain +chainstay +chair +chairlift +chairman +chairperson +chaise +chalet +chalice +chalk +challenge +champion +championship +chance +chandelier +change +channel +chaos +chap +chapel +chapter +character +chard +charge +charity +charlatan +charles +charm +chart +chastity +chasuble +chateau +chauffeur +chauvinist +check +checkroom +cheek +cheetah +chef +chemical +chemistry +cheque +cherries +cherry +chess +chest +chick +chicken +chicory +chief +chiffonier +child +childhood +children +chill +chime +chimpanzee +chin +chino +chip +chipmunk +chit-chat +chivalry +chive +chocolate +choice +choker +chop +chopstick +chord +chowder +chrome +chromolithograph +chronograph +chronometer +chub +chug +church +churn +cicada +cigarette +cinema +circle +circulation +circumference +cirrus +citizenship +city +civilisation +claim +clam +clank +clapboard +clarinet +clasp +class +classic +classroom +clause +clave +clavicle +clavier +cleaner +cleat +cleavage +clef +cleric +clerk +click +client +cliff +climate +climb +clip +clipper +cloak +cloakroom +clock +clockwork +clogs +cloister +close +closet +cloth +clothes +clothing +cloud +cloudburst +cloudy +clove +clover +club +clue +clutch +coach +coal +coast +coat +cob +cobweb +cockpit +cockroach +cocktail +cocoa +cod +code +codon +codpiece +coevolution +coffee +coffin +coil +coin +coinsurance +coke +cold +coliseum +collar +collection +college +collision +colloquia +colon +colonisation +colony +color +colt +column +columnist +comb +combat +combination +combine +comfort +comfortable +comic +comma +command +comment +commerce +commercial +commission +committee +common +communicant +communication +community +company +comparison +compassion +competition +competitor +complaint +complement +complex +component +comportment +composer +composition +compost +comprehension +compulsion +computer +comradeship +concentrate +concept +concern +concert +conclusion +concrete +condition +condominium +condor +conductor +cone +confectionery +conference +confidence +confirmation +conflict +confusion +conga +congo +congress +congressman +congressperson +conifer +connection +consent +consequence +consideration +consist +console +consonant +conspirator +constant +constellation +construction +consul +consulate +contact +contact lens +contagion +content +contest +context +continent +contract +contrail +contrary +contribution +control +convection +conversation +convert +convertible +cook +cookie +cooking +coonskin +cope +cop-out +copper +co-producer +copy +copyright +copywriter +cord +corduroy +cork +cormorant +corn +corner +cornerstone +cornet +corral +correspondent +corridor +corruption +corsage +cost +costume +cot +cottage +cotton +couch +cougar +cough +council +councilman +councilor +councilperson +count +counter +counter-force +countess +country +county +couple +courage +course +court +cousin +covariate +cover +coverall +cow +cowbell +cowboy +crab +crack +cracker +crackers +cradle +craft +craftsman +crash +crate +cravat +craw +crawdad +crayfish +crayon +crazy +cream +creative +creator +creature +creche +credenza +credit +creditor +creek +creme brulee +crest +crew +crib +cribbage +cricket +cricketer +crime +criminal +crinoline +criteria +criterion +criticism +crocodile +crocus +croissant +crook +crop +cross +cross-contamination +cross-stitch +crotch +croup +crow +crowd +crown +crude +crush +cry +crystallography +cub +cuckoo +cucumber +cuff-links +cultivar +cultivator +culture +culvert +cummerbund +cup +cupboard +cupcake +cupola +curio +curl +curler +currency +current +cursor +curtain +curve +cushion +custard +customer +cut +cuticle +cutlet +cutover +cutting +cyclamen +cycle +cyclone +cylinder +cymbal +cymbals +cynic +cyst +cytoplasm +dad +daffodil +dagger +dahlia +daisy +damage +dame +dance +dancer +dancing +danger +daniel +dare +dark +dart +dash +dashboard +data +database +date +daughter +david +day +daybed +dead +deadline +deal +dealer +dear +death +deathwatch +debate +debt +debtor +decade +decimal +decision +deck +declination +decongestant +decrease +decryption +dedication +deep +deer +defense +deficit +definition +deformation +degree +delay +delete +delight +delivery +demand +demur +den +denim +dentist +deodorant +department +departure +dependent +deployment +deposit +depression +depressive +depth +deputy +derby +derrick +description +desert +design +designer +desire +desk +dessert +destiny +destroyer +destruction +detail +detainment +detective +detention +determination +development +deviance +device +devil +dew +dhow +diadem +diamond +diaphragm +diarist +dibble +dickey +dictaphone +diction +dictionary +diet +difference +differential +difficulty +dig +digestion +digger +digital +dignity +dilapidation +dill +dime +dimension +dimple +diner +dinghy +dinner +dinosaur +diploma +dipstick +direction +director +dirndl +dirt +disadvantage +disarmament +disaster +discipline +disco +disconnection +discount +discovery +discrepancy +discussion +disease +disembodiment +disengagement +disguise +disgust +dish +dishes +dishwasher +disk +display +disposer +distance +distribution +distributor +district +divan +diver +divide +divider +diving +division +dock +doctor +document +doe +dog +dogsled +dogwood +doll +dollar +dolman +dolphin +domain +donkey +door +doorknob +doorpost +dory +dot +double +doubling +doubt +doubter +downforce +downgrade +downtown +draft +drag +dragon +dragonfly +dragster +drain +drake +drama +dramaturge +draw +drawbridge +drawer +drawing +dream +dredger +dress +dresser +dressing +drill +drink +drive +driver +driveway +driving +drizzle +dromedary +drop +drug +drum +drummer +drunk +dry +dryer +duck +duckling +dud +due +duffel +dugout +dulcimer +dumbwaiter +dump +dump truck +dune buggy +dungarees +dungeon +duplexer +dust +dust storm +duster +duty +dwarf +dwelling +dynamo +eagle +ear +eardrum +earmuffs +earplug +earrings +earth +earthquake +earthworm +ease +easel +east +eat +eave +eavesdropper +e-book +ecclesia +eclipse +ecliptic +economics +economy +ecumenist +eddy +edge +edger +editor +editorial +education +edward +eel +effacement +effect +effective +efficacy +efficiency +effort +egg +egghead +eggnog +eggplant +eight +ejector +elbow +election +electricity +electrocardiogram +element +elephant +elevator +elixir +elk +ellipse +elm +elongation +embossing +emergence +emergency +emergent +emery +emotion +emphasis +employ +employee +employer +employment +empowerment +emu +encirclement +encyclopedia +end +endothelium +enemy +energy +engine +engineer +engineering +enigma +enjoyment +enquiry +entertainment +enthusiasm +entrance +entry +environment +envy +epauliere +epee +ephemera +ephemeris +epoch +eponym +epoxy +equal +equinox +equipment +equivalent +era +e-reader +error +escape +ese +espadrille +espalier +essay +establishment +estate +estimate +estrogen +estuary +ethernet +ethics +euphonium +eurocentrism +europe +evaluator +evening +evening-wear +event +eviction +evidence +evocation +evolution +exam +examination +examiner +example +exchange +excitement +exclamation +excuse +executor +exercise +exhaust +ex-husband +exile +existence +exit +expansion +expansionism +experience +expert +explanation +exposition +expression +extension +extent +external +extreme +ex-wife +eye +eyeball +eyebrow +eyebrows +eyeglasses +eyelash +eyelashes +eyelid +eyelids +eyeliner +eyestrain +face +facelift +facet +facilities +facsimile +fact +factor +factory +faculty +fahrenheit +fail +failure +fairies +fairy +faith +fall +falling-out +fame +familiar +family +fan +fang +fanlight +fanny +fanny-pack +farm +farmer +fascia +fat +father +father-in-law +fatigues +faucet +fault +fawn +fax +fear +feast +feather +feature +fedelini +fedora +fee +feed +feedback +feel +feeling +feet +felony +female +fen +fence +fencing +fender +ferry +ferryboat +fertilizer +few +fiber +fiberglass +fibre +fiction +fiddle +field +fifth +fight +fighter +figure +figurine +file +fill +filly +film +filth +final +finance +find +finding +fine +finger +fingernail +finish +finisher +fir +fire +fireman +fireplace +firewall +fish +fishbone +fisherman +fishery +fishing +fishmonger +fishnet +fisting +fix +fixture +flag +flame +flanker +flare +flash +flat +flatboat +flavor +flax +fleck +fleece +flesh +flight +flintlock +flip-flops +flock +flood +floor +floozie +flour +flow +flower +flu +flugelhorn +fluke +flute +fly +flytrap +foam +fob +focus +fog +fold +folder +following +fondue +font +food +foot +football +footnote +footrest +foot-rest +footstool +foray +force +forearm +forebear +forecast +forehead +forest +forestry +forever +forgery +fork +form +formal +format +former +fort +fortnight +fortress +fortune +forum +foundation +fountain +fowl +fox +foxglove +fragrance +frame +fratricide +fraudster +frazzle +freckle +freedom +freeplay +freeze +freezer +freight +freighter +freon +fresco +friction +fridge +friend +friendship +frigate +fringe +frock +frog +front +frost +frown +fruit +frustration +fuel +fulfillment +full +fun +function +fundraising +funeral +funny +fur +furnace +furniture +fusarium +futon +future +gaffer +gain +gaiters +gale +gall-bladder +gallery +galley +gallon +galn +galoshes +game +gamebird +gamma-ray +gander +gap +garage +garb +garbage +garden +garlic +garment +garter +gas +gasoline +gastropod +gate +gateway +gather +gauge +gauntlet +gazebo +gazelle +gear +gearshift +geese +gelding +gem +gemsbok +gender +gene +general +genetics +geography +geology +geometry +george +geranium +gerbil +geyser +gherkin +ghost +giant +gift +gigantism +ginseng +giraffe +girdle +girl +girlfriend +git +give +glad +gladiolus +gland +glass +glasses +glen +glider +gliding +glockenspiel +glove +gloves +glue +glut +go +goal +goat +gobbler +god +godmother +goggles +go-kart +gold +goldfish +golf +gondola +gong +good +goodbye +good-bye +goodie +goose +gopher +gore-tex +gorilla +gosling +gossip +governance +government +governor +gown +grab +grab-bag +grade +grain +gram +grammar +grand +granddaughter +grandfather +grandmom +grandmother +grandson +granny +grape +grapefruit +graph +graphic +grass +grasshopper +grassland +gratitude +gray +grease +great +great-grandfather +great-grandmother +greek +green +greenhouse +grenade +grey +grief +grill +grip +grit +grocery +ground +group +grouper +grouse +growth +guarantee +guard +guess +guest +guestbook +guidance +guide +guilt +guilty +guitar +guitarist +gum +gumshoes +gun +gutter +guy +gym +gymnast +gymnastics +gynaecology +gyro +habit +hacienda +hacksaw +hackwork +hail +hair +haircut +half +half-brother +half-sister +halibut +hall +hallway +hamaki +hamburger +hammer +hammock +hamster +hand +handball +hand-holding +handicap +handle +handlebar +handmaiden +handsaw +hang +happiness +harbor +harbour +hardboard +hardcover +hardening +hardhat +hard-hat +hardware +harm +harmonica +harmony +harp +harpooner +harpsichord +hassock +hat +hatbox +hatchet +hate +hatred +haunt +haversack +hawk +hay +head +headlight +headline +headrest +health +hearing +heart +heartache +hearth +hearthside +heart-throb +heartwood +heat +heater +heaven +heavy +hedge +hedgehog +heel +height +heirloom +helen +helicopter +helium +hell +hellcat +hello +helmet +helo +help +hemp +hen +herb +heron +herring +hexagon +heyday +hide +high +highlight +high-rise +highway +hill +hip +hippodrome +hippopotamus +hire +history +hit +hive +hobbies +hobbit +hobby +hockey +hoe +hog +hold +hole +holiday +home +homework +homogenate +homonym +honesty +honey +honeybee +honoree +hood +hoof +hook +hope +hops +horn +hornet +horror +horse +hose +hosiery +hospice +hospital +hospitality +host +hostel +hostess +hot +hot-dog +hotel +hour +hourglass +house +houseboat +housework +housing +hovel +hovercraft +howitzer +hub +hubcap +hugger +human +humidity +humor +humour +hunger +hunt +hurdler +hurricane +hurry +hurt +husband +hut +hutch +hyacinth +hybridisation +hydrant +hydraulics +hydrofoil +hydrogen +hyena +hygienic +hyphenation +hypochondria +hypothermia +ice +icebreaker +icecream +ice-cream +icicle +icon +idea +ideal +if +igloo +ikebana +illegal +image +imagination +impact +implement +importance +impress +impression +imprisonment +improvement +impudence +impulse +inbox +incandescence +inch +incident +income +increase +independence +independent +index +indication +indigence +individual +industry +inevitable +infancy +inflammation +inflation +influence +information +infusion +inglenook +ingrate +initial +initiative +in-joke +injury +injustice +ink +in-laws +inlay +inn +innervation +innocence +innocent +input +inquiry +inscription +insect +inside +insolence +inspection +inspector +instance +instruction +instrument +instrumentalist +instrumentation +insulation +insurance +insurgence +intelligence +intention +interaction +interactive +interest +interferometer +interior +interloper +internal +international +internet +interpreter +intervenor +interview +interviewer +intestine +intestines +introduction +invention +inventor +inventory +investment +invite +invoice +iridescence +iris +iron +ironclad +irony +island +issue +it +item +jackal +jacket +jaguar +jail +jailhouse +jam +james +jar +jasmine +jaw +jealousy +jeans +jeep +jeff +jelly +jellyfish +jet +jewel +jewelry +jiffy +job +jockey +jodhpurs +joey +jogging +join +joint +joke +jot +journey +joy +judge +judgment +judo +juggernaut +juice +jumbo +jump +jumper +jumpsuit +junior +junk +junker +junket +jury +justice +jute +kale +kamikaze +kangaroo +karate +karen +kayak +kazoo +keep +kendo +ketch +ketchup +kettle +kettledrum +key +keyboard +keyboarding +keystone +kick +kick-off +kid +kidney +kidneys +kielbasa +kill +kilogram +kilometer +kilt +kimono +kind +kindness +king +kingfish +kiosk +kiss +kitchen +kite +kitten +kitty +kleenex +klomps +knee +kneejerk +knickers +knife +knife-edge +knight +knitting +knot +knowledge +knuckle +koala +kohlrabi +lab +laborer +labour +lace +lack +lacquerware +ladder +lady +ladybug +lake +lamb +lamp +lan +lanai +land +landform +landmine +landscape +language +lantern +lap +laparoscope +lapdog +laptop +larch +larder +lark +laryngitis +lasagna +latency +latex +lathe +latte +laugh +laughter +laundry +lava +law +lawn +lawsuit +lawyer +lay +layer +lead +leader +leadership +leading +leaf +league +leaker +learning +leash +leather +leave +leaver +lecture +leek +leg +legal +legging +legume +lei +leisure +lemon +lemonade +lemur +length +lentil +leprosy +lesson +let +letter +lettuce +level +lever +leverage +license +lie +lier +life +lift +light +lighting +lightning +lilac +lily +limit +limo +line +linen +liner +linguistics +link +linseed +lion +lip +lipstick +liquid +liquor +lisa +list +listen +literature +litigation +litter +liver +livestock +living +lizard +llama +load +loaf +loafer +loan +lobotomy +lobster +local +location +lock +locker +locket +locomotive +locust +loft +log +loggia +logic +loincloth +loneliness +long +look +loss +lot +lotion +lounge +lout +love +low +loyalty +luck +luggage +lumber +lumberman +lunch +luncheonette +lunchroom +lung +lunge +lute +luttuce +lycra +lye +lymphocyte +lynx +lyocell +lyre +lyric +macadamia +macaroni +machine +machinery +macrame +macrofauna +maelstrom +maestro +magazine +magic +maid +maiden +mail +mailbox +mailman +main +maintenance +major +major-league +make +makeup +male +mall +mallet +mambo +mammoth +man +management +manager +mandarin +mandolin +mangrove +manhunt +maniac +manicure +mankind +manner +manor +mansard +manservant +mansion +mantel +mantle +mantua +manufacturer +manx +many +map +maple +maraca +maracas +marble +mare +margin +mariachi +marimba +mark +market +marketing +marksman +marriage +marsh +marshland +marxism +mascara +mask +mass +massage +master +mastication +mastoid +mat +match +mate +material +math +mathematics +matter +mattock +mattress +maximum +maybe +mayonnaise +mayor +meal +meaning +measles +measure +measurement +meat +mechanic +media +medicine +medium +meet +meeting +megaliac +melody +member +membership +memory +men +menorah +mention +menu +mercury +mess +message +metal +metallurgist +meteor +meteorology +meter +methane +method +methodology +metro +metronome +mezzanine +mice +microlending +microwave +mid-course +middle +middleman +midi +midline +midnight +midwife +might +migrant +mile +milk +milkshake +millennium +millimeter +millisecond +mime +mimosa +mind +mine +mini +minibus +minimum +minion +mini-skirt +minister +minor +minor-league +mint +minute +mirror +miscarriage +miscommunication +misfit +misogyny +misplacement +misreading +miss +missile +mission +mist +mistake +mister +miter +mitten +mix +mixer +mixture +moat +mobile +moccasins +mocha +mode +model +modem +mole +mom +moment +monastery +monasticism +money +monger +monitor +monkey +monocle +monotheism +monsoon +monster +month +mood +moon +moonscape +moonshine +mop +morning +morsel +mortgage +mortise +mosque +mosquito +most +motel +moth +mother +mother-in-law +motion +motor +motorboat +motorcar +motorcycle +mound +mountain +mouse +mouser +mousse +moustache +mouth +mouton +move +mover +movie +mower +mud +mug +mukluk +mule +multimedia +muscle +musculature +museum +music +music-box +music-making +mustache +mustard +mutt +mycoplasma +n +nail +name +naming +nanoparticle +napkin +nasty +nation +national +native +natural +naturalisation +nature +neat +necessary +neck +necklace +necktie +need +needle +negative +negligee +negotiation +neologism +neon +nephew +nerve +nest +net +netball +netbook +netsuke +network +neurobiologist +neuropathologist +neuropsychiatry +news +newspaper +newsprint +newsstand +nexus +nicety +niche +nickel +niece +night +nightclub +nightgown +nightingale +nightlight +nitrogen +nobody +node +noise +nonbeliever +nonconformist +nondisclosure +nonsense +noodle +normal +norse +north +nose +note +notebook +nothing +notice +notify +notoriety +nougat +novel +nudge +number +numeracy +numeric +numismatist +nurse +nursery +nurture +nut +nutrition +nylon +oak +oar +oasis +oatmeal +obedience +obesity +obi +object +objective +obligation +oboe +observation +observatory +occasion +occupation +ocean +ocelot +octagon +octave +octavo +octet +octopus +odometer +oeuvre +offence +offer +office +officer +official +off-ramp +oil +okra +oldie +olive +omega +omelet +oncology +one +onion +open +opening +opera +operation +ophthalmologist +opinion +opium +opossum +opportunist +opportunity +opposite +option +orange +orangutan +orator +orchard +orchestra +orchid +order +ordinary +ordination +organ +organisation +organization +original +ornament +osmosis +osprey +ostrich +other +others +ott +otter +ounce +outback +outcome +outfit +outhouse +outlay +output +outrigger +outset +outside +oval +ovary +oven +overcharge +overclocking +overcoat +overexertion +overflight +overnighter +overshoot +owl +owner +ox +oxen +oxford +oxygen +oyster +pace +pacemaker +pack +package +packet +pad +paddle +paddock +page +pagoda +pail +pain +paint +painter +painting +paintwork +pair +pajama +pajamas +palm +pamphlet +pan +pancake +pancreas +panda +panic +pannier +panpipe +pansy +panther +panties +pantologist +pantology +pantry +pants +pantsuit +panty +pantyhose +paper +paperback +parable +parachute +parade +parallelogram +paramedic +parcel +parchment +pard +parent +parentheses +park +parka +parking +parrot +parsnip +part +participant +particle +particular +partner +partridge +party +pass +passage +passbook +passenger +passion +passive +past +pasta +paste +pastor +pastoralist +pastry +patch +path +patience +patient +patina +patio +patriarch +patricia +patrimony +patriot +patrol +pattern +pause +pavement +pavilion +paw +pawnshop +pay +payee +payment +pea +peace +peach +peacoat +peacock +peak +peanut +pear +pearl +pedal +peen +peer +peer-to-peer +pegboard +pelican +pelt +pen +penalty +pencil +pendant +pendulum +penicillin +pension +pentagon +peony +people +pepper +percentage +perception +perch +performance +perfume +period +periodical +peripheral +permafrost +permission +permit +perp +person +personal +personality +perspective +pest +pet +petal +petticoat +pew +pha +pharmacist +pharmacopoeia +phase +pheasant +philosopher +philosophy +phone +photo +photographer +phrase +physical +physics +pianist +piano +piccolo +pick +pickax +picket +pickle +picture +pie +piece +pier +piety +pig +pigeon +pike +pile +pilgrimage +pillbox +pillow +pilot +pimp +pimple +pin +pinafore +pince-nez +pine +pineapple +pinecone +ping +pink +pinkie +pinstripe +pint +pinto +pinworm +pioneer +pipe +piracy +piss +pitch +pitching +pith +pizza +place +plain +plan +plane +planet +plant +plantation +planter +plaster +plasterboard +plastic +plate +platform +platinum +platypus +play +player +playground +playroom +pleasure +pleated +plenty +plier +plot +plough +plover +plow +plowman +plume +plunger +plywood +pneumonia +pocket +pocketbook +pocket-watch +poem +poet +poetry +poignance +point +poison +poisoning +pole +polenta +police +policeman +policy +polish +politics +pollution +polo +polyester +pompom +poncho +pond +pony +poof +pool +pop +popcorn +poppy +popsicle +population +populist +porch +porcupine +port +porter +portfolio +porthole +position +positive +possession +possibility +possible +post +postage +postbox +poster +pot +potato +potential +potty +pouch +poultry +pound +pounding +poverty +powder +power +practice +precedent +precipitation +preface +preference +prelude +premeditation +premier +preoccupation +preparation +presence +present +presentation +president +press +pressroom +pressure +pressurisation +price +pride +priest +priesthood +primary +primate +prince +princess +principal +principle +print +printer +prior +priority +prison +private +prize +prizefight +probation +problem +procedure +process +processing +produce +producer +product +production +profession +professional +professor +profile +profit +program +progress +project +promise +promotion +prompt +pronunciation +proof +proof-reader +propane +property +proposal +prose +prosecution +protection +protest +protocol +prow +pruner +pseudoscience +psychiatrist +psychoanalyst +psychologist +psychology +ptarmigan +public +publicity +publisher +pudding +puddle +puffin +pull +pulley +puma +pump +pumpkin +pumpkinseed +punch +punctuation +punishment +pupa +pupil +puppy +purchase +puritan +purple +purpose +purse +push +pusher +put +pvc +pyjama +pyramid +quadrant +quail +quality +quantity +quart +quarter +quartz +queen +question +quicksand +quiet +quill +quilt +quince +quit +quiver +quotation +quote +rabbi +rabbit +raccoon +race +racer +racing +racism +racist +rack +radar +radiator +radio +radiosonde +radish +raffle +raft +rag +rage +rail +railway +raiment +rain +rainbow +raincoat +rainmaker +rainstorm +raise +rake +ram +rambler +ramie +ranch +random +randomisation +range +rank +raspberry +rat +rate +ratio +raven +ravioli +raw +rawhide +ray +rayon +reach +reactant +reaction +read +reading +reality +reamer +rear +reason +receipt +reception +recess +recipe +recliner +recognition +recommendation +record +recorder +recording +recover +recreation +recruit +rectangle +red +redesign +rediscovery +reduction +reef +refectory +reference +reflection +refrigerator +refund +refuse +region +register +regret +regular +regulation +reindeer +reinscription +reject +relation +relationship +relative +relaxation +release +reliability +relief +religion +relish +reminder +remote +remove +rent +repair +reparation +repeat +replace +replacement +replication +reply +report +representative +reprocessing +republic +reputation +request +requirement +resale +research +reserve +resident +resist +resolution +resolve +resort +resource +respect +respite +respond +response +responsibility +rest +restaurant +result +retailer +rethinking +retina +retouch +return +reveal +revenant +revenge +revenue +review +revolution +revolve +revolver +reward +rheumatism +rhinoceros +rhyme +rhythm +rice +rich +riddle +ride +rider +ridge +rifle +right +rim +ring +ringworm +rip +ripple +rise +riser +risk +river +riverbed +rivulet +road +roadway +roast +robe +robin +rock +rocker +rocket +rocket-ship +rod +role +roll +roller +roof +room +rooster +root +rope +rose +rostrum +rotate +rough +round +roundabout +route +router +routine +row +rowboat +royal +rub +rubber +rubbish +rubric +ruckus +ruffle +rugby +ruin +rule +rum +run +runaway +runner +rush +rutabaga +ruth +ry +sabre +sack +sad +saddle +safe +safety +sage +sail +sailboat +sailor +salad +salary +sale +salesman +salmon +salon +saloon +salt +samovar +sampan +sample +samurai +sand +sandals +sandbar +sandwich +sardine +sari +sarong +sash +satellite +satin +satire +satisfaction +sauce +sausage +save +saving +savings +savior +saviour +saw +saxophone +scale +scallion +scanner +scarecrow +scarf +scarification +scene +scenery +scent +schedule +scheme +schizophrenic +schnitzel +school +schoolhouse +schooner +science +scimitar +scissors +scooter +score +scorn +scow +scraper +scratch +screamer +screen +screenwriting +screw +screwdriver +screw-up +scrim +scrip +script +sculpting +sculpture +sea +seafood +seagull +seal +seaplane +search +seashore +seaside +season +seat +second +secret +secretariat +secretary +section +sectional +sector +secure +security +seed +seeder +segment +select +selection +self +sell +semicircle +semicolon +senator +senior +sense +sensitive +sentence +sepal +septicaemia +series +servant +serve +server +service +session +set +setting +settler +sewer +sex +shack +shade +shadow +shadowbox +shake +shakedown +shaker +shallot +shame +shampoo +shanty +shape +share +shark +sharon +shawl +she +shearling +shears +sheath +shed +sheep +sheet +shelf +shell +shelter +sherry +shield +shift +shin +shine +shingle +ship +shirt +shirtdress +shoat +shock +shoe +shoehorn +shoe-horn +shoelace +shoemaker +shoes +shoestring +shofar +shoot +shootdown +shop +shopper +shopping +shore +shortage +shorts +shortwave +shot +shoulder +shovel +show +shower +show-stopper +shred +shrimp +shrine +sibling +sick +side +sideboard +sideburns +sidecar +sidestream +sidewalk +siding +sign +signal +signature +signet +significance +signup +silence +silica +silk +silkworm +sill +silly +silo +silver +simple +sing +singer +single +sink +sir +sister +sister-in-law +sitar +site +situation +size +skate +skiing +skill +skin +skirt +skull +skullcap +skullduggery +skunk +sky +skylight +skyscraper +skywalk +slapstick +slash +slave +sled +sledge +sleep +sleet +sleuth +slice +slide +slider +slime +slip +slipper +slippers +slope +sloth +smash +smell +smelting +smile +smock +smog +smoke +smoking +smuggling +snail +snake +snakebite +sneakers +sneeze +snob +snorer +snow +snowboarding +snowflake +snowman +snowmobiling +snowplow +snowstorm +snowsuit +snuggle +soap +soccer +society +sociology +sock +socks +soda +sofa +soft +softball +softdrink +softening +software +soil +soldier +solid +solitaire +solution +sombrero +somersault +somewhere +son +song +songbird +sonnet +soot +soprano +sorbet +sorrow +sort +soulmate +sound +soup +source +sourwood +sousaphone +south +south america +south korea +sow +soy +soybean +space +spacing +spade +spaghetti +spandex +spank +spare +spark +sparrow +spasm +speaker +speakerphone +spear +special +specialist +specific +spectacle +spectacles +spectrograph +speech +speed +speedboat +spell +spelling +spend +sphere +sphynx +spider +spike +spinach +spine +spiral +spirit +spiritual +spite +spleen +split +sponge +spoon +sport +spot +spotlight +spray +spread +spring +sprinter +sprout +spruce +spume +spur +spy +square +squash +squatter +squeegee +squid +squirrel +stable +stack +stacking +stadium +staff +stag +stage +stain +stair +staircase +stallion +stamen +stamina +stamp +stance +stand +standard +standoff +star +start +starter +state +statement +station +station-wagon +statistic +status +stay +steak +steal +steam +steamroller +steel +steeple +stem +stencil +step +step-aunt +step-brother +stepdaughter +step-daughter +step-father +step-grandfather +step-grandmother +stepmother +step-mother +stepping-stone +steps +step-sister +stepson +step-son +step-uncle +stew +stick +stiletto +still +stinger +stitch +stock +stocking +stockings +stock-in-trade +stole +stomach +stone +stonework +stool +stop +stopsign +stopwatch +storage +store +storey +storm +story +storyboard +story-telling +stove +strain +strait +stranger +strap +strategy +straw +strawberry +stream +street +streetcar +strength +stress +stretch +strike +string +strip +stroke +structure +struggle +stud +student +studio +study +stuff +stumbling +stupid +stupidity +sturgeon +style +styling +stylus +subcomponent +subconscious +subject +submarine +subroutine +subsidence +substance +suburb +subway +success +suck +suede +suffocation +sugar +suggestion +suit +suitcase +sultan +summer +sun +sunbeam +sunbonnet +sunday +sundial +sunflower +sunglasses +sunlamp +sunroom +sunshine +supermarket +supply +support +supporter +suppression +surface +surfboard +surgeon +surgery +surname +surprise +surround +survey +sushi +suspect +suspenders +sustainment +SUV +swallow +swamp +swan +swath +sweat +sweater +sweats +sweatshirt +sweatshop +sweatsuit +swedish +sweet +sweets +swell +swim +swimming +swimsuit +swing +swiss +switch +switchboard +swivel +sword +swordfish +sycamore +symmetry +sympathy +syndicate +synergy +synod +syrup +system +tabby +tabernacle +table +tablecloth +tabletop +tachometer +tackle +tadpole +tail +tailor +tailspin +tale +talk +tam +tambour +tambourine +tam-o'-shanter +tandem +tangerine +tank +tanker +tankful +tank-top +tap +tard +target +task +tassel +taste +tatami +tattler +tattoo +tavern +tax +taxi +taxicab +tea +teach +teacher +teaching +team +tear +technologist +technology +teen +teeth +telephone +telescreen +teletype +television +tell +teller +temp +temper +temperature +temple +tempo +temporariness +temporary +temptress +tendency +tenement +tennis +tenor +tension +tent +tepee +term +terracotta +terrapin +territory +test +text +textbook +texture +thanks +thaw +theater +theism +theme +theory +therapist +thermals +thermometer +thigh +thing +thinking +thirst +thistle +thomas +thong +thongs +thorn +thought +thread +thrill +throat +throne +thrush +thumb +thunder +thunderbolt +thunderhead +thunderstorm +tiara +tic +ticket +tie +tiger +tight +tights +tile +till +timbale +timber +time +timeline +timeout +timer +timpani +tin +tinderbox +tinkle +tintype +tip +tire +tissue +titanium +title +toad +toast +today +toe +toenail +toga +togs +toilet +tolerance +tom +tomato +tomography +tomorrow +tom-tom +ton +tone +tongue +tonight +tool +toot +tooth +toothbrush +toothpaste +toothpick +top +top-hat +topic +topsail +toque +torchiere +toreador +tornado +torso +tortellini +tortoise +tosser +total +tote +touch +tough +tough-guy +tour +tourist +towel +tower +town +townhouse +tow-truck +toy +trachoma +track +tracksuit +tractor +trade +tradition +traditionalism +traffic +trail +trailer +train +trainer +training +tram +tramp +transaction +transition +translation +transmission +transom +transport +transportation +trapdoor +trapezium +trapezoid +trash +travel +tray +treat +treatment +tree +trellis +tremor +trench +trial +triangle +tribe +trick +trigonometry +trim +trinket +trip +tripod +trolley +trombone +trooper +trouble +trousers +trout +trove +trowel +truck +truckit +trumpet +trunk +trust +truth +try +t-shirt +tsunami +tub +tuba +tube +tugboat +tulip +tummy +tuna +tune +tune-up +tunic +tunnel +turban +turkish +turn +turnip +turnover +turnstile +turret +turtle +tussle +tutu +tuxedo +tv +twig +twilight +twine +twist +twister +two +type +typewriter +typhoon +tyvek +ukulele +umbrella +unblinking +uncle +underclothes +underground +underneath +underpants +underpass +undershirt +understanding +underwear +underwire +unemployment +unibody +uniform +union +unique +unit +unity +university +upper +upstairs +urn +usage +use +user +usher +usual +utensil +vacation +vacuum +vagrant +valance +validity +valley +valuable +value +van +vane +vanity +variation +variety +vase +vast +vault +vaulting +veal +vegetable +vegetarianism +vegetation +vehicle +veil +vein +veldt +vellum +velodrome +velvet +vengeance +venom +veranda +verdict +vermicelli +verse +version +vertigo +verve +vessel +vest +vestment +vibe +vibraphone +vibration +video +view +villa +village +vineyard +vinyl +viola +violence +violet +violin +virginal +virtue +virus +viscose +vise +vision +visit +visitor +visor +visual +vitality +vixen +voice +volcano +volleyball +volume +voyage +vulture +wad +wafer +waffle +waist +waistband +wait +waiter +waitress +wake +walk +walker +walkway +wall +wallaby +wallet +walnut +walrus +wampum +wannabe +war +warden +warlock +warmth +warm-up +warning +wash +washbasin +washcloth +washer +washtub +wasp +waste +wastebasket +watch +watchmaker +water +waterbed +waterfall +waterskiing +waterspout +wave +wax +way +weakness +wealth +weapon +wear +weasel +weather +web +wedding +wedge +weed +weeder +weedkiller +week +weekend +weekender +weight +weird +welcome +welfare +well +west +western +wet-bar +wetsuit +whale +wharf +wheat +wheel +whereas +while +whip +whirlpool +whirlwind +whisker +whiskey +whistle +white +whole +wholesale +wholesaler +whorl +width +wife +wilderness +wildlife +will +willow +win +wind +windage +wind-chime +window +windscreen +windshield +wine +wing +wingman +wingtip +winner +winter +wire +wisdom +wiseguy +wish +wisteria +witch +witch-hunt +withdrawal +witness +wolf +wombat +women +wonder +wood +woodland +woodshed +woodwind +wool +woolen +word +work +workbench +worker +workhorse +working +worklife +workshop +world +worm +worry +worth +worthy +wound +wrap +wraparound +wrecker +wren +wrench +wrestler +wrinkle +wrist +writer +writing +wrong +xylophone +yacht +yak +yam +yard +yarmulke +yarn +yawl +year +yeast +yellow +yesterday +yew +yin +yoga +yogurt +yoke +you +young +youth +yurt +zampone +zebra +zebrafish +zephyr +ziggurat +zinc +zipper +zither +zone +zoo +zoologist +zoology +zoot-suit +zucchini
\ No newline at end of file diff --git a/testing/condprofile/condprof/tests/__init__.py b/testing/condprofile/condprof/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/condprofile/condprof/tests/__init__.py diff --git a/testing/condprofile/condprof/tests/fakefirefox.py b/testing/condprofile/condprof/tests/fakefirefox.py new file mode 100755 index 0000000000..82e24dd081 --- /dev/null +++ b/testing/condprofile/condprof/tests/fakefirefox.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 +VERSION = """\ +Mozilla Firefox 70.0\ +""" + +if __name__ == "__main__": + print(VERSION) diff --git a/testing/condprofile/condprof/tests/fakegeckodriver.py b/testing/condprofile/condprof/tests/fakegeckodriver.py new file mode 100755 index 0000000000..ab40401aaf --- /dev/null +++ b/testing/condprofile/condprof/tests/fakegeckodriver.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +import argparse +import json +from http.server import BaseHTTPRequestHandler, HTTPServer +from uuid import uuid4 + +_SESSIONS = {} + + +class Window: + def __init__(self, handle, title="about:blank"): + self.handle = handle + self.title = title + + def visit_url(self, url): + print("Visiting %s" % url) + # XXX todo, load the URL for real + self.url = url + + +class Session: + def __init__(self, uuid): + self.session_id = uuid + self.autoinc = 0 + self.windows = {} + self.active_handle = self.new_window() + + def visit(self, url): + self.windows[self.active_handle].visit_url(url) + + def new_window(self): + w = Window(self.autoinc) + self.windows[w.handle] = w + self.autoinc += 1 + return w.handle + + +class RequestHandler(BaseHTTPRequestHandler): + def _set_headers(self, status=200): + self.send_response(status) + self.send_header("Content-type", "application/json") + self.end_headers() + + def _send_response(self, status=200, data=None): + if data is None: + data = {} + data = json.dumps(data).encode("utf8") + self._set_headers(status) + self.wfile.write(data) + + def _parse_path(self): + path = self.path.lstrip("/") + sections = path.split("/") + session = None + action = [] + if len(sections) > 1: + session_id = sections[1] + if session_id in _SESSIONS: + session = _SESSIONS[session_id] + action = sections[2:] + return session, action + + def do_GET(self): + print("GET " + self.path) + if self.path == "/status": + return self._send_response(data={"ready": "OK"}) + + session, action = self._parse_path() + if action == ["window", "handles"]: + data = {"value": list(session.windows.keys())} + return self._send_response(data=data) + + if action == ["moz", "context"]: + data = {"value": "chrome"} + return self._send_response(data=data) + + return self._send_response(status=404) + + def do_POST(self): + print("POST " + self.path) + content_length = int(self.headers["Content-Length"]) + post_data = json.loads(self.rfile.read(content_length)) + + # new session + if self.path == "/session": + uuid = str(uuid4()) + _SESSIONS[uuid] = Session(uuid) + return self._send_response(data={"sessionId": uuid}) + + session, action = self._parse_path() + if action == ["url"]: + session.visit(post_data["url"]) + return self._send_response() + + if action == ["window", "new"]: + if session is None: + return self._send_response(404) + handle = session.new_window() + return self._send_response(data={"handle": handle, "type": "tab"}) + + if action == ["timeouts"]: + return self._send_response() + + if action == ["execute", "async"]: + return self._send_response(data={"logs": []}) + + # other commands not supported yet, we just return 200s + return self._send_response() + + def do_DELETE(self): + return self._send_response() + session, action = self._parse_path() + if session is not None: + del _SESSIONS[session.session_id] + return self._send_response() + return self._send_response(status=404) + + +VERSION = """\ +geckodriver 0.24.0 ( 2019-01-28) + +The source code of this program is available from +testing/geckodriver in https://hg.mozilla.org/mozilla-central. + +This program is subject to the terms of the Mozilla Public License 2.0. +You can obtain a copy of the license at https://mozilla.org/MPL/2.0/.\ +""" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="FakeGeckodriver") + parser.add_argument("--log", type=str, default=None) + parser.add_argument("--port", type=int, default=4444) + parser.add_argument("--marionette-port", type=int, default=2828) + parser.add_argument("--version", action="store_true", default=False) + parser.add_argument("--verbose", "-v", action="count") + args = parser.parse_args() + + if args.version: + print(VERSION) + else: + HTTPServer.allow_reuse_address = True + server = HTTPServer(("127.0.0.1", args.port), RequestHandler) + server.serve_forever() diff --git a/testing/condprofile/condprof/tests/ftp_mozilla.html b/testing/condprofile/condprof/tests/ftp_mozilla.html new file mode 100644 index 0000000000..c970f8483b --- /dev/null +++ b/testing/condprofile/condprof/tests/ftp_mozilla.html @@ -0,0 +1,1484 @@ +<!DOCTYPE html> +<html> + <head> + <meta charset="UTF-8"> + <title>Directory Listing: /pub/firefox/nightly/latest-mozilla-central/</title> + </head> + <body> + <h1>Index of /pub/firefox/nightly/latest-mozilla-central/</h1> + <table> + <tr> + <th>Type</th> + <th>Name</th> + <th>Size</th> + <th>Last Modified</th> + </tr> + + <tr> + <td>Dir</td> + <td><a href="/pub/firefox/nightly/">..</a></td> + <td></td> + <td></td> + </tr> + + + <tr> + <td>Dir</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/mar-tools/">mar-tools/</a></td> + <td></td> + <td></td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/Firefox%20Installer.en-US.exe">Firefox Installer.en-US.exe</a></td> + <td>294K</td> + <td>02-Sep-2019 00:48</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.langpack.xpi">firefox-70.0a1.en-US.langpack.xpi</a></td> + <td>453K</td> + <td>02-Sep-2019 00:06</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.awsy.tests.tar.gz">firefox-70.0a1.en-US.linux-i686.awsy.tests.tar.gz</a></td> + <td>20K</td> + <td>01-Sep-2019 23:49</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.buildhub.json">firefox-70.0a1.en-US.linux-i686.buildhub.json</a></td> + <td>1K</td> + <td>01-Sep-2019 23:49</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.checksums">firefox-70.0a1.en-US.linux-i686.checksums</a></td> + <td>8K</td> + <td>01-Sep-2019 23:49</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.common.tests.tar.gz">firefox-70.0a1.en-US.linux-i686.common.tests.tar.gz</a></td> + <td>45M</td> + <td>01-Sep-2019 23:49</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.complete.mar">firefox-70.0a1.en-US.linux-i686.complete.mar</a></td> + <td>57M</td> + <td>01-Sep-2019 23:49</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.cppunittest.tests.tar.gz">firefox-70.0a1.en-US.linux-i686.cppunittest.tests.tar.gz</a></td> + <td>12M</td> + <td>01-Sep-2019 23:49</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.crashreporter-symbols.zip">firefox-70.0a1.en-US.linux-i686.crashreporter-symbols.zip</a></td> + <td>79M</td> + <td>01-Sep-2019 23:49</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.json">firefox-70.0a1.en-US.linux-i686.json</a></td> + <td>855</td> + <td>01-Sep-2019 23:49</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.mochitest.tests.tar.gz">firefox-70.0a1.en-US.linux-i686.mochitest.tests.tar.gz</a></td> + <td>65M</td> + <td>01-Sep-2019 23:49</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.mozinfo.json">firefox-70.0a1.en-US.linux-i686.mozinfo.json</a></td> + <td>916</td> + <td>01-Sep-2019 23:49</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.reftest.tests.tar.gz">firefox-70.0a1.en-US.linux-i686.reftest.tests.tar.gz</a></td> + <td>53M</td> + <td>01-Sep-2019 23:49</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.talos.tests.tar.gz">firefox-70.0a1.en-US.linux-i686.talos.tests.tar.gz</a></td> + <td>18M</td> + <td>01-Sep-2019 23:49</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.tar.bz2">firefox-70.0a1.en-US.linux-i686.tar.bz2</a></td> + <td>71M</td> + <td>01-Sep-2019 23:49</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.tar.bz2.asc">firefox-70.0a1.en-US.linux-i686.tar.bz2.asc</a></td> + <td>833</td> + <td>01-Sep-2019 23:49</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.test_packages.json">firefox-70.0a1.en-US.linux-i686.test_packages.json</a></td> + <td>1K</td> + <td>01-Sep-2019 23:49</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.txt">firefox-70.0a1.en-US.linux-i686.txt</a></td> + <td>99</td> + <td>01-Sep-2019 23:49</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.web-platform.tests.tar.gz">firefox-70.0a1.en-US.linux-i686.web-platform.tests.tar.gz</a></td> + <td>53M</td> + <td>01-Sep-2019 23:49</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686.xpcshell.tests.tar.gz">firefox-70.0a1.en-US.linux-i686.xpcshell.tests.tar.gz</a></td> + <td>9M</td> + <td>01-Sep-2019 23:49</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-i686_info.txt">firefox-70.0a1.en-US.linux-i686_info.txt</a></td> + <td>23</td> + <td>01-Sep-2019 23:49</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.awsy.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.awsy.tests.tar.gz</a></td> + <td>20K</td> + <td>02-Sep-2019 01:43</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.buildhub.json">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.buildhub.json</a></td> + <td>1K</td> + <td>02-Sep-2019 01:43</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.checksums">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.checksums</a></td> + <td>8K</td> + <td>02-Sep-2019 01:44</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.common.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.common.tests.tar.gz</a></td> + <td>55M</td> + <td>02-Sep-2019 01:43</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.complete.mar">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.complete.mar</a></td> + <td>208M</td> + <td>02-Sep-2019 01:43</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.cppunittest.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.cppunittest.tests.tar.gz</a></td> + <td>118M</td> + <td>02-Sep-2019 01:43</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.json">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.json</a></td> + <td>860</td> + <td>02-Sep-2019 01:43</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.mochitest.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.mochitest.tests.tar.gz</a></td> + <td>65M</td> + <td>02-Sep-2019 01:43</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.mozinfo.json">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.mozinfo.json</a></td> + <td>927</td> + <td>02-Sep-2019 01:43</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.reftest.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.reftest.tests.tar.gz</a></td> + <td>53M</td> + <td>02-Sep-2019 01:43</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.talos.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.talos.tests.tar.gz</a></td> + <td>18M</td> + <td>02-Sep-2019 01:43</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.tar.bz2">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.tar.bz2</a></td> + <td>276M</td> + <td>02-Sep-2019 01:43</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.tar.bz2.asc">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.tar.bz2.asc</a></td> + <td>833</td> + <td>02-Sep-2019 01:43</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.test_packages.json">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.test_packages.json</a></td> + <td>1K</td> + <td>02-Sep-2019 01:43</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.txt">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.txt</a></td> + <td>99</td> + <td>02-Sep-2019 01:43</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.web-platform.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.web-platform.tests.tar.gz</a></td> + <td>53M</td> + <td>02-Sep-2019 01:43</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter.xpcshell.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64-asan-reporter.xpcshell.tests.tar.gz</a></td> + <td>10M</td> + <td>02-Sep-2019 01:43</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64-asan-reporter_info.txt">firefox-70.0a1.en-US.linux-x86_64-asan-reporter_info.txt</a></td> + <td>23</td> + <td>02-Sep-2019 01:43</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.awsy.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64.awsy.tests.tar.gz</a></td> + <td>20K</td> + <td>02-Sep-2019 00:06</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.buildhub.json">firefox-70.0a1.en-US.linux-x86_64.buildhub.json</a></td> + <td>1K</td> + <td>02-Sep-2019 00:06</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.checksums">firefox-70.0a1.en-US.linux-x86_64.checksums</a></td> + <td>8K</td> + <td>02-Sep-2019 00:07</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.common.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64.common.tests.tar.gz</a></td> + <td>45M</td> + <td>02-Sep-2019 00:06</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.complete.mar">firefox-70.0a1.en-US.linux-x86_64.complete.mar</a></td> + <td>57M</td> + <td>02-Sep-2019 00:06</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.cppunittest.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64.cppunittest.tests.tar.gz</a></td> + <td>12M</td> + <td>02-Sep-2019 00:06</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.crashreporter-symbols.zip">firefox-70.0a1.en-US.linux-x86_64.crashreporter-symbols.zip</a></td> + <td>74M</td> + <td>02-Sep-2019 00:06</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.json">firefox-70.0a1.en-US.linux-x86_64.json</a></td> + <td>846</td> + <td>02-Sep-2019 00:06</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.mochitest.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64.mochitest.tests.tar.gz</a></td> + <td>65M</td> + <td>02-Sep-2019 00:06</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.mozinfo.json">firefox-70.0a1.en-US.linux-x86_64.mozinfo.json</a></td> + <td>921</td> + <td>02-Sep-2019 00:06</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.reftest.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64.reftest.tests.tar.gz</a></td> + <td>53M</td> + <td>02-Sep-2019 00:06</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.talos.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64.talos.tests.tar.gz</a></td> + <td>18M</td> + <td>02-Sep-2019 00:06</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.tar.bz2">firefox-70.0a1.en-US.linux-x86_64.tar.bz2</a></td> + <td>71M</td> + <td>02-Sep-2019 00:06</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.tar.bz2.asc">firefox-70.0a1.en-US.linux-x86_64.tar.bz2.asc</a></td> + <td>833</td> + <td>02-Sep-2019 00:06</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.test_packages.json">firefox-70.0a1.en-US.linux-x86_64.test_packages.json</a></td> + <td>1K</td> + <td>02-Sep-2019 00:06</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.txt">firefox-70.0a1.en-US.linux-x86_64.txt</a></td> + <td>99</td> + <td>02-Sep-2019 00:06</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.web-platform.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64.web-platform.tests.tar.gz</a></td> + <td>53M</td> + <td>02-Sep-2019 00:06</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64.xpcshell.tests.tar.gz">firefox-70.0a1.en-US.linux-x86_64.xpcshell.tests.tar.gz</a></td> + <td>9M</td> + <td>02-Sep-2019 00:06</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.linux-x86_64_info.txt">firefox-70.0a1.en-US.linux-x86_64_info.txt</a></td> + <td>23</td> + <td>02-Sep-2019 00:06</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.awsy.tests.tar.gz">firefox-70.0a1.en-US.mac.awsy.tests.tar.gz</a></td> + <td>20K</td> + <td>01-Sep-2019 23:19</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.buildhub.json">firefox-70.0a1.en-US.mac.buildhub.json</a></td> + <td>1K</td> + <td>01-Sep-2019 23:19</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.checksums">firefox-70.0a1.en-US.mac.checksums</a></td> + <td>7K</td> + <td>01-Sep-2019 23:19</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.common.tests.tar.gz">firefox-70.0a1.en-US.mac.common.tests.tar.gz</a></td> + <td>20M</td> + <td>01-Sep-2019 23:19</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.complete.mar">firefox-70.0a1.en-US.mac.complete.mar</a></td> + <td>60M</td> + <td>01-Sep-2019 23:19</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.cppunittest.tests.tar.gz">firefox-70.0a1.en-US.mac.cppunittest.tests.tar.gz</a></td> + <td>10M</td> + <td>01-Sep-2019 23:19</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.crashreporter-symbols.zip">firefox-70.0a1.en-US.mac.crashreporter-symbols.zip</a></td> + <td>54M</td> + <td>01-Sep-2019 23:19</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.dmg">firefox-70.0a1.en-US.mac.dmg</a></td> + <td>79M</td> + <td>01-Sep-2019 23:19</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.json">firefox-70.0a1.en-US.mac.json</a></td> + <td>1K</td> + <td>01-Sep-2019 23:19</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.mochitest.tests.tar.gz">firefox-70.0a1.en-US.mac.mochitest.tests.tar.gz</a></td> + <td>65M</td> + <td>01-Sep-2019 23:19</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.mozinfo.json">firefox-70.0a1.en-US.mac.mozinfo.json</a></td> + <td>923</td> + <td>01-Sep-2019 23:19</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.pkg">firefox-70.0a1.en-US.mac.pkg</a></td> + <td>83M</td> + <td>01-Sep-2019 23:19</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.reftest.tests.tar.gz">firefox-70.0a1.en-US.mac.reftest.tests.tar.gz</a></td> + <td>53M</td> + <td>01-Sep-2019 23:19</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.talos.tests.tar.gz">firefox-70.0a1.en-US.mac.talos.tests.tar.gz</a></td> + <td>18M</td> + <td>01-Sep-2019 23:19</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.test_packages.json">firefox-70.0a1.en-US.mac.test_packages.json</a></td> + <td>1K</td> + <td>01-Sep-2019 23:19</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.txt">firefox-70.0a1.en-US.mac.txt</a></td> + <td>99</td> + <td>01-Sep-2019 23:19</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.web-platform.tests.tar.gz">firefox-70.0a1.en-US.mac.web-platform.tests.tar.gz</a></td> + <td>53M</td> + <td>01-Sep-2019 23:19</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac.xpcshell.tests.tar.gz">firefox-70.0a1.en-US.mac.xpcshell.tests.tar.gz</a></td> + <td>9M</td> + <td>01-Sep-2019 23:19</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.mac_info.txt">firefox-70.0a1.en-US.mac_info.txt</a></td> + <td>23</td> + <td>01-Sep-2019 23:19</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.awsy.tests.tar.gz">firefox-70.0a1.en-US.win32.awsy.tests.tar.gz</a></td> + <td>20K</td> + <td>02-Sep-2019 00:48</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.buildhub.json">firefox-70.0a1.en-US.win32.buildhub.json</a></td> + <td>1K</td> + <td>02-Sep-2019 00:48</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.checksums">firefox-70.0a1.en-US.win32.checksums</a></td> + <td>8K</td> + <td>02-Sep-2019 00:49</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.common.tests.tar.gz">firefox-70.0a1.en-US.win32.common.tests.tar.gz</a></td> + <td>22M</td> + <td>02-Sep-2019 00:48</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.complete.mar">firefox-70.0a1.en-US.win32.complete.mar</a></td> + <td>51M</td> + <td>02-Sep-2019 00:48</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.cppunittest.tests.tar.gz">firefox-70.0a1.en-US.win32.cppunittest.tests.tar.gz</a></td> + <td>10M</td> + <td>02-Sep-2019 00:48</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.crashreporter-symbols.zip">firefox-70.0a1.en-US.win32.crashreporter-symbols.zip</a></td> + <td>34M</td> + <td>02-Sep-2019 00:48</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.installer.exe">firefox-70.0a1.en-US.win32.installer.exe</a></td> + <td>48M</td> + <td>02-Sep-2019 00:48</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.installer.msi">firefox-70.0a1.en-US.win32.installer.msi</a></td> + <td>48M</td> + <td>02-Sep-2019 00:48</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.json">firefox-70.0a1.en-US.win32.json</a></td> + <td>884</td> + <td>02-Sep-2019 00:48</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.mochitest.tests.tar.gz">firefox-70.0a1.en-US.win32.mochitest.tests.tar.gz</a></td> + <td>65M</td> + <td>02-Sep-2019 00:48</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.mozinfo.json">firefox-70.0a1.en-US.win32.mozinfo.json</a></td> + <td>948</td> + <td>02-Sep-2019 00:48</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.reftest.tests.tar.gz">firefox-70.0a1.en-US.win32.reftest.tests.tar.gz</a></td> + <td>53M</td> + <td>02-Sep-2019 00:48</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.talos.tests.tar.gz">firefox-70.0a1.en-US.win32.talos.tests.tar.gz</a></td> + <td>18M</td> + <td>02-Sep-2019 00:48</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.test_packages.json">firefox-70.0a1.en-US.win32.test_packages.json</a></td> + <td>1K</td> + <td>02-Sep-2019 00:48</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.txt">firefox-70.0a1.en-US.win32.txt</a></td> + <td>99</td> + <td>02-Sep-2019 00:48</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.web-platform.tests.tar.gz">firefox-70.0a1.en-US.win32.web-platform.tests.tar.gz</a></td> + <td>53M</td> + <td>02-Sep-2019 00:48</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.xpcshell.tests.tar.gz">firefox-70.0a1.en-US.win32.xpcshell.tests.tar.gz</a></td> + <td>9M</td> + <td>02-Sep-2019 00:48</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32.zip">firefox-70.0a1.en-US.win32.zip</a></td> + <td>70M</td> + <td>02-Sep-2019 00:48</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win32_info.txt">firefox-70.0a1.en-US.win32_info.txt</a></td> + <td>23</td> + <td>02-Sep-2019 00:48</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.awsy.tests.tar.gz">firefox-70.0a1.en-US.win64-aarch64.awsy.tests.tar.gz</a></td> + <td>20K</td> + <td>02-Sep-2019 01:15</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.buildhub.json">firefox-70.0a1.en-US.win64-aarch64.buildhub.json</a></td> + <td>914</td> + <td>02-Sep-2019 01:15</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.checksums">firefox-70.0a1.en-US.win64-aarch64.checksums</a></td> + <td>8K</td> + <td>02-Sep-2019 01:16</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.common.tests.tar.gz">firefox-70.0a1.en-US.win64-aarch64.common.tests.tar.gz</a></td> + <td>20M</td> + <td>02-Sep-2019 01:15</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.complete.mar">firefox-70.0a1.en-US.win64-aarch64.complete.mar</a></td> + <td>78M</td> + <td>02-Sep-2019 01:15</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.cppunittest.tests.tar.gz">firefox-70.0a1.en-US.win64-aarch64.cppunittest.tests.tar.gz</a></td> + <td>10M</td> + <td>02-Sep-2019 01:15</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.crashreporter-symbols.zip">firefox-70.0a1.en-US.win64-aarch64.crashreporter-symbols.zip</a></td> + <td>20M</td> + <td>02-Sep-2019 01:15</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.installer.exe">firefox-70.0a1.en-US.win64-aarch64.installer.exe</a></td> + <td>74M</td> + <td>02-Sep-2019 01:15</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.json">firefox-70.0a1.en-US.win64-aarch64.json</a></td> + <td>705</td> + <td>02-Sep-2019 01:15</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.mochitest.tests.tar.gz">firefox-70.0a1.en-US.win64-aarch64.mochitest.tests.tar.gz</a></td> + <td>65M</td> + <td>02-Sep-2019 01:15</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.mozinfo.json">firefox-70.0a1.en-US.win64-aarch64.mozinfo.json</a></td> + <td>946</td> + <td>02-Sep-2019 01:15</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.reftest.tests.tar.gz">firefox-70.0a1.en-US.win64-aarch64.reftest.tests.tar.gz</a></td> + <td>53M</td> + <td>02-Sep-2019 01:15</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.talos.tests.tar.gz">firefox-70.0a1.en-US.win64-aarch64.talos.tests.tar.gz</a></td> + <td>18M</td> + <td>02-Sep-2019 01:15</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.test_packages.json">firefox-70.0a1.en-US.win64-aarch64.test_packages.json</a></td> + <td>1K</td> + <td>02-Sep-2019 01:15</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.txt">firefox-70.0a1.en-US.win64-aarch64.txt</a></td> + <td>99</td> + <td>02-Sep-2019 01:15</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.web-platform.tests.tar.gz">firefox-70.0a1.en-US.win64-aarch64.web-platform.tests.tar.gz</a></td> + <td>53M</td> + <td>02-Sep-2019 01:15</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.xpcshell.tests.tar.gz">firefox-70.0a1.en-US.win64-aarch64.xpcshell.tests.tar.gz</a></td> + <td>9M</td> + <td>02-Sep-2019 01:15</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64.zip">firefox-70.0a1.en-US.win64-aarch64.zip</a></td> + <td>110M</td> + <td>02-Sep-2019 01:15</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-aarch64_info.txt">firefox-70.0a1.en-US.win64-aarch64_info.txt</a></td> + <td>23</td> + <td>02-Sep-2019 01:15</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.awsy.tests.tar.gz">firefox-70.0a1.en-US.win64-asan-reporter.awsy.tests.tar.gz</a></td> + <td>20K</td> + <td>02-Sep-2019 00:16</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.buildhub.json">firefox-70.0a1.en-US.win64-asan-reporter.buildhub.json</a></td> + <td>1K</td> + <td>02-Sep-2019 00:16</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.checksums">firefox-70.0a1.en-US.win64-asan-reporter.checksums</a></td> + <td>7K</td> + <td>02-Sep-2019 00:17</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.common.tests.tar.gz">firefox-70.0a1.en-US.win64-asan-reporter.common.tests.tar.gz</a></td> + <td>22M</td> + <td>02-Sep-2019 00:16</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.complete.mar">firefox-70.0a1.en-US.win64-asan-reporter.complete.mar</a></td> + <td>201M</td> + <td>02-Sep-2019 00:16</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.cppunittest.tests.tar.gz">firefox-70.0a1.en-US.win64-asan-reporter.cppunittest.tests.tar.gz</a></td> + <td>56M</td> + <td>02-Sep-2019 00:16</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.installer.exe">firefox-70.0a1.en-US.win64-asan-reporter.installer.exe</a></td> + <td>191M</td> + <td>02-Sep-2019 00:16</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.json">firefox-70.0a1.en-US.win64-asan-reporter.json</a></td> + <td>894</td> + <td>02-Sep-2019 00:16</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.mochitest.tests.tar.gz">firefox-70.0a1.en-US.win64-asan-reporter.mochitest.tests.tar.gz</a></td> + <td>65M</td> + <td>02-Sep-2019 00:16</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.mozinfo.json">firefox-70.0a1.en-US.win64-asan-reporter.mozinfo.json</a></td> + <td>957</td> + <td>02-Sep-2019 00:16</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.reftest.tests.tar.gz">firefox-70.0a1.en-US.win64-asan-reporter.reftest.tests.tar.gz</a></td> + <td>53M</td> + <td>02-Sep-2019 00:16</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.talos.tests.tar.gz">firefox-70.0a1.en-US.win64-asan-reporter.talos.tests.tar.gz</a></td> + <td>18M</td> + <td>02-Sep-2019 00:16</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.test_packages.json">firefox-70.0a1.en-US.win64-asan-reporter.test_packages.json</a></td> + <td>1K</td> + <td>02-Sep-2019 00:16</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.txt">firefox-70.0a1.en-US.win64-asan-reporter.txt</a></td> + <td>99</td> + <td>02-Sep-2019 00:16</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.web-platform.tests.tar.gz">firefox-70.0a1.en-US.win64-asan-reporter.web-platform.tests.tar.gz</a></td> + <td>53M</td> + <td>02-Sep-2019 00:16</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.xpcshell.tests.tar.gz">firefox-70.0a1.en-US.win64-asan-reporter.xpcshell.tests.tar.gz</a></td> + <td>9M</td> + <td>02-Sep-2019 00:16</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter.zip">firefox-70.0a1.en-US.win64-asan-reporter.zip</a></td> + <td>307M</td> + <td>02-Sep-2019 00:16</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64-asan-reporter_info.txt">firefox-70.0a1.en-US.win64-asan-reporter_info.txt</a></td> + <td>23</td> + <td>02-Sep-2019 00:16</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.awsy.tests.tar.gz">firefox-70.0a1.en-US.win64.awsy.tests.tar.gz</a></td> + <td>20K</td> + <td>02-Sep-2019 00:44</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.buildhub.json">firefox-70.0a1.en-US.win64.buildhub.json</a></td> + <td>1K</td> + <td>02-Sep-2019 00:44</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.checksums">firefox-70.0a1.en-US.win64.checksums</a></td> + <td>8K</td> + <td>02-Sep-2019 00:45</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.common.tests.tar.gz">firefox-70.0a1.en-US.win64.common.tests.tar.gz</a></td> + <td>22M</td> + <td>02-Sep-2019 00:44</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.complete.mar">firefox-70.0a1.en-US.win64.complete.mar</a></td> + <td>53M</td> + <td>02-Sep-2019 00:44</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.cppunittest.tests.tar.gz">firefox-70.0a1.en-US.win64.cppunittest.tests.tar.gz</a></td> + <td>11M</td> + <td>02-Sep-2019 00:44</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.crashreporter-symbols.zip">firefox-70.0a1.en-US.win64.crashreporter-symbols.zip</a></td> + <td>25M</td> + <td>02-Sep-2019 00:44</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.installer.exe">firefox-70.0a1.en-US.win64.installer.exe</a></td> + <td>50M</td> + <td>02-Sep-2019 00:44</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.installer.msi">firefox-70.0a1.en-US.win64.installer.msi</a></td> + <td>50M</td> + <td>02-Sep-2019 00:44</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.json">firefox-70.0a1.en-US.win64.json</a></td> + <td>880</td> + <td>02-Sep-2019 00:44</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.mochitest.tests.tar.gz">firefox-70.0a1.en-US.win64.mochitest.tests.tar.gz</a></td> + <td>65M</td> + <td>02-Sep-2019 00:44</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.mozinfo.json">firefox-70.0a1.en-US.win64.mozinfo.json</a></td> + <td>951</td> + <td>02-Sep-2019 00:44</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.reftest.tests.tar.gz">firefox-70.0a1.en-US.win64.reftest.tests.tar.gz</a></td> + <td>53M</td> + <td>02-Sep-2019 00:44</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.talos.tests.tar.gz">firefox-70.0a1.en-US.win64.talos.tests.tar.gz</a></td> + <td>18M</td> + <td>02-Sep-2019 00:44</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.test_packages.json">firefox-70.0a1.en-US.win64.test_packages.json</a></td> + <td>1K</td> + <td>02-Sep-2019 00:44</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.txt">firefox-70.0a1.en-US.win64.txt</a></td> + <td>99</td> + <td>02-Sep-2019 00:44</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.web-platform.tests.tar.gz">firefox-70.0a1.en-US.win64.web-platform.tests.tar.gz</a></td> + <td>53M</td> + <td>02-Sep-2019 00:44</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.xpcshell.tests.tar.gz">firefox-70.0a1.en-US.win64.xpcshell.tests.tar.gz</a></td> + <td>9M</td> + <td>02-Sep-2019 00:44</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64.zip">firefox-70.0a1.en-US.win64.zip</a></td> + <td>73M</td> + <td>02-Sep-2019 00:44</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/firefox-70.0a1.en-US.win64_info.txt">firefox-70.0a1.en-US.win64_info.txt</a></td> + <td>23</td> + <td>02-Sep-2019 00:44</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/jsshell-linux-i686.zip">jsshell-linux-i686.zip</a></td> + <td>11M</td> + <td>01-Sep-2019 23:49</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/jsshell-linux-x86_64.zip">jsshell-linux-x86_64.zip</a></td> + <td>11M</td> + <td>02-Sep-2019 00:06</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/jsshell-mac.zip">jsshell-mac.zip</a></td> + <td>11M</td> + <td>01-Sep-2019 23:19</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/jsshell-win32.zip">jsshell-win32.zip</a></td> + <td>10M</td> + <td>02-Sep-2019 00:48</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/jsshell-win64-aarch64.zip">jsshell-win64-aarch64.zip</a></td> + <td>1M</td> + <td>02-Sep-2019 01:15</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/jsshell-win64.zip">jsshell-win64.zip</a></td> + <td>11M</td> + <td>02-Sep-2019 00:44</td> + </tr> + + + + <tr> + <td>File</td> + <td><a href="/pub/firefox/nightly/latest-mozilla-central/mozharness.zip">mozharness.zip</a></td> + <td>2M</td> + <td>02-Sep-2019 01:43</td> + </tr> + + + </table> + </body> +</html> diff --git a/testing/condprofile/condprof/tests/profile/prefs.js b/testing/condprofile/condprof/tests/profile/prefs.js new file mode 100644 index 0000000000..abbe0fb70b --- /dev/null +++ b/testing/condprofile/condprof/tests/profile/prefs.js @@ -0,0 +1 @@ +user_pref("gfx.blacklist.direct2d", 1); diff --git a/testing/condprofile/condprof/tests/profile/user.js b/testing/condprofile/condprof/tests/profile/user.js new file mode 100644 index 0000000000..c1efc79c10 --- /dev/null +++ b/testing/condprofile/condprof/tests/profile/user.js @@ -0,0 +1,9 @@ + +#Prefs used for the unit test +user_pref("focusmanager.testmode", true); +user_pref("marionette.defaultPrefs.port", 2828); +user_pref("marionette.port", 2828); +user_pref("remote.log.level", "Trace"); +user_pref("marionette.log.truncate", false); +user_pref("extensions.autoDisableScopes", 0); +user_pref("devtools.debugger.remote-enabled", true); diff --git a/testing/condprofile/condprof/tests/python.toml b/testing/condprofile/condprof/tests/python.toml new file mode 100644 index 0000000000..14cffb2f5c --- /dev/null +++ b/testing/condprofile/condprof/tests/python.toml @@ -0,0 +1,4 @@ +[DEFAULT] +subsuite = "condprof" + +["test_client.py"] diff --git a/testing/condprofile/condprof/tests/test_client.py b/testing/condprofile/condprof/tests/test_client.py new file mode 100644 index 0000000000..8410133e16 --- /dev/null +++ b/testing/condprofile/condprof/tests/test_client.py @@ -0,0 +1,126 @@ +import json +import os +import re +import shutil +import tarfile +import tempfile +import unittest + +import responses +from mozprofile.prefs import Preferences + +from condprof.client import ROOT_URL, TC_SERVICE, get_profile +from condprof.util import _DEFAULT_SERVER + +PROFILE = re.compile(ROOT_URL + "/.*/.*tgz") +PROFILE_FOR_TESTS = os.path.join(os.path.dirname(__file__), "profile") +SECRETS = re.compile(_DEFAULT_SERVER + "/.*") +SECRETS_PROXY = re.compile("http://taskcluster/secrets/.*") + + +class TestClient(unittest.TestCase): + def setUp(self): + self.profile_dir = tempfile.mkdtemp() + + # creating profile.tgz on the fly for serving it + profile_tgz = os.path.join(self.profile_dir, "profile.tgz") + with tarfile.open(profile_tgz, "w:gz") as tar: + tar.add(PROFILE_FOR_TESTS, arcname=".") + + # self.profile_data is the tarball we're sending back via HTTP + with open(profile_tgz, "rb") as f: + self.profile_data = f.read() + + self.target = tempfile.mkdtemp() + self.download_dir = os.path.expanduser("~/.condprof-cache") + if os.path.exists(self.download_dir): + shutil.rmtree(self.download_dir) + + responses.add( + responses.GET, + PROFILE, + body=self.profile_data, + headers={"content-length": str(len(self.profile_data)), "ETag": "'12345'"}, + status=200, + ) + + responses.add( + responses.HEAD, + PROFILE, + body="", + headers={"content-length": str(len(self.profile_data)), "ETag": "'12345'"}, + status=200, + ) + + responses.add(responses.HEAD, TC_SERVICE, body="", status=200) + + secret = {"secret": {"username": "user", "password": "pass"}} + secret = json.dumps(secret) + for pattern in (SECRETS, SECRETS_PROXY): + responses.add( + responses.GET, + pattern, + body=secret, + headers={"content-length": str(len(secret))}, + status=200, + ) + + def tearDown(self): + shutil.rmtree(self.target) + shutil.rmtree(self.download_dir) + shutil.rmtree(self.profile_dir) + + @responses.activate + def test_cache(self): + download_dir = os.path.expanduser("~/.condprof-cache") + if os.path.exists(download_dir): + num_elmts = len(os.listdir(download_dir)) + else: + num_elmts = 0 + + get_profile(self.target, "win64", "settled", "default") + + # grabbing a profile should generate two files + self.assertEqual(len(os.listdir(download_dir)), num_elmts + 2) + + # we do at least two network calls when getting a file, + # a HEAD and a GET and possibly a TC secret + self.assertTrue(len(responses.calls) >= 2) + + # reseting the response counters + responses.calls.reset() + + # and we should reuse them without downloading the file again + get_profile(self.target, "win64", "settled", "default") + + # grabbing a profile should not download new stuff + self.assertEqual(len(os.listdir(download_dir)), num_elmts + 2) + + # and do a single extra HEAD call, everything else is cached, + # even the TC secret + self.assertEqual(len(responses.calls), 2) + + prefs_js = os.path.join(self.target, "prefs.js") + prefs = Preferences.read_prefs(prefs_js) + + # check that the gfx.blacklist prefs where cleaned out + for name, value in prefs: + self.assertFalse(name.startswith("gfx.blacklist")) + + # check that we have the startupScanScopes option forced + prefs = dict(prefs) + self.assertEqual(prefs["extensions.startupScanScopes"], 1) + + # make sure we don't have any marionette option set + user_js = os.path.join(self.target, "user.js") + for name, value in Preferences.read_prefs(user_js): + self.assertFalse(name.startswith("marionette.")) + + +if __name__ == "__main__": + try: + import mozunit + except ImportError: + pass + else: + mozunit.main(runwith="unittest") diff --git a/testing/condprofile/condprof/tests/test_runner.py b/testing/condprofile/condprof/tests/test_runner.py new file mode 100644 index 0000000000..e200a537e1 --- /dev/null +++ b/testing/condprofile/condprof/tests/test_runner.py @@ -0,0 +1,123 @@ +import asyncio +import os +import re +import shutil +import tarfile +import tempfile +import unittest + +import responses + +from condprof import client +from condprof.client import ROOT_URL, TC_SERVICE +from condprof.main import main + +client.RETRIES = 1 +client.RETRY_PAUSE = 0 +GECKODRIVER = os.path.join(os.path.dirname(__file__), "fakegeckodriver.py") +FIREFOX = os.path.join(os.path.dirname(__file__), "fakefirefox.py") +CHANGELOG = re.compile(ROOT_URL + "/.*/changelog.json") +FTP = "https://ftp.mozilla.org/pub/firefox/nightly/latest-mozilla-central/" +PROFILE = re.compile(ROOT_URL + "/.*/.*tgz") + + +with open(os.path.join(os.path.dirname(__file__), "ftp_mozilla.html")) as f: + FTP_PAGE = f.read() + +FTP_ARCHIVE = re.compile( + "https://ftp.mozilla.org/pub/firefox/nightly/" "latest-mozilla-central/firefox.*" +) + +ADDON = re.compile("https://addons.mozilla.org/.*/.*xpi") + + +async def fakesleep(duration): + if duration > 1: + duration = 1 + await asyncio.realsleep(duration) + + +asyncio.realsleep = asyncio.sleep +asyncio.sleep = fakesleep + + +class TestRunner(unittest.TestCase): + def setUp(self): + self.archive_dir = tempfile.mkdtemp() + responses.add(responses.GET, CHANGELOG, json={"error": "not found"}, status=404) + responses.add( + responses.GET, FTP, content_type="text/html", body=FTP_PAGE, status=200 + ) + + profile_tgz = os.path.join(os.path.dirname(__file__), "profile.tgz") + profile = os.path.join(os.path.dirname(__file__), "profile") + + with tarfile.open(profile_tgz, "w:gz") as tar: + tar.add(profile, arcname=".") + + with open(profile_tgz, "rb") as f: + PROFILE_DATA = f.read() + + os.remove(profile_tgz) + + responses.add( + responses.GET, + FTP_ARCHIVE, + body="1", + headers={"content-length": "1"}, + status=200, + ) + + responses.add( + responses.GET, + PROFILE, + body=PROFILE_DATA, + headers={"content-length": str(len(PROFILE_DATA))}, + status=200, + ) + + responses.add( + responses.HEAD, + PROFILE, + body="", + headers={"content-length": str(len(PROFILE_DATA))}, + status=200, + ) + + responses.add(responses.HEAD, FTP_ARCHIVE, body="", status=200) + + responses.add( + responses.GET, ADDON, body="1", headers={"content-length": "1"}, status=200 + ) + + responses.add( + responses.HEAD, ADDON, body="", headers={"content-length": "1"}, status=200 + ) + + responses.add(responses.HEAD, TC_SERVICE, body="", status=200) + + def tearDown(self): + shutil.rmtree(self.archive_dir) + + @responses.activate + def test_runner(self): + if "FXA_USERNAME" not in os.environ: + os.environ["FXA_USERNAME"] = "me" + if "FXA_PASSWORD" not in os.environ: + os.environ["FXA_PASSWORD"] = "password" + try: + args = [ + "--geckodriver", + GECKODRIVER, + "--firefox", + FIREFOX, + self.archive_dir, + ] + main(args) + # XXX we want a bunch of assertions here to check + # that the archives dir gets filled correctly + finally: + if os.environ["FXA_USERNAME"] == "me": + del os.environ["FXA_USERNAME"] + if os.environ["FXA_PASSWORD"] == "password": + del os.environ["FXA_PASSWORD"] diff --git a/testing/condprofile/condprof/util.py b/testing/condprofile/condprof/util.py new file mode 100644 index 0000000000..444398b9f4 --- /dev/null +++ b/testing/condprofile/condprof/util.py @@ -0,0 +1,484 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This module needs to stay Python 2 and 3 compatible +# +import contextlib +import os +import platform +import shutil +import sys +import tempfile +import time +from subprocess import PIPE, Popen + +import mozlog +import requests +import yaml +from requests.exceptions import ConnectionError +from requests.packages.urllib3.util.retry import Retry + +from condprof import progress + +TASK_CLUSTER = "MOZ_AUTOMATION" in os.environ.keys() +DOWNLOAD_TIMEOUT = 30 + + +class ArchiveNotFound(Exception): + pass + + +DEFAULT_PREFS = { + "focusmanager.testmode": True, + "marionette.defaultPrefs.port": 2828, + "marionette.port": 2828, + "remote.log.level": "Trace", + "marionette.log.truncate": False, + "extensions.autoDisableScopes": 10, + "devtools.debugger.remote-enabled": True, + "devtools.console.stdout.content": True, + "devtools.console.stdout.chrome": True, +} + +DEFAULT_CUSTOMIZATION = os.path.join( + os.path.dirname(__file__), "customization", "default.json" +) +STRUCTLOG_PAD_SIZE = 20 + + +class BridgeLogger: + def __init__(self, logger): + self.logger = logger + + def _find(self, text, *names): + # structlog's ConsoleRenderer pads values + for name in names: + if name + " " * STRUCTLOG_PAD_SIZE in text: + return True + return False + + def _convert(self, message): + return obfuscate(message)[1] + + def info(self, message, *args, **kw): + if not isinstance(message, str): + message = str(message) + # converting Arsenic request/response struct log + if self._find(message, "request", "response"): + self.logger.debug(self._convert(message), *args, **kw) + else: + self.logger.info(self._convert(message), *args, **kw) + + def error(self, message, *args, **kw): + self.logger.error(self._convert(message), *args, **kw) + + def warning(self, message, *args, **kw): + self.logger.warning(self._convert(message), *args, **kw) + + +logger = None + + +def get_logger(): + global logger + if logger is not None: + return logger + new_logger = mozlog.get_default_logger("condprof") + if new_logger is None: + new_logger = mozlog.unstructured.getLogger("condprof") + + # wrap the logger into the BridgeLogger + new_logger = BridgeLogger(new_logger) + + # bridge for Arsenic + if sys.version_info.major == 3: + try: + from arsenic import connection + from structlog import wrap_logger + + connection.log = wrap_logger(new_logger) + except ImportError: + # Arsenic is not installed for client-only usage + pass + logger = new_logger + return logger + + +# initializing the logger right away +get_logger() + + +def fresh_profile(profile, customization_data): + from mozprofile import create_profile # NOQA + + # XXX on android we mgiht need to run it on the device? + logger.info("Creating a fresh profile") + new_profile = create_profile(app="firefox") + prefs = customization_data["prefs"] + prefs.update(DEFAULT_PREFS) + logger.info("Setting prefs %s" % str(prefs.items())) + new_profile.set_preferences(prefs) + extensions = [] + for name, url in customization_data["addons"].items(): + logger.info("Downloading addon %s" % name) + # When running on the CI, we expect the xpi files to have been + # fetched by the firefox-addons fetch task dependency (see + # taskcluster/ci/fetch/browsertime.yml) and the condprof-addons + # linter enforces the content of the archive to be unpacked into + # a subdirectory named "firefox-addons". + extension = download_file(url, mozfetches_subdir="firefox-addons") + extensions.append(extension) + logger.info("Installing addons") + new_profile.addons.install(extensions) + shutil.copytree(new_profile.profile, profile) + return profile + + +link = "https://ftp.mozilla.org/pub/firefox/nightly/latest-mozilla-central/" + + +def get_firefox_download_link(): + try: + from bs4 import BeautifulSoup + except ImportError: + raise ImportError("You need to run pip install beautifulsoup4") + if platform.system() == "Darwin": + extension = ".dmg" + elif platform.system() == "Linux": + arch = platform.machine() + extension = ".linux-%s.tar.bz2" % arch + else: + raise NotImplementedError(platform.system()) + + page = requests.get(link).text + soup = BeautifulSoup(page, "html.parser") + for node in soup.find_all("a", href=True): + href = node["href"] + if href.endswith(extension): + return "https://ftp.mozilla.org" + href + + raise Exception() + + +def check_exists(archive, server=None, all_types=False): + if server is not None: + archive = server + "/" + archive + try: + logger.info("Getting headers at %s" % archive) + resp = requests.head(archive, timeout=DOWNLOAD_TIMEOUT) + except ConnectionError: + return False, {} + + if resp.status_code in (302, 303): + logger.info("Redirected") + return check_exists(resp.headers["Location"]) + + # see Bug 1574854 + if ( + not all_types + and resp.status_code == 200 + and "text/html" in resp.headers["Content-Type"] + ): + logger.info("Got an html page back") + exists = False + else: + logger.info("Response code is %d" % resp.status_code) + exists = resp.status_code + + return exists, resp.headers + + +def check_mozfetches_dir(target, mozfetches_subdir): + logger.info("Checking for existence of: %s in MOZ_FETCHES_DIR" % target) + fetches = os.environ.get("MOZ_FETCHES_DIR") + if fetches is None: + return None + fetches_target = os.path.join(fetches, mozfetches_subdir, target) + if not os.path.exists(fetches_target): + return None + logger.info("Already fetched and available in MOZ_FETCHES_DIR: %s" % fetches_target) + return fetches_target + + +def download_file(url, target=None, mozfetches_subdir=None): + if target is None: + target = url.split("/")[-1] + + # check if the assets has been fetched through a taskgraph fetch task dependency + # and already available in the MOZ_FETCHES_DIR passed as an additional parameter. + if mozfetches_subdir is not None: + filepath = check_mozfetches_dir(target, mozfetches_subdir) + if filepath is not None: + return filepath + + present, headers = check_exists(url) + if not present: + logger.info("Cannot find %r" % url) + raise ArchiveNotFound(url) + + etag = headers.get("ETag") + + logger.info("Checking for existence of: %s" % target) + if os.path.exists(target): + # XXX for now, reusing downloads without checking them + # when we don't have an .etag file + if etag is None or not os.path.exists(target + ".etag"): + logger.info("No existing etag downloads.") + return target + with open(target + ".etag") as f: + current_etag = f.read() + if etag == current_etag: + logger.info("Already Downloaded.") + # should at least check the size? + return target + else: + logger.info("Changed!") + else: + logger.info("Could not find an existing archive.") + # Add some debugging logs for the directory content + try: + archivedir = os.path.dirname(target) + logger.info( + "Content in cache directory %s: %s" + % (archivedir, os.listdir(archivedir)) + ) + except Exception: + logger.info("Failed to list cache directory contents") + + logger.info("Downloading %s" % url) + req = requests.get(url, stream=True, timeout=DOWNLOAD_TIMEOUT) + total_length = int(req.headers.get("content-length")) + target_dir = os.path.dirname(target) + if target_dir != "" and not os.path.exists(target_dir): + logger.info("Creating dir %s" % target_dir) + os.makedirs(target_dir) + + with open(target, "wb") as f: + if TASK_CLUSTER: + for chunk in req.iter_content(chunk_size=1024): + if chunk: + f.write(chunk) + f.flush() + else: + iter = req.iter_content(chunk_size=1024) + # pylint --py3k W1619 + size = total_length / 1024 + 1 + for chunk in progress.bar(iter, expected_size=size): + if chunk: + f.write(chunk) + f.flush() + + if etag is not None: + with open(target + ".etag", "w") as f: + f.write(etag) + + return target + + +def extract_from_dmg(dmg, target): + mount = tempfile.mkdtemp() + cmd = "hdiutil attach -nobrowse -mountpoint %s %s" + os.system(cmd % (mount, dmg)) + try: + found = False + for f in os.listdir(mount): + if not f.endswith(".app"): + continue + app = os.path.join(mount, f) + shutil.copytree(app, target) + found = True + break + finally: + os.system("hdiutil detach " + mount) + shutil.rmtree(mount) + if not found: + raise IOError("No app file found in %s" % dmg) + + +@contextlib.contextmanager +def latest_nightly(binary=None): + if binary is None: + # we want to use the latest nightly + nightly_archive = get_firefox_download_link() + logger.info("Downloading %s" % nightly_archive) + target = download_file(nightly_archive) + # on macOs we just mount the DMG + # XXX replace with extract_from_dmg + if platform.system() == "Darwin": + cmd = "hdiutil attach -mountpoint /Volumes/Nightly %s" + os.system(cmd % target) + binary = "/Volumes/Nightly/Firefox Nightly.app/Contents/MacOS/firefox" + # on linux we unpack it + elif platform.system() == "Linux": + cmd = "bunzip2 %s" % target + os.system(cmd) + cmd = "tar -xvf %s" % target[: -len(".bz2")] + os.system(cmd) + binary = "firefox/firefox" + + mounted = True + else: + mounted = False + try: + yield binary + finally: + # XXX replace with extract_from_dmg + if mounted: + if platform.system() == "Darwin": + logger.info("Unmounting Firefox") + time.sleep(10) + os.system("hdiutil detach /Volumes/Nightly") + elif platform.system() == "Linux": + # XXX we should keep it for next time + shutil.rmtree("firefox") + + +def write_yml_file(yml_file, yml_data): + logger.info("writing %s to %s" % (yml_data, yml_file)) + try: + with open(yml_file, "w") as outfile: + yaml.dump(yml_data, outfile, default_flow_style=False) + except Exception: + logger.error("failed to write yaml file", exc_info=True) + + +def get_version(firefox): + p = Popen([firefox, "--version"], stdin=PIPE, stdout=PIPE, stderr=PIPE) + output, __ = p.communicate() + first_line = output.strip().split(b"\n")[0] + res = first_line.split()[-1] + return res.decode("utf-8") + + +def get_current_platform(): + """Returns a combination of system and arch info that matches TC standards. + + e.g. macosx64, win32, linux64, etc.. + """ + arch = sys.maxsize == 2**63 - 1 and "64" or "32" + plat = platform.system().lower() + if plat == "windows": + plat = "win" + elif plat == "darwin": + plat = "macosx" + return plat + arch + + +class BaseEnv: + def __init__(self, profile, firefox, geckodriver, archive, device_name): + self.profile = profile + self.firefox = firefox + self.geckodriver = geckodriver + if profile is None: + self.profile = os.path.join(tempfile.mkdtemp(), "profile") + else: + self.profile = profile + self.archive = archive + self.device_name = device_name + + @property + def target_platform(self): + return self.get_target_platform() + + def get_target_platform(self): + raise NotImplementedError() + + def get_device(self, *args, **kw): + raise NotImplementedError() + + @contextlib.contextmanager + def get_browser(self, path): + raise NotImplementedError() + + def get_browser_args(self, headless): + raise NotImplementedError() + + def prepare(self, logfile): + pass + + def check_session(self, session): + pass + + def dump_logs(self): + pass + + def get_browser_version(self): + raise NotImplementedError() + + def get_geckodriver(self, log_file): + raise NotImplementedError() + + def collect_profile(self): + pass + + def stop_browser(self): + pass + + +_URL = ( + "{0}/secrets/v1/secret/project" + "{1}releng{1}gecko{1}build{1}level-{2}{1}conditioned-profiles" +) +_DEFAULT_SERVER = "https://firefox-ci-tc.services.mozilla.com" + + +def get_tc_secret(): + if not TASK_CLUSTER: + raise OSError("Not running in Taskcluster") + session = requests.Session() + retry = Retry(total=5, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504]) + http_adapter = requests.adapters.HTTPAdapter(max_retries=retry) + session.mount("https://", http_adapter) + session.mount("http://", http_adapter) + secrets_url = _URL.format( + os.environ.get("TASKCLUSTER_PROXY_URL", _DEFAULT_SERVER), + "%2F", + os.environ.get("MOZ_SCM_LEVEL", "1"), + ) + res = session.get(secrets_url, timeout=DOWNLOAD_TIMEOUT) + res.raise_for_status() + return res.json()["secret"] + + +_CACHED = {} + + +def obfuscate(text): + if "CONDPROF_RUNNER" not in os.environ: + return True, text + username, password = get_credentials() + if username is None: + return False, text + if username not in text and password not in text: + return False, text + text = text.replace(password, "<PASSWORD>") + text = text.replace(username, "<USERNAME>") + return True, text + + +def obfuscate_file(path): + if "CONDPROF_RUNNER" not in os.environ: + return + with open(path) as f: + data = f.read() + hit, data = obfuscate(data) + if not hit: + return + with open(path, "w") as f: + f.write(data) + + +def get_credentials(): + if "creds" in _CACHED: + return _CACHED["creds"] + password = os.environ.get("FXA_PASSWORD") + username = os.environ.get("FXA_USERNAME") + if username is None or password is None: + if not TASK_CLUSTER: + return None, None + secret = get_tc_secret() + password = secret["password"] + username = secret["username"] + _CACHED["creds"] = username, password + return username, password diff --git a/testing/condprofile/mach_commands.py b/testing/condprofile/mach_commands.py new file mode 100644 index 0000000000..e9a0d54867 --- /dev/null +++ b/testing/condprofile/mach_commands.py @@ -0,0 +1,121 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. +import logging +import os +import sys +import tempfile + +from mach.decorators import Command, CommandArgument +from mozbuild.base import BinaryNotFoundException + +requirements = os.path.join(os.path.dirname(__file__), "requirements", "base.txt") + + +def _init(command_context): + command_context.activate_virtualenv() + command_context.virtualenv_manager.install_pip_requirements( + requirements, require_hashes=False + ) + + +@Command("fetch-condprofile", category="testing") +@CommandArgument("--target-dir", default=None, help="Target directory") +@CommandArgument("--platform", default=None, help="Platform") +@CommandArgument("--scenario", default="full", help="Scenario") # grab choices +@CommandArgument("--customization", default="default", help="Customization") # same +@CommandArgument("--task-id", default=None, help="Task ID") +@CommandArgument("--download-cache", action="store_true", default=True) +@CommandArgument( + "--repo", + default="mozilla-central", + choices=["mozilla-central", "try"], + help="Repository", +) +def fetch( + command_context, + target_dir, + platform, + scenario, + customization, + task_id, + download_cache, + repo, +): + _init(command_context) + from condprof.client import get_profile + from condprof.util import get_current_platform, get_version + + if platform is None: + platform = get_current_platform() + + if target_dir is None: + target_dir = tempfile.mkdtemp() + + version = get_version(command_context.get_binary_path()) + + get_profile( + target_dir, + platform, + scenario, + customization, + task_id, + download_cache, + repo, + version, + ) + print("Downloaded conditioned profile can be found at: %s" % target_dir) + + +@Command("run-condprofile", category="testing") +@CommandArgument("archive", help="Archives Dir", type=str, default=None) +@CommandArgument("--firefox", help="Firefox Binary", type=str, default=None) +@CommandArgument("--scenario", help="Scenario to use", type=str, default="all") +@CommandArgument("--profile", help="Existing profile Dir", type=str, default=None) +@CommandArgument( + "--customization", help="Profile customization to use", type=str, default="all" +) +@CommandArgument( + "--visible", help="Don't use headless mode", action="store_true", default=False +) +@CommandArgument( + "--archives-dir", help="Archives local dir", type=str, default="/tmp/archives" +) +@CommandArgument( + "--force-new", help="Create from scratch", action="store_true", default=False +) +@CommandArgument( + "--strict", + help="Errors out immediatly on a scenario failure", + action="store_true", + default=True, +) +@CommandArgument( + "--geckodriver", + help="Path to the geckodriver binary", + type=str, + default=sys.platform.startswith("win") and "geckodriver.exe" or "geckodriver", +) +@CommandArgument("--device-name", help="Name of the device", type=str, default=None) +def run(command_context, **kw): + os.environ["MANUAL_MACH_RUN"] = "1" + _init(command_context) + + if kw["firefox"] is None: + try: + kw["firefox"] = command_context.get_binary_path() + except BinaryNotFoundException as e: + command_context.log( + logging.ERROR, + "run-condprofile", + {"error": str(e)}, + "ERROR: {error}", + ) + command_context.log( + logging.INFO, "run-condprofile", {"help": e.help()}, "{help}" + ) + return 1 + + from condprof.runner import run + + run(**kw) diff --git a/testing/condprofile/moz.build b/testing/condprofile/moz.build new file mode 100644 index 0000000000..94cd2f1555 --- /dev/null +++ b/testing/condprofile/moz.build @@ -0,0 +1,7 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +with Files("**"): + BUG_COMPONENT = ("Testing", "Condprofile") + SCHEDULES.exclusive = ["condprofile"] diff --git a/testing/condprofile/requirements/base.txt b/testing/condprofile/requirements/base.txt new file mode 100644 index 0000000000..fc7d70013a --- /dev/null +++ b/testing/condprofile/requirements/base.txt @@ -0,0 +1,5 @@ +aiohttp==3.7.4.post0 +https://pypi.pub.build.mozilla.org/pub/arsenic-19.1-py3-none-any.whl +requests==2.22.0 +pyyaml==5.1.2 +mozversion>=1.0 diff --git a/testing/condprofile/requirements/ci-client.txt b/testing/condprofile/requirements/ci-client.txt new file mode 100644 index 0000000000..f04776863f --- /dev/null +++ b/testing/condprofile/requirements/ci-client.txt @@ -0,0 +1,10 @@ +# pulled when running in TaskCluster for python 2 only +# we just pull dependencies required by condprof.client +requests==2.22.0 +pyyaml==5.1.2 + +# the target.condprof.tests.tar.gz archive pulls those dependencies +# directly into the condprof project root +./mozfile +./mozlog +./mozprofile diff --git a/testing/condprofile/requirements/ci.txt b/testing/condprofile/requirements/ci.txt new file mode 100644 index 0000000000..d1e4f62bef --- /dev/null +++ b/testing/condprofile/requirements/ci.txt @@ -0,0 +1,7 @@ +# pulled when running in TaskCluster +# the target.condprof.tests.tar.gz archive pulls those dependencies +# directly into the condprof project root. +./mozfile +./mozlog +./mozprofile +./mozdevice diff --git a/testing/condprofile/requirements/local-client.txt b/testing/condprofile/requirements/local-client.txt new file mode 100644 index 0000000000..8a5e4da1e2 --- /dev/null +++ b/testing/condprofile/requirements/local-client.txt @@ -0,0 +1,6 @@ +./mozbase/mozfile +./mozbase/mozlog +./mozbase/mozprofile + +requests==2.22.0 +pyyaml==5.1.2 diff --git a/testing/condprofile/requirements/local.txt b/testing/condprofile/requirements/local.txt new file mode 100644 index 0000000000..b487db7274 --- /dev/null +++ b/testing/condprofile/requirements/local.txt @@ -0,0 +1,5 @@ +# pulled when running locally +../mozbase/mozfile +../mozbase/mozlog +../mozbase/mozprofile +../mozbase/mozdevice diff --git a/testing/condprofile/requirements/tox.txt b/testing/condprofile/requirements/tox.txt new file mode 100644 index 0000000000..772a4a4818 --- /dev/null +++ b/testing/condprofile/requirements/tox.txt @@ -0,0 +1,8 @@ +# pulled when running tox locally +-r requirements/local.txt + +pytest +pytest-cov +pytest-random-order +coveralls +responses diff --git a/testing/condprofile/setup.py b/testing/condprofile/setup.py new file mode 100644 index 0000000000..0648bff232 --- /dev/null +++ b/testing/condprofile/setup.py @@ -0,0 +1,32 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from setuptools import find_packages, setup + +entry_points = """ + [console_scripts] + cp-creator = condprof.main:main + cp-client = condprof.client:main + """ + +setup( + name="conditioned-profile", + version="0.2", + packages=find_packages(), + description="Firefox Heavy Profile creator", + include_package_data=True, + data_files=[ + ( + "condprof", + [ + "condprof/customization/default.json", + "condprof/customization/youtube.json", + "condprof/customization/webext.json", + ], + ) + ], + zip_safe=False, + install_requires=[], # use requirements files + entry_points=entry_points, +) diff --git a/testing/condprofile/tox.ini b/testing/condprofile/tox.ini new file mode 100644 index 0000000000..ddaca5911a --- /dev/null +++ b/testing/condprofile/tox.ini @@ -0,0 +1,16 @@ +[tox] +downloadcache = {toxworkdir}/cache/ +envlist = py36,flake8 + +[testenv] +passenv = TRAVIS TRAVIS_JOB_ID TRAVIS_BRANCH +deps = -rrequirements/tox.txt +commands = + pytest --random-order-bucket=global -sv --cov-report= --cov-config .coveragerc --cov condprof condprof/tests + - coverage report -m + - coveralls + +[testenv:flake8] +commands = flake8 condprof +deps = + flake8 |