Adding upstream version 110.0.1.upstream/110.0.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

20 files changed, 1298 insertions, 0 deletions

diff --git a/testing/tps/.gitignore b/testing/tps/.gitignore
new file mode 100644
index 0000000000..65f8b6e053
--- /dev/null
+++ b/testing/tps/.gitignore
@@ -0,0 +1,4 @@
+# These files are added by running the TPS test suite.
+build/
+dist/
+tps.egg-info/
diff --git a/testing/tps/README b/testing/tps/README
new file mode 100644
index 0000000000..0612bea4c6
--- /dev/null
+++ b/testing/tps/README
@@ -0,0 +1,58 @@
+TPS is a test automation framework for Firefox Sync. See
+https://developer.mozilla.org/en/TPS for documentation.
+
+Installation
+============
+
+TPS requires several packages to operate properly. To install TPS and
+required packages, use the INSTALL.sh script, provided:
+
+  python3 create_venv.py /path/to/create/virtualenv
+
+This script will create a virtalenv and install TPS into it.
+
+You must then activate the virtualenv by executing:
+
+-  (mac/linux): source /path/to/virtualenv/Scripts/activate
+-  (win): /path/to/virtualenv/Scripts/activate.bat
+
+TPS can then be run by executing:
+
+  runtps --binary=/path/to/firefox
+
+> Note: You can run the tps tests in headless mode by using `MOZ_HEADLESS=1`. This will make
+> your computer somewhat useable while the tests are running.
+
+When you are done with TPS, you can deactivate the virtualenv by executing
+`deactivate`
+
+Configuration
+=============
+To edit the TPS configuration, do not edit config/config.json.in in the tree.
+Instead, edit config.json inside your virtualenv; it will be located at the
+top level of where you specified the virtualenv be created - eg, for the
+example above, it will be `/path/to/create/virtualenv/config.json`
+
+Setting Up Test Accounts
+========================
+
+Firefox Accounts
+----------------
+To create a test account for using the Firefox Account authentication perform the
+following steps:
+
+> Note: Currently, the TPS tests rely on how restmail returns the verification code
+> You should use restmail or something very similar.
+> Gmail and other providers might give a `The request was blocked for security reasons`
+
+1. Go to the URL: http://restmail.net/mail/%account_prefix%@restmail.net
+  - Replace `%account_prefix%` with your own test name
+2. Go to https://accounts.firefox.com/signup?service=sync&context=fx_desktop_v1
+3. Sign in with the previous chosen email address and a password
+4. Go back to the Restmail URL, reload the page
+5. Search for the verification link and open that page
+
+Now you will be able to use this account for TPS. Note that these
+steps can be done in either a test profile or in a private browsing window - you
+might want to avoid doing that in a "real" profile that's already connected to
+Sync.
diff --git a/testing/tps/config/config.json.in b/testing/tps/config/config.json.in
new file mode 100644
index 0000000000..38cf374628
--- /dev/null
+++ b/testing/tps/config/config.json.in
@@ -0,0 +1,13 @@
+{
+  "fx_account": {
+    "username": "__FX_ACCOUNT_USERNAME__",
+    "password": "__FX_ACCOUNT_PASSWORD__"
+  },
+  "auth_type": "fx_account",
+  "es": "localhost:9200",
+  "os": "Ubuntu",
+  "platform": "linux64",
+  "serverURL": null,
+  "extensiondir": "__EXTENSIONDIR__",
+  "testdir": "__TESTDIR__"
+}
diff --git a/testing/tps/create_venv.py b/testing/tps/create_venv.py
new file mode 100755
index 0000000000..3d938ffc78
--- /dev/null
+++ b/testing/tps/create_venv.py
@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+This scripts sets up a virtualenv and installs TPS into it.
+It's probably best to specify a path NOT inside the repo, otherwise
+all the virtualenv files will show up in e.g. hg status.
+"""
+
+import optparse
+import os
+import subprocess
+import sys
+import venv
+
+here = os.path.dirname(os.path.abspath(__file__))
+usage_message = """
+***********************************************************************
+
+To run TPS, activate the virtualenv using:
+    source {TARGET}/{BIN_NAME}
+
+To change your TPS config, please edit the file:
+    {TARGET}/config.json
+
+To execute tps use:
+    runtps --binary=/path/to/firefox
+
+See runtps --help for all options
+
+***********************************************************************
+"""
+
+if sys.platform == "win32":
+    bin_name = os.path.join("Scripts", "activate.bat")
+    python_env = os.path.join("Scripts", "python.exe")
+else:
+    bin_name = os.path.join("bin", "activate")
+    python_env = os.path.join("bin", "python")
+
+
+def setup_virtualenv(target):
+    print("Creating new virtual environment:", os.path.abspath(target))
+    # system_site_packages=True so we have access to setuptools.
+    venv.create(target, system_site_packages=True)
+
+
+def update_configfile(source, target, replacements):
+    lines = []
+
+    with open(source) as config:
+        for line in config:
+            for source_string, target_string in replacements.items():
+                if target_string:
+                    line = line.replace(source_string, target_string)
+            lines.append(line)
+
+    with open(target, "w") as config:
+        for line in lines:
+            config.write(line)
+
+
+def main():
+    parser = optparse.OptionParser("Usage: %prog [options] path_to_venv")
+    parser.add_option(
+        "--keep-config",
+        dest="keep_config",
+        action="store_true",
+        help="Keep the existing config file.",
+    )
+    parser.add_option(
+        "--password",
+        type="string",
+        dest="password",
+        metavar="FX_ACCOUNT_PASSWORD",
+        default=None,
+        help="The Firefox Account password.",
+    )
+    parser.add_option(
+        "-p",
+        "--python",
+        type="string",
+        dest="python",
+        metavar="PYTHON_BIN",
+        default=None,
+        help="The Python interpreter to use.",
+    )
+    parser.add_option(
+        "--sync-passphrase",
+        type="string",
+        dest="sync_passphrase",
+        metavar="SYNC_ACCOUNT_PASSPHRASE",
+        default=None,
+        help="The old Firefox Sync account passphrase.",
+    )
+    parser.add_option(
+        "--sync-password",
+        type="string",
+        dest="sync_password",
+        metavar="SYNC_ACCOUNT_PASSWORD",
+        default=None,
+        help="The old Firefox Sync account password.",
+    )
+    parser.add_option(
+        "--sync-username",
+        type="string",
+        dest="sync_username",
+        metavar="SYNC_ACCOUNT_USERNAME",
+        default=None,
+        help="The old Firefox Sync account username.",
+    )
+    parser.add_option(
+        "--username",
+        type="string",
+        dest="username",
+        metavar="FX_ACCOUNT_USERNAME",
+        default=None,
+        help="The Firefox Account username.",
+    )
+
+    (options, args) = parser.parse_args(args=None, values=None)
+
+    if len(args) != 1:
+        parser.error("Path to the environment has to be specified")
+    target = args[0]
+    assert target
+
+    setup_virtualenv(target)
+
+    # Activate tps environment
+    activate(target)
+
+    # Install TPS in environment
+    subprocess.check_call(
+        [os.path.join(target, python_env), os.path.join(here, "setup.py"), "install"]
+    )
+
+    # Get the path to tests and extensions directory by checking check where
+    # the tests and extensions directories are located
+    sync_dir = os.path.abspath(os.path.join(here, "..", "..", "services", "sync"))
+    if os.path.exists(sync_dir):
+        testdir = os.path.join(sync_dir, "tests", "tps")
+        extdir = os.path.join(sync_dir, "tps", "extensions")
+    else:
+        testdir = os.path.join(here, "tests")
+        extdir = os.path.join(here, "extensions")
+
+    if not options.keep_config:
+        update_configfile(
+            os.path.join(here, "config", "config.json.in"),
+            os.path.join(target, "config.json"),
+            replacements={
+                "__TESTDIR__": testdir.replace("\\", "/"),
+                "__EXTENSIONDIR__": extdir.replace("\\", "/"),
+                "__FX_ACCOUNT_USERNAME__": options.username,
+                "__FX_ACCOUNT_PASSWORD__": options.password,
+                "__SYNC_ACCOUNT_USERNAME__": options.sync_username,
+                "__SYNC_ACCOUNT_PASSWORD__": options.sync_password,
+                "__SYNC_ACCOUNT_PASSPHRASE__": options.sync_passphrase,
+            },
+        )
+
+        if not (options.username and options.password):
+            print("\nFirefox Account credentials not specified.")
+        if not (options.sync_username and options.sync_password and options.passphrase):
+            print("\nFirefox Sync account credentials not specified.")
+
+    # Print the user instructions
+    print(usage_message.format(TARGET=target, BIN_NAME=bin_name))
+
+
+def activate(target):
+    # This is a lightly modified copy of `activate_this.py`, which existed when
+    # venv was an external package, but doesn't come with the builtin venv support.
+    old_os_path = os.environ.get("PATH", "")
+    os.environ["PATH"] = (
+        os.path.dirname(os.path.abspath(__file__)) + os.pathsep + old_os_path
+    )
+    base = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    if sys.platform == "win32":
+        site_packages = os.path.join(base, "Lib", "site-packages")
+    else:
+        site_packages = os.path.join(
+            base, "lib", "python%s" % sys.version[:3], "site-packages"
+        )
+    prev_sys_path = list(sys.path)
+    import site
+
+    site.addsitedir(site_packages)
+    sys.real_prefix = sys.prefix
+    sys.prefix = base
+    # Move the added items to the front of the path:
+    new_sys_path = []
+    for item in list(sys.path):
+        if item not in prev_sys_path:
+            new_sys_path.append(item)
+            sys.path.remove(item)
+    sys.path[:0] = new_sys_path
+
+
+if __name__ == "__main__":
+    main()
diff --git a/testing/tps/mach_commands.py b/testing/tps/mach_commands.py
new file mode 100644
index 0000000000..34b20ff799
--- /dev/null
+++ b/testing/tps/mach_commands.py
@@ -0,0 +1,35 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, # You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import os
+
+from mach.decorators import Command, CommandArgument
+from mozpack.copier import Jarrer
+from mozpack.files import FileFinder
+
+
+@Command("tps-build", category="testing", description="Build TPS add-on.")
+@CommandArgument("--dest", default=None, help="Where to write add-on.")
+def build(command_context, dest):
+    src = os.path.join(
+        command_context.topsrcdir, "services", "sync", "tps", "extensions", "tps"
+    )
+    dest = os.path.join(
+        dest or os.path.join(command_context.topobjdir, "services", "sync"),
+        "tps.xpi",
+    )
+
+    if not os.path.exists(os.path.dirname(dest)):
+        os.makedirs(os.path.dirname(dest))
+
+    if os.path.isfile(dest):
+        os.unlink(dest)
+
+    jarrer = Jarrer()
+    for p, f in FileFinder(src).find("*"):
+        jarrer.add(p, f)
+    jarrer.copy(dest)
+
+    print("Built TPS add-on as %s" % dest)
diff --git a/testing/tps/moz.build b/testing/tps/moz.build
new file mode 100644
index 0000000000..06a74e1bfc
--- /dev/null
+++ b/testing/tps/moz.build
@@ -0,0 +1,8 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+with Files("**"):
+    BUG_COMPONENT = ("Testing", "General")
diff --git a/testing/tps/pages/microsummary1.txt b/testing/tps/pages/microsummary1.txt
new file mode 100644
index 0000000000..9451d4b5c0
--- /dev/null
+++ b/testing/tps/pages/microsummary1.txt
@@ -0,0 +1 @@
+Static microsummary #1

diff --git a/testing/tps/pages/microsummary2.txt b/testing/tps/pages/microsummary2.txt
new file mode 100644
index 0000000000..7e32085044
--- /dev/null
+++ b/testing/tps/pages/microsummary2.txt
@@ -0,0 +1 @@
+Static microsummary #2

diff --git a/testing/tps/pages/microsummary3.txt b/testing/tps/pages/microsummary3.txt
new file mode 100644
index 0000000000..00ac62bbf2
--- /dev/null
+++ b/testing/tps/pages/microsummary3.txt
@@ -0,0 +1 @@
+Static microsummary #3

diff --git a/testing/tps/pages/page1.html b/testing/tps/pages/page1.html
new file mode 100644
index 0000000000..5178bae13a
--- /dev/null
+++ b/testing/tps/pages/page1.html
@@ -0,0 +1,14 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<html>
+<head>
+<title>Crossweave Test Page 1</title>
+</head>
+<body>
+<p>
+Crossweave Test Page 1
+</p>
+</body>
+</html>
diff --git a/testing/tps/pages/page2.html b/testing/tps/pages/page2.html
new file mode 100644
index 0000000000..da117bcc8e
--- /dev/null
+++ b/testing/tps/pages/page2.html
@@ -0,0 +1,14 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<html>
+<head>
+<title>Crossweave Test Page 2</title>
+</head>
+<body>
+<p>
+Crossweave Test Page 2
+</p>
+</body>
+</html>
diff --git a/testing/tps/pages/page3.html b/testing/tps/pages/page3.html
new file mode 100644
index 0000000000..001762be83
--- /dev/null
+++ b/testing/tps/pages/page3.html
@@ -0,0 +1,14 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<html>
+<head>
+<title>Crossweave Test Page 3</title>
+</head>
+<body>
+<p>
+Crossweave Test Page 3
+</p>
+</body>
+</html>
diff --git a/testing/tps/pages/page4.html b/testing/tps/pages/page4.html
new file mode 100644
index 0000000000..5234089679
--- /dev/null
+++ b/testing/tps/pages/page4.html
@@ -0,0 +1,14 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<html>
+<head>
+<title>Crossweave Test Page 4</title>
+</head>
+<body>
+<p>
+Crossweave Test Page 4
+</p>
+</body>
+</html>
diff --git a/testing/tps/pages/page5.html b/testing/tps/pages/page5.html
new file mode 100644
index 0000000000..34adbb6827
--- /dev/null
+++ b/testing/tps/pages/page5.html
@@ -0,0 +1,14 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<html>
+<head>
+<title>Crossweave Test Page 5</title>
+</head>
+<body>
+<p>
+Crossweave Test Page 5
+</p>
+</body>
+</html>
diff --git a/testing/tps/setup.py b/testing/tps/setup.py
new file mode 100644
index 0000000000..0bcf4387f0
--- /dev/null
+++ b/testing/tps/setup.py
@@ -0,0 +1,50 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+from setuptools import find_packages, setup
+
+version = "0.6"
+
+deps = [
+    "httplib2 == 0.9.2",
+    "mozfile >= 1.2",
+    "wptserve >= 3.0",
+    "mozinfo >= 1.2",
+    "mozinstall == 2.0.1",
+    "mozprocess == 1.3",
+    "mozprofile ~= 2.1",
+    "mozrunner ~= 8.2",
+    "mozversion == 2.3",
+    "PyYAML >= 4.0",
+]
+
+setup(
+    name="tps",
+    version=version,
+    description="run automated multi-profile sync tests",
+    long_description="""\
+""",
+    classifiers=[
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 2 :: Only",
+    ],  # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
+    keywords="",
+    author="Mozilla Automation and Tools team",
+    author_email="tools@lists.mozilla.org",
+    url="https://developer.mozilla.org/en-US/docs/TPS",
+    license="MPL 2.0",
+    packages=find_packages(exclude=["ez_setup", "examples", "tests"]),
+    include_package_data=True,
+    zip_safe=False,
+    install_requires=deps,
+    entry_points="""
+      # -*- Entry points: -*-
+      [console_scripts]
+      runtps = tps.cli:main
+      """,
+    data_files=[
+        ("tps", ["config/config.json.in"]),
+    ],
+)
diff --git a/testing/tps/tps/__init__.py b/testing/tps/tps/__init__.py
new file mode 100644
index 0000000000..a4c384c1ce
--- /dev/null
+++ b/testing/tps/tps/__init__.py
@@ -0,0 +1,9 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# flake8: noqa
+
+
+from .firefoxrunner import TPSFirefoxRunner
+from .testrunner import TPSTestRunner
diff --git a/testing/tps/tps/cli.py b/testing/tps/tps/cli.py
new file mode 100644
index 0000000000..9ef6186b4d
--- /dev/null
+++ b/testing/tps/tps/cli.py
@@ -0,0 +1,157 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import optparse
+import os
+import re
+import sys
+from threading import RLock
+
+from tps import TPSTestRunner
+
+
+def main():
+    parser = optparse.OptionParser()
+    parser.add_option(
+        "--binary",
+        action="store",
+        type="string",
+        dest="binary",
+        default=None,
+        help="path to the Firefox binary, specified either as "
+        "a local file or a url; if omitted, the PATH "
+        "will be searched;",
+    )
+    parser.add_option(
+        "--configfile",
+        action="store",
+        type="string",
+        dest="configfile",
+        default=None,
+        help="path to the config file to use default: %default]",
+    )
+    parser.add_option(
+        "--debug",
+        action="store_true",
+        dest="debug",
+        default=False,
+        help="run in debug mode",
+    )
+    parser.add_option(
+        "--ignore-unused-engines",
+        default=False,
+        action="store_true",
+        dest="ignore_unused_engines",
+        help="If defined, do not load unused engines in individual tests."
+        " Has no effect for pulse monitor.",
+    )
+    parser.add_option(
+        "--logfile",
+        action="store",
+        type="string",
+        dest="logfile",
+        default="tps.log",
+        help="path to the log file [default: %default]",
+    )
+    parser.add_option(
+        "--mobile",
+        action="store_true",
+        dest="mobile",
+        default=False,
+        help="run with mobile settings",
+    )
+    parser.add_option(
+        "--pulsefile",
+        action="store",
+        type="string",
+        dest="pulsefile",
+        default=None,
+        help="path to file containing a pulse message in "
+        "json format that you want to inject into the monitor",
+    )
+    parser.add_option(
+        "--resultfile",
+        action="store",
+        type="string",
+        dest="resultfile",
+        default="tps_result.json",
+        help="path to the result file [default: %default]",
+    )
+    parser.add_option(
+        "--testfile",
+        action="store",
+        type="string",
+        dest="testfile",
+        default="all_tests.json",
+        help="path to the test file to run [default: %default]",
+    )
+    parser.add_option(
+        "--stop-on-error",
+        action="store_true",
+        dest="stop_on_error",
+        help="stop running tests after the first failure",
+    )
+    (options, args) = parser.parse_args()
+
+    configfile = options.configfile
+    if configfile is None:
+        virtual_env = os.environ.get("VIRTUAL_ENV")
+        if virtual_env:
+            configfile = os.path.join(virtual_env, "config.json")
+        if configfile is None or not os.access(configfile, os.F_OK):
+            raise Exception(
+                "Unable to find config.json in a VIRTUAL_ENV; you must "
+                "specify a config file using the --configfile option"
+            )
+
+    # load the config file
+    f = open(configfile, "r")
+    configcontent = f.read()
+    f.close()
+    config = json.loads(configcontent)
+    testfile = os.path.join(config.get("testdir", ""), options.testfile)
+
+    rlock = RLock()
+
+    print("using result file", options.resultfile)
+
+    extensionDir = config.get("extensiondir")
+    if not extensionDir or extensionDir == "__EXTENSIONDIR__":
+        extensionDir = os.path.join(
+            os.getcwd(), "..", "..", "services", "sync", "tps", "extensions"
+        )
+    else:
+        if sys.platform == "win32":
+            # replace msys-style paths with proper Windows paths
+            m = re.match("^\/\w\/", extensionDir)
+            if m:
+                extensionDir = "%s:/%s" % (m.group(0)[1:2], extensionDir[3:])
+                extensionDir = extensionDir.replace("/", "\\")
+    if sys.platform == "darwin":
+        # Needed to avoid tab crashes on mac due to level 3 sandboxing
+        sourceRoot = os.path.join(extensionDir, "..", "..", "..", "..")
+        os.environ["MOZ_DEVELOPER_REPO_DIR"] = os.path.abspath(sourceRoot)
+
+    TPS = TPSTestRunner(
+        extensionDir,
+        binary=options.binary,
+        config=config,
+        debug=options.debug,
+        ignore_unused_engines=options.ignore_unused_engines,
+        logfile=options.logfile,
+        mobile=options.mobile,
+        resultfile=options.resultfile,
+        rlock=rlock,
+        testfile=testfile,
+        stop_on_error=options.stop_on_error,
+    )
+    TPS.run_tests()
+
+    if TPS.numfailed > 0 or TPS.numpassed == 0:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/testing/tps/tps/firefoxrunner.py b/testing/tps/tps/firefoxrunner.py
new file mode 100644
index 0000000000..e82392c6c0
--- /dev/null
+++ b/testing/tps/tps/firefoxrunner.py
@@ -0,0 +1,84 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+
+import httplib2
+import mozfile
+import mozinstall
+from mozprofile import Profile
+from mozrunner import FirefoxRunner
+
+
+class TPSFirefoxRunner(object):
+
+    PROCESS_TIMEOUT = 240
+
+    def __init__(self, binary):
+        if binary is not None and ("http://" in binary or "ftp://" in binary):
+            self.url = binary
+            self.binary = None
+        else:
+            self.url = None
+            self.binary = binary
+
+        self.installdir = None
+
+    def __del__(self):
+        if self.installdir:
+            mozfile.remove(self.installdir, True)
+
+    def download_url(self, url, dest=None):
+        h = httplib2.Http()
+        resp, content = h.request(url, "GET")
+        if dest is None:
+            dest = os.path.basename(url)
+
+        local = open(dest, "wb")
+        local.write(content)
+        local.close()
+        return dest
+
+    def download_build(self, installdir="downloadedbuild", appname="firefox"):
+        self.installdir = os.path.abspath(installdir)
+        buildName = os.path.basename(self.url)
+        pathToBuild = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)), buildName
+        )
+
+        # delete the build if it already exists
+        if os.access(pathToBuild, os.F_OK):
+            os.remove(pathToBuild)
+
+        # download the build
+        print("downloading build")
+        self.download_url(self.url, pathToBuild)
+
+        # install the build
+        print("installing {}".format(pathToBuild))
+        mozfile.remove(self.installdir, True)
+        binary = mozinstall.install(src=pathToBuild, dest=self.installdir)
+
+        # remove the downloaded archive
+        os.remove(pathToBuild)
+
+        return binary
+
+    def run(self, profile=None, timeout=PROCESS_TIMEOUT, env=None, args=None):
+        """Runs the given FirefoxRunner with the given Profile, waits
+        for completion, then returns the process exit code
+        """
+        if profile is None:
+            profile = Profile()
+        self.profile = profile
+
+        if self.binary is None and self.url:
+            self.binary = self.download_build()
+
+        runner = FirefoxRunner(
+            profile=self.profile, binary=self.binary, env=env, cmdargs=args
+        )
+
+        runner.start(timeout=timeout)
+        return runner.wait()
diff --git a/testing/tps/tps/phase.py b/testing/tps/tps/phase.py
new file mode 100644
index 0000000000..f51cf9e4c6
--- /dev/null
+++ b/testing/tps/tps/phase.py
@@ -0,0 +1,88 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os.path
+import re
+
+
+class TPSTestPhase(object):
+
+    lineRe = re.compile(
+        r"^(.*?)test phase (?P<matchphase>[^\s]+): (?P<matchstatus>.*)$"
+    )
+
+    def __init__(
+        self,
+        phase,
+        profile,
+        testname,
+        testpath,
+        logfile,
+        env,
+        firefoxRunner,
+        logfn,
+        ignore_unused_engines=False,
+    ):
+        self.phase = phase
+        self.profile = profile
+        self.testname = str(testname)  # this might be passed in as unicode
+        self.testpath = testpath
+        self.logfile = logfile
+        self.env = env
+        self.firefoxRunner = firefoxRunner
+        self.log = logfn
+        self.ignore_unused_engines = ignore_unused_engines
+        self._status = None
+        self.errline = ""
+
+    @property
+    def status(self):
+        return self._status if self._status else "unknown"
+
+    def run(self):
+        # launch Firefox
+
+        prefs = {
+            "testing.tps.testFile": os.path.abspath(self.testpath),
+            "testing.tps.testPhase": self.phase,
+            "testing.tps.logFile": self.logfile,
+            "testing.tps.ignoreUnusedEngines": self.ignore_unused_engines,
+        }
+
+        self.profile.set_preferences(prefs)
+
+        self.log(
+            "\nLaunching Firefox for phase %s with prefs %s\n"
+            % (self.phase, str(prefs))
+        )
+
+        self.firefoxRunner.run(env=self.env, args=[], profile=self.profile)
+
+        # parse the logfile and look for results from the current test phase
+        found_test = False
+        f = open(self.logfile, "r")
+        for line in f:
+
+            # skip to the part of the log file that deals with the test we're running
+            if not found_test:
+                if line.find("Running test %s" % self.testname) > -1:
+                    found_test = True
+                else:
+                    continue
+
+            # look for the status of the current phase
+            match = self.lineRe.match(line)
+            if match:
+                if match.group("matchphase") == self.phase:
+                    self._status = match.group("matchstatus")
+                    break
+
+            # set the status to FAIL if there is TPS error
+            if line.find("CROSSWEAVE ERROR: ") > -1 and not self._status:
+                self._status = "FAIL"
+                self.errline = line[
+                    line.find("CROSSWEAVE ERROR: ") + len("CROSSWEAVE ERROR: ") :
+                ]
+
+        f.close()
diff --git a/testing/tps/tps/testrunner.py b/testing/tps/tps/testrunner.py
new file mode 100644
index 0000000000..66a68b3580
--- /dev/null
+++ b/testing/tps/tps/testrunner.py
@@ -0,0 +1,515 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import json
+import os
+import re
+import tempfile
+import time
+import traceback
+
+import mozinfo
+import mozversion
+import yaml
+from mozprofile import Profile
+from wptserve import server
+
+from .firefoxrunner import TPSFirefoxRunner
+from .phase import TPSTestPhase
+
+
+class TempFile(object):
+    """Class for temporary files that delete themselves when garbage-collected."""
+
+    def __init__(self, prefix=None):
+        self.fd, self.filename = self.tmpfile = tempfile.mkstemp(prefix=prefix)
+
+    def write(self, data):
+        if self.fd:
+            os.write(self.fd, data)
+
+    def close(self):
+        if self.fd:
+            os.close(self.fd)
+            self.fd = None
+
+    def cleanup(self):
+        if self.fd:
+            self.close()
+        if os.access(self.filename, os.F_OK):
+            os.remove(self.filename)
+
+    __del__ = cleanup
+
+
+class TPSTestRunner(object):
+
+    extra_env = {
+        "MOZ_CRASHREPORTER_DISABLE": "1",
+        "GNOME_DISABLE_CRASH_DIALOG": "1",
+        "XRE_NO_WINDOWS_CRASH_DIALOG": "1",
+        "MOZ_NO_REMOTE": "1",
+        "XPCOM_DEBUG_BREAK": "warn",
+    }
+
+    default_preferences = {
+        "app.update.checkInstallTime": False,
+        "app.update.disabledForTesting": True,
+        "security.turn_off_all_security_so_that_viruses_can_take_over_this_computer": True,
+        "browser.dom.window.dump.enabled": True,
+        "devtools.console.stdout.chrome": True,
+        "browser.sessionstore.resume_from_crash": False,
+        "browser.shell.checkDefaultBrowser": False,
+        "browser.tabs.warnOnClose": False,
+        "browser.warnOnQuit": False,
+        # Allow installing extensions dropped into the profile folder
+        "extensions.autoDisableScopes": 10,
+        "extensions.getAddons.get.url": "http://127.0.0.1:4567/addons/api/%IDS%.json",
+        # Our pretend addons server doesn't support metadata...
+        "extensions.getAddons.cache.enabled": False,
+        "extensions.install.requireSecureOrigin": False,
+        "extensions.update.enabled": False,
+        # Don't open a dialog to show available add-on updates
+        "extensions.update.notifyUser": False,
+        "services.sync.firstSync": "notReady",
+        "services.sync.lastversion": "1.0",
+        "toolkit.startup.max_resumed_crashes": -1,
+        # hrm - not sure what the release/beta channels will do?
+        "xpinstall.signatures.required": False,
+        "services.sync.testing.tps": True,
+        "engine.bookmarks.repair.enabled": False,
+        "extensions.experiments.enabled": True,
+        "webextensions.storage.sync.kinto": False,
+    }
+
+    debug_preferences = {
+        "services.sync.log.appender.console": "Trace",
+        "services.sync.log.appender.dump": "Trace",
+        "services.sync.log.appender.file.level": "Trace",
+        "services.sync.log.appender.file.logOnSuccess": True,
+        "services.sync.log.logger": "Trace",
+        "services.sync.log.logger.engine": "Trace",
+    }
+
+    syncVerRe = re.compile(r"Sync version: (?P<syncversion>.*)\n")
+    ffVerRe = re.compile(r"Firefox version: (?P<ffver>.*)\n")
+    ffBuildIDRe = re.compile(r"Firefox buildid: (?P<ffbuildid>.*)\n")
+
+    def __init__(
+        self,
+        extensionDir,
+        binary=None,
+        config=None,
+        debug=False,
+        ignore_unused_engines=False,
+        logfile="tps.log",
+        mobile=False,
+        rlock=None,
+        resultfile="tps_result.json",
+        testfile=None,
+        stop_on_error=False,
+    ):
+        self.binary = binary
+        self.config = config if config else {}
+        self.debug = debug
+        self.extensions = []
+        self.ignore_unused_engines = ignore_unused_engines
+        self.logfile = os.path.abspath(logfile)
+        self.mobile = mobile
+        self.rlock = rlock
+        self.resultfile = resultfile
+        self.testfile = testfile
+        self.stop_on_error = stop_on_error
+
+        self.addonversion = None
+        self.branch = None
+        self.changeset = None
+        self.errorlogs = {}
+        self.extensionDir = extensionDir
+        self.firefoxRunner = None
+        self.nightly = False
+        self.numfailed = 0
+        self.numpassed = 0
+        self.postdata = {}
+        self.productversion = None
+        self.repo = None
+        self.tpsxpi = None
+
+    @property
+    def mobile(self):
+        return self._mobile
+
+    @mobile.setter
+    def mobile(self, value):
+        self._mobile = value
+        self.synctype = "desktop" if not self._mobile else "mobile"
+
+    def log(self, msg, printToConsole=False):
+        """Appends a string to the logfile"""
+
+        f = open(self.logfile, "a")
+        f.write(msg)
+        f.close()
+        if printToConsole:
+            print(msg)
+
+    def writeToResultFile(self, postdata, body=None, sendTo=["crossweave@mozilla.com"]):
+        """Writes results to test file"""
+
+        results = {"results": []}
+
+        if os.access(self.resultfile, os.F_OK):
+            f = open(self.resultfile, "r")
+            results = json.loads(f.read())
+            f.close()
+
+        f = open(self.resultfile, "w")
+        if body is not None:
+            postdata["body"] = body
+        if self.numpassed is not None:
+            postdata["numpassed"] = self.numpassed
+        if self.numfailed is not None:
+            postdata["numfailed"] = self.numfailed
+        if self.firefoxRunner and self.firefoxRunner.url:
+            postdata["firefoxrunnerurl"] = self.firefoxRunner.url
+
+        postdata["sendTo"] = sendTo
+        results["results"].append(postdata)
+        f.write(json.dumps(results, indent=2))
+        f.close()
+
+    def _zip_add_file(self, zip, file, rootDir):
+        zip.write(os.path.join(rootDir, file), file)
+
+    def _zip_add_dir(self, zip, dir, rootDir):
+        try:
+            zip.write(os.path.join(rootDir, dir), dir)
+        except Exception:
+            # on some OS's, adding directory entries doesn't seem to work
+            pass
+        for root, dirs, files in os.walk(os.path.join(rootDir, dir)):
+            for f in files:
+                zip.write(os.path.join(root, f), os.path.join(dir, f))
+
+    def handle_phase_failure(self, profiles):
+        for profile in profiles:
+            self.log("\nDumping sync log for profile %s\n" % profiles[profile].profile)
+            for root, dirs, files in os.walk(
+                os.path.join(profiles[profile].profile, "weave", "logs")
+            ):
+                for f in files:
+                    weavelog = os.path.join(
+                        profiles[profile].profile, "weave", "logs", f
+                    )
+                    if os.access(weavelog, os.F_OK):
+                        with open(weavelog, "r") as fh:
+                            for line in fh:
+                                possible_time = line[0:13]
+                                if len(possible_time) == 13 and possible_time.isdigit():
+                                    time_ms = int(possible_time)
+                                    # pylint --py3k W1619
+                                    formatted = time.strftime(
+                                        "%Y-%m-%d %H:%M:%S",
+                                        time.localtime(time_ms / 1000),
+                                    )
+                                    self.log(
+                                        "%s.%03d %s"
+                                        % (formatted, time_ms % 1000, line[14:])
+                                    )
+                                else:
+                                    self.log(line)
+
+    def run_single_test(self, testdir, testname):
+        testpath = os.path.join(testdir, testname)
+        self.log("Running test %s\n" % testname, True)
+
+        # Read and parse the test file, merge it with the contents of the config
+        # file, and write the combined output to a temporary file.
+        f = open(testpath, "r")
+        testcontent = f.read()
+        f.close()
+        # We use yaml to parse the tests because it is a superset of json
+        # but tolerates things like property names not being quoted, trailing
+        # commas, etc.
+        try:
+            test = yaml.safe_load(testcontent)
+        except Exception:
+            test = yaml.safe_load(
+                testcontent[testcontent.find("{") : testcontent.find("}") + 1]
+            )
+
+        self.preferences["tps.seconds_since_epoch"] = int(time.time())
+
+        # generate the profiles defined in the test, and a list of test phases
+        profiles = {}
+        phaselist = []
+        for phase in test:
+            profilename = test[phase]
+
+            # create the profile if necessary
+            if profilename not in profiles:
+                profiles[profilename] = Profile(
+                    preferences=self.preferences.copy(), addons=self.extensions
+                )
+
+            # create the test phase
+            phaselist.append(
+                TPSTestPhase(
+                    phase,
+                    profiles[profilename],
+                    testname,
+                    testpath,
+                    self.logfile,
+                    self.env,
+                    self.firefoxRunner,
+                    self.log,
+                    ignore_unused_engines=self.ignore_unused_engines,
+                )
+            )
+
+        # sort the phase list by name
+        phaselist = sorted(phaselist, key=lambda phase: phase.phase)
+
+        # run each phase in sequence, aborting at the first failure
+        failed = False
+        for phase in phaselist:
+            phase.run()
+            if phase.status != "PASS":
+                failed = True
+                break
+
+        for profilename in profiles:
+            print("### Cleanup Profile ", profilename)
+            cleanup_phase = TPSTestPhase(
+                "cleanup-" + profilename,
+                profiles[profilename],
+                testname,
+                testpath,
+                self.logfile,
+                self.env,
+                self.firefoxRunner,
+                self.log,
+            )
+
+            cleanup_phase.run()
+            if cleanup_phase.status != "PASS":
+                failed = True
+                # Keep going to run the remaining cleanup phases.
+
+        if failed:
+            self.handle_phase_failure(profiles)
+
+        # grep the log for FF and sync versions
+        f = open(self.logfile)
+        logdata = f.read()
+        match = self.syncVerRe.search(logdata)
+        sync_version = match.group("syncversion") if match else "unknown"
+        match = self.ffVerRe.search(logdata)
+        firefox_version = match.group("ffver") if match else "unknown"
+        match = self.ffBuildIDRe.search(logdata)
+        firefox_buildid = match.group("ffbuildid") if match else "unknown"
+        f.close()
+        if phase.status == "PASS":
+            logdata = ""
+        else:
+            # we only care about the log data for this specific test
+            logdata = logdata[logdata.find("Running test %s" % (str(testname))) :]
+
+        result = {
+            "PASS": lambda x: ("TEST-PASS", ""),
+            "FAIL": lambda x: ("TEST-UNEXPECTED-FAIL", x.rstrip()),
+            "unknown": lambda x: ("TEST-UNEXPECTED-FAIL", "test did not complete"),
+        }[phase.status](phase.errline)
+        logstr = "\n%s | %s%s\n" % (
+            result[0],
+            testname,
+            (" | %s" % result[1] if result[1] else ""),
+        )
+
+        try:
+            repoinfo = mozversion.get_version(self.binary)
+        except Exception:
+            repoinfo = {}
+        apprepo = repoinfo.get("application_repository", "")
+        appchangeset = repoinfo.get("application_changeset", "")
+
+        # save logdata to a temporary file for posting to the db
+        tmplogfile = None
+        if logdata:
+            tmplogfile = TempFile(prefix="tps_log_")
+            tmplogfile.write(logdata.encode("utf-8"))
+            tmplogfile.close()
+            self.errorlogs[testname] = tmplogfile
+
+        resultdata = {
+            "productversion": {
+                "version": firefox_version,
+                "buildid": firefox_buildid,
+                "builddate": firefox_buildid[0:8],
+                "product": "Firefox",
+                "repository": apprepo,
+                "changeset": appchangeset,
+            },
+            "addonversion": {"version": sync_version, "product": "Firefox Sync"},
+            "name": testname,
+            "message": result[1],
+            "state": result[0],
+            "logdata": logdata,
+        }
+
+        self.log(logstr, True)
+        for phase in phaselist:
+            print("\t{}: {}".format(phase.phase, phase.status))
+
+        return resultdata
+
+    def update_preferences(self):
+        self.preferences = self.default_preferences.copy()
+
+        if self.mobile:
+            self.preferences.update({"services.sync.client.type": "mobile"})
+
+        # If we are using legacy Sync, then set a dummy username to force the
+        # correct authentication type. Without this pref set to a value
+        # without an '@' character, Sync will initialize for FxA.
+        if self.config.get("auth_type", "fx_account") != "fx_account":
+            self.preferences.update({"services.sync.username": "dummy"})
+
+        if self.debug:
+            self.preferences.update(self.debug_preferences)
+
+        if "preferences" in self.config:
+            self.preferences.update(self.config["preferences"])
+
+        self.preferences["tps.config"] = json.dumps(self.config)
+
+    def run_tests(self):
+        # delete the logfile if it already exists
+        if os.access(self.logfile, os.F_OK):
+            os.remove(self.logfile)
+
+        # Copy the system env variables, and update them for custom settings
+        self.env = os.environ.copy()
+        self.env.update(self.extra_env)
+
+        # Update preferences for custom settings
+        self.update_preferences()
+
+        # Acquire a lock to make sure no other threads are running tests
+        # at the same time.
+        if self.rlock:
+            self.rlock.acquire()
+
+        try:
+            # Create the Firefox runner, which will download and install the
+            # build, as needed.
+            if not self.firefoxRunner:
+                self.firefoxRunner = TPSFirefoxRunner(self.binary)
+
+            # now, run the test group
+            self.run_test_group()
+
+        except Exception:
+            traceback.print_exc()
+            self.numpassed = 0
+            self.numfailed = 1
+            try:
+                self.writeToResultFile(
+                    self.postdata, "<pre>%s</pre>" % traceback.format_exc()
+                )
+            except Exception:
+                traceback.print_exc()
+        else:
+            try:
+
+                self.writeToResultFile(self.postdata)
+            except Exception:
+                traceback.print_exc()
+                try:
+                    self.writeToResultFile(
+                        self.postdata, "<pre>%s</pre>" % traceback.format_exc()
+                    )
+                except Exception:
+                    traceback.print_exc()
+
+        # release our lock
+        if self.rlock:
+            self.rlock.release()
+
+        # dump out a summary of test results
+        print("Test Summary\n")
+        for test in self.postdata.get("tests", {}):
+            print("{} | {} | {}".format(test["state"], test["name"], test["message"]))
+
+    def run_test_group(self):
+        self.results = []
+
+        # reset number of passed/failed tests
+        self.numpassed = 0
+        self.numfailed = 0
+
+        # build our tps.xpi extension
+        self.extensions = []
+        self.extensions.append(os.path.join(self.extensionDir, "tps"))
+
+        # build the test list
+        try:
+            f = open(self.testfile)
+            jsondata = f.read()
+            f.close()
+            testfiles = json.loads(jsondata)
+            testlist = []
+            for (filename, meta) in testfiles["tests"].items():
+                skip_reason = meta.get("disabled")
+                if skip_reason:
+                    print("Skipping test {} - {}".format(filename, skip_reason))
+                else:
+                    testlist.append(filename)
+        except ValueError:
+            testlist = [os.path.basename(self.testfile)]
+        testdir = os.path.dirname(self.testfile)
+
+        self.server = server.WebTestHttpd(port=4567, doc_root=testdir)
+        self.server.start()
+
+        # run each test, and save the results
+        for test in testlist:
+            result = self.run_single_test(testdir, test)
+
+            if not self.productversion:
+                self.productversion = result["productversion"]
+            if not self.addonversion:
+                self.addonversion = result["addonversion"]
+
+            self.results.append(
+                {
+                    "state": result["state"],
+                    "name": result["name"],
+                    "message": result["message"],
+                    "logdata": result["logdata"],
+                }
+            )
+            if result["state"] == "TEST-PASS":
+                self.numpassed += 1
+            else:
+                self.numfailed += 1
+                if self.stop_on_error:
+                    print(
+                        "\nTest failed with --stop-on-error specified; "
+                        "not running any more tests.\n"
+                    )
+                    break
+
+        self.server.stop()
+
+        # generate the postdata we'll use to post the results to the db
+        self.postdata = {
+            "tests": self.results,
+            "os": "%s %sbit" % (mozinfo.version, mozinfo.bits),
+            "testtype": "crossweave",
+            "productversion": self.productversion,
+            "addonversion": self.addonversion,
+            "synctype": self.synctype,
+        }