summaryrefslogtreecommitdiffstats
path: root/testing/tps
diff options
context:
space:
mode:
Diffstat (limited to 'testing/tps')
-rw-r--r--testing/tps/.gitignore4
-rw-r--r--testing/tps/README58
-rw-r--r--testing/tps/config/config.json.in13
-rwxr-xr-xtesting/tps/create_venv.py204
-rw-r--r--testing/tps/mach_commands.py35
-rw-r--r--testing/tps/moz.build8
-rw-r--r--testing/tps/pages/microsummary1.txt1
-rw-r--r--testing/tps/pages/microsummary2.txt1
-rw-r--r--testing/tps/pages/microsummary3.txt1
-rw-r--r--testing/tps/pages/page1.html14
-rw-r--r--testing/tps/pages/page2.html14
-rw-r--r--testing/tps/pages/page3.html14
-rw-r--r--testing/tps/pages/page4.html14
-rw-r--r--testing/tps/pages/page5.html14
-rw-r--r--testing/tps/setup.py50
-rw-r--r--testing/tps/tps/__init__.py9
-rw-r--r--testing/tps/tps/cli.py157
-rw-r--r--testing/tps/tps/firefoxrunner.py84
-rw-r--r--testing/tps/tps/phase.py88
-rw-r--r--testing/tps/tps/testrunner.py517
20 files changed, 1300 insertions, 0 deletions
diff --git a/testing/tps/.gitignore b/testing/tps/.gitignore
new file mode 100644
index 0000000000..65f8b6e053
--- /dev/null
+++ b/testing/tps/.gitignore
@@ -0,0 +1,4 @@
+# These files are added by running the TPS test suite.
+build/
+dist/
+tps.egg-info/
diff --git a/testing/tps/README b/testing/tps/README
new file mode 100644
index 0000000000..0612bea4c6
--- /dev/null
+++ b/testing/tps/README
@@ -0,0 +1,58 @@
+TPS is a test automation framework for Firefox Sync. See
+https://developer.mozilla.org/en/TPS for documentation.
+
+Installation
+============
+
+TPS requires several packages to operate properly. To install TPS and
+required packages, use the INSTALL.sh script, provided:
+
+ python3 create_venv.py /path/to/create/virtualenv
+
+This script will create a virtalenv and install TPS into it.
+
+You must then activate the virtualenv by executing:
+
+- (mac/linux): source /path/to/virtualenv/Scripts/activate
+- (win): /path/to/virtualenv/Scripts/activate.bat
+
+TPS can then be run by executing:
+
+ runtps --binary=/path/to/firefox
+
+> Note: You can run the tps tests in headless mode by using `MOZ_HEADLESS=1`. This will make
+> your computer somewhat useable while the tests are running.
+
+When you are done with TPS, you can deactivate the virtualenv by executing
+`deactivate`
+
+Configuration
+=============
+To edit the TPS configuration, do not edit config/config.json.in in the tree.
+Instead, edit config.json inside your virtualenv; it will be located at the
+top level of where you specified the virtualenv be created - eg, for the
+example above, it will be `/path/to/create/virtualenv/config.json`
+
+Setting Up Test Accounts
+========================
+
+Firefox Accounts
+----------------
+To create a test account for using the Firefox Account authentication perform the
+following steps:
+
+> Note: Currently, the TPS tests rely on how restmail returns the verification code
+> You should use restmail or something very similar.
+> Gmail and other providers might give a `The request was blocked for security reasons`
+
+1. Go to the URL: http://restmail.net/mail/%account_prefix%@restmail.net
+ - Replace `%account_prefix%` with your own test name
+2. Go to https://accounts.firefox.com/signup?service=sync&context=fx_desktop_v1
+3. Sign in with the previous chosen email address and a password
+4. Go back to the Restmail URL, reload the page
+5. Search for the verification link and open that page
+
+Now you will be able to use this account for TPS. Note that these
+steps can be done in either a test profile or in a private browsing window - you
+might want to avoid doing that in a "real" profile that's already connected to
+Sync.
diff --git a/testing/tps/config/config.json.in b/testing/tps/config/config.json.in
new file mode 100644
index 0000000000..38cf374628
--- /dev/null
+++ b/testing/tps/config/config.json.in
@@ -0,0 +1,13 @@
+{
+ "fx_account": {
+ "username": "__FX_ACCOUNT_USERNAME__",
+ "password": "__FX_ACCOUNT_PASSWORD__"
+ },
+ "auth_type": "fx_account",
+ "es": "localhost:9200",
+ "os": "Ubuntu",
+ "platform": "linux64",
+ "serverURL": null,
+ "extensiondir": "__EXTENSIONDIR__",
+ "testdir": "__TESTDIR__"
+}
diff --git a/testing/tps/create_venv.py b/testing/tps/create_venv.py
new file mode 100755
index 0000000000..3d938ffc78
--- /dev/null
+++ b/testing/tps/create_venv.py
@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+This scripts sets up a virtualenv and installs TPS into it.
+It's probably best to specify a path NOT inside the repo, otherwise
+all the virtualenv files will show up in e.g. hg status.
+"""
+
+import optparse
+import os
+import subprocess
+import sys
+import venv
+
+here = os.path.dirname(os.path.abspath(__file__))
+usage_message = """
+***********************************************************************
+
+To run TPS, activate the virtualenv using:
+ source {TARGET}/{BIN_NAME}
+
+To change your TPS config, please edit the file:
+ {TARGET}/config.json
+
+To execute tps use:
+ runtps --binary=/path/to/firefox
+
+See runtps --help for all options
+
+***********************************************************************
+"""
+
+if sys.platform == "win32":
+ bin_name = os.path.join("Scripts", "activate.bat")
+ python_env = os.path.join("Scripts", "python.exe")
+else:
+ bin_name = os.path.join("bin", "activate")
+ python_env = os.path.join("bin", "python")
+
+
+def setup_virtualenv(target):
+ print("Creating new virtual environment:", os.path.abspath(target))
+ # system_site_packages=True so we have access to setuptools.
+ venv.create(target, system_site_packages=True)
+
+
+def update_configfile(source, target, replacements):
+ lines = []
+
+ with open(source) as config:
+ for line in config:
+ for source_string, target_string in replacements.items():
+ if target_string:
+ line = line.replace(source_string, target_string)
+ lines.append(line)
+
+ with open(target, "w") as config:
+ for line in lines:
+ config.write(line)
+
+
+def main():
+ parser = optparse.OptionParser("Usage: %prog [options] path_to_venv")
+ parser.add_option(
+ "--keep-config",
+ dest="keep_config",
+ action="store_true",
+ help="Keep the existing config file.",
+ )
+ parser.add_option(
+ "--password",
+ type="string",
+ dest="password",
+ metavar="FX_ACCOUNT_PASSWORD",
+ default=None,
+ help="The Firefox Account password.",
+ )
+ parser.add_option(
+ "-p",
+ "--python",
+ type="string",
+ dest="python",
+ metavar="PYTHON_BIN",
+ default=None,
+ help="The Python interpreter to use.",
+ )
+ parser.add_option(
+ "--sync-passphrase",
+ type="string",
+ dest="sync_passphrase",
+ metavar="SYNC_ACCOUNT_PASSPHRASE",
+ default=None,
+ help="The old Firefox Sync account passphrase.",
+ )
+ parser.add_option(
+ "--sync-password",
+ type="string",
+ dest="sync_password",
+ metavar="SYNC_ACCOUNT_PASSWORD",
+ default=None,
+ help="The old Firefox Sync account password.",
+ )
+ parser.add_option(
+ "--sync-username",
+ type="string",
+ dest="sync_username",
+ metavar="SYNC_ACCOUNT_USERNAME",
+ default=None,
+ help="The old Firefox Sync account username.",
+ )
+ parser.add_option(
+ "--username",
+ type="string",
+ dest="username",
+ metavar="FX_ACCOUNT_USERNAME",
+ default=None,
+ help="The Firefox Account username.",
+ )
+
+ (options, args) = parser.parse_args(args=None, values=None)
+
+ if len(args) != 1:
+ parser.error("Path to the environment has to be specified")
+ target = args[0]
+ assert target
+
+ setup_virtualenv(target)
+
+ # Activate tps environment
+ activate(target)
+
+ # Install TPS in environment
+ subprocess.check_call(
+ [os.path.join(target, python_env), os.path.join(here, "setup.py"), "install"]
+ )
+
+ # Get the path to tests and extensions directory by checking check where
+ # the tests and extensions directories are located
+ sync_dir = os.path.abspath(os.path.join(here, "..", "..", "services", "sync"))
+ if os.path.exists(sync_dir):
+ testdir = os.path.join(sync_dir, "tests", "tps")
+ extdir = os.path.join(sync_dir, "tps", "extensions")
+ else:
+ testdir = os.path.join(here, "tests")
+ extdir = os.path.join(here, "extensions")
+
+ if not options.keep_config:
+ update_configfile(
+ os.path.join(here, "config", "config.json.in"),
+ os.path.join(target, "config.json"),
+ replacements={
+ "__TESTDIR__": testdir.replace("\\", "/"),
+ "__EXTENSIONDIR__": extdir.replace("\\", "/"),
+ "__FX_ACCOUNT_USERNAME__": options.username,
+ "__FX_ACCOUNT_PASSWORD__": options.password,
+ "__SYNC_ACCOUNT_USERNAME__": options.sync_username,
+ "__SYNC_ACCOUNT_PASSWORD__": options.sync_password,
+ "__SYNC_ACCOUNT_PASSPHRASE__": options.sync_passphrase,
+ },
+ )
+
+ if not (options.username and options.password):
+ print("\nFirefox Account credentials not specified.")
+ if not (options.sync_username and options.sync_password and options.passphrase):
+ print("\nFirefox Sync account credentials not specified.")
+
+ # Print the user instructions
+ print(usage_message.format(TARGET=target, BIN_NAME=bin_name))
+
+
+def activate(target):
+ # This is a lightly modified copy of `activate_this.py`, which existed when
+ # venv was an external package, but doesn't come with the builtin venv support.
+ old_os_path = os.environ.get("PATH", "")
+ os.environ["PATH"] = (
+ os.path.dirname(os.path.abspath(__file__)) + os.pathsep + old_os_path
+ )
+ base = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ if sys.platform == "win32":
+ site_packages = os.path.join(base, "Lib", "site-packages")
+ else:
+ site_packages = os.path.join(
+ base, "lib", "python%s" % sys.version[:3], "site-packages"
+ )
+ prev_sys_path = list(sys.path)
+ import site
+
+ site.addsitedir(site_packages)
+ sys.real_prefix = sys.prefix
+ sys.prefix = base
+ # Move the added items to the front of the path:
+ new_sys_path = []
+ for item in list(sys.path):
+ if item not in prev_sys_path:
+ new_sys_path.append(item)
+ sys.path.remove(item)
+ sys.path[:0] = new_sys_path
+
+
+if __name__ == "__main__":
+ main()
diff --git a/testing/tps/mach_commands.py b/testing/tps/mach_commands.py
new file mode 100644
index 0000000000..34b20ff799
--- /dev/null
+++ b/testing/tps/mach_commands.py
@@ -0,0 +1,35 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, # You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import os
+
+from mach.decorators import Command, CommandArgument
+from mozpack.copier import Jarrer
+from mozpack.files import FileFinder
+
+
+@Command("tps-build", category="testing", description="Build TPS add-on.")
+@CommandArgument("--dest", default=None, help="Where to write add-on.")
+def build(command_context, dest):
+ src = os.path.join(
+ command_context.topsrcdir, "services", "sync", "tps", "extensions", "tps"
+ )
+ dest = os.path.join(
+ dest or os.path.join(command_context.topobjdir, "services", "sync"),
+ "tps.xpi",
+ )
+
+ if not os.path.exists(os.path.dirname(dest)):
+ os.makedirs(os.path.dirname(dest))
+
+ if os.path.isfile(dest):
+ os.unlink(dest)
+
+ jarrer = Jarrer()
+ for p, f in FileFinder(src).find("*"):
+ jarrer.add(p, f)
+ jarrer.copy(dest)
+
+ print("Built TPS add-on as %s" % dest)
diff --git a/testing/tps/moz.build b/testing/tps/moz.build
new file mode 100644
index 0000000000..06a74e1bfc
--- /dev/null
+++ b/testing/tps/moz.build
@@ -0,0 +1,8 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+with Files("**"):
+ BUG_COMPONENT = ("Testing", "General")
diff --git a/testing/tps/pages/microsummary1.txt b/testing/tps/pages/microsummary1.txt
new file mode 100644
index 0000000000..9451d4b5c0
--- /dev/null
+++ b/testing/tps/pages/microsummary1.txt
@@ -0,0 +1 @@
+Static microsummary #1
diff --git a/testing/tps/pages/microsummary2.txt b/testing/tps/pages/microsummary2.txt
new file mode 100644
index 0000000000..7e32085044
--- /dev/null
+++ b/testing/tps/pages/microsummary2.txt
@@ -0,0 +1 @@
+Static microsummary #2
diff --git a/testing/tps/pages/microsummary3.txt b/testing/tps/pages/microsummary3.txt
new file mode 100644
index 0000000000..00ac62bbf2
--- /dev/null
+++ b/testing/tps/pages/microsummary3.txt
@@ -0,0 +1 @@
+Static microsummary #3
diff --git a/testing/tps/pages/page1.html b/testing/tps/pages/page1.html
new file mode 100644
index 0000000000..5178bae13a
--- /dev/null
+++ b/testing/tps/pages/page1.html
@@ -0,0 +1,14 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+ - License, v. 2.0. If a copy of the MPL was not distributed with this
+ - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<html>
+<head>
+<title>Crossweave Test Page 1</title>
+</head>
+<body>
+<p>
+Crossweave Test Page 1
+</p>
+</body>
+</html>
diff --git a/testing/tps/pages/page2.html b/testing/tps/pages/page2.html
new file mode 100644
index 0000000000..da117bcc8e
--- /dev/null
+++ b/testing/tps/pages/page2.html
@@ -0,0 +1,14 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+ - License, v. 2.0. If a copy of the MPL was not distributed with this
+ - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<html>
+<head>
+<title>Crossweave Test Page 2</title>
+</head>
+<body>
+<p>
+Crossweave Test Page 2
+</p>
+</body>
+</html>
diff --git a/testing/tps/pages/page3.html b/testing/tps/pages/page3.html
new file mode 100644
index 0000000000..001762be83
--- /dev/null
+++ b/testing/tps/pages/page3.html
@@ -0,0 +1,14 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+ - License, v. 2.0. If a copy of the MPL was not distributed with this
+ - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<html>
+<head>
+<title>Crossweave Test Page 3</title>
+</head>
+<body>
+<p>
+Crossweave Test Page 3
+</p>
+</body>
+</html>
diff --git a/testing/tps/pages/page4.html b/testing/tps/pages/page4.html
new file mode 100644
index 0000000000..5234089679
--- /dev/null
+++ b/testing/tps/pages/page4.html
@@ -0,0 +1,14 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+ - License, v. 2.0. If a copy of the MPL was not distributed with this
+ - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<html>
+<head>
+<title>Crossweave Test Page 4</title>
+</head>
+<body>
+<p>
+Crossweave Test Page 4
+</p>
+</body>
+</html>
diff --git a/testing/tps/pages/page5.html b/testing/tps/pages/page5.html
new file mode 100644
index 0000000000..34adbb6827
--- /dev/null
+++ b/testing/tps/pages/page5.html
@@ -0,0 +1,14 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+ - License, v. 2.0. If a copy of the MPL was not distributed with this
+ - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<html>
+<head>
+<title>Crossweave Test Page 5</title>
+</head>
+<body>
+<p>
+Crossweave Test Page 5
+</p>
+</body>
+</html>
diff --git a/testing/tps/setup.py b/testing/tps/setup.py
new file mode 100644
index 0000000000..0bcf4387f0
--- /dev/null
+++ b/testing/tps/setup.py
@@ -0,0 +1,50 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+from setuptools import find_packages, setup
+
+version = "0.6"
+
+deps = [
+ "httplib2 == 0.9.2",
+ "mozfile >= 1.2",
+ "wptserve >= 3.0",
+ "mozinfo >= 1.2",
+ "mozinstall == 2.0.1",
+ "mozprocess == 1.3",
+ "mozprofile ~= 2.1",
+ "mozrunner ~= 8.2",
+ "mozversion == 2.3",
+ "PyYAML >= 4.0",
+]
+
+setup(
+ name="tps",
+ version=version,
+ description="run automated multi-profile sync tests",
+ long_description="""\
+""",
+ classifiers=[
+ "Programming Language :: Python :: 2.7",
+ "Programming Language :: Python :: 2 :: Only",
+ ], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
+ keywords="",
+ author="Mozilla Automation and Tools team",
+ author_email="tools@lists.mozilla.org",
+ url="https://developer.mozilla.org/en-US/docs/TPS",
+ license="MPL 2.0",
+ packages=find_packages(exclude=["ez_setup", "examples", "tests"]),
+ include_package_data=True,
+ zip_safe=False,
+ install_requires=deps,
+ entry_points="""
+ # -*- Entry points: -*-
+ [console_scripts]
+ runtps = tps.cli:main
+ """,
+ data_files=[
+ ("tps", ["config/config.json.in"]),
+ ],
+)
diff --git a/testing/tps/tps/__init__.py b/testing/tps/tps/__init__.py
new file mode 100644
index 0000000000..a4c384c1ce
--- /dev/null
+++ b/testing/tps/tps/__init__.py
@@ -0,0 +1,9 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# flake8: noqa
+
+
+from .firefoxrunner import TPSFirefoxRunner
+from .testrunner import TPSTestRunner
diff --git a/testing/tps/tps/cli.py b/testing/tps/tps/cli.py
new file mode 100644
index 0000000000..9ef6186b4d
--- /dev/null
+++ b/testing/tps/tps/cli.py
@@ -0,0 +1,157 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import optparse
+import os
+import re
+import sys
+from threading import RLock
+
+from tps import TPSTestRunner
+
+
+def main():
+ parser = optparse.OptionParser()
+ parser.add_option(
+ "--binary",
+ action="store",
+ type="string",
+ dest="binary",
+ default=None,
+ help="path to the Firefox binary, specified either as "
+ "a local file or a url; if omitted, the PATH "
+ "will be searched;",
+ )
+ parser.add_option(
+ "--configfile",
+ action="store",
+ type="string",
+ dest="configfile",
+ default=None,
+ help="path to the config file to use default: %default]",
+ )
+ parser.add_option(
+ "--debug",
+ action="store_true",
+ dest="debug",
+ default=False,
+ help="run in debug mode",
+ )
+ parser.add_option(
+ "--ignore-unused-engines",
+ default=False,
+ action="store_true",
+ dest="ignore_unused_engines",
+ help="If defined, do not load unused engines in individual tests."
+ " Has no effect for pulse monitor.",
+ )
+ parser.add_option(
+ "--logfile",
+ action="store",
+ type="string",
+ dest="logfile",
+ default="tps.log",
+ help="path to the log file [default: %default]",
+ )
+ parser.add_option(
+ "--mobile",
+ action="store_true",
+ dest="mobile",
+ default=False,
+ help="run with mobile settings",
+ )
+ parser.add_option(
+ "--pulsefile",
+ action="store",
+ type="string",
+ dest="pulsefile",
+ default=None,
+ help="path to file containing a pulse message in "
+ "json format that you want to inject into the monitor",
+ )
+ parser.add_option(
+ "--resultfile",
+ action="store",
+ type="string",
+ dest="resultfile",
+ default="tps_result.json",
+ help="path to the result file [default: %default]",
+ )
+ parser.add_option(
+ "--testfile",
+ action="store",
+ type="string",
+ dest="testfile",
+ default="all_tests.json",
+ help="path to the test file to run [default: %default]",
+ )
+ parser.add_option(
+ "--stop-on-error",
+ action="store_true",
+ dest="stop_on_error",
+ help="stop running tests after the first failure",
+ )
+ (options, args) = parser.parse_args()
+
+ configfile = options.configfile
+ if configfile is None:
+ virtual_env = os.environ.get("VIRTUAL_ENV")
+ if virtual_env:
+ configfile = os.path.join(virtual_env, "config.json")
+ if configfile is None or not os.access(configfile, os.F_OK):
+ raise Exception(
+ "Unable to find config.json in a VIRTUAL_ENV; you must "
+ "specify a config file using the --configfile option"
+ )
+
+ # load the config file
+ f = open(configfile, "r")
+ configcontent = f.read()
+ f.close()
+ config = json.loads(configcontent)
+ testfile = os.path.join(config.get("testdir", ""), options.testfile)
+
+ rlock = RLock()
+
+ print("using result file", options.resultfile)
+
+ extensionDir = config.get("extensiondir")
+ if not extensionDir or extensionDir == "__EXTENSIONDIR__":
+ extensionDir = os.path.join(
+ os.getcwd(), "..", "..", "services", "sync", "tps", "extensions"
+ )
+ else:
+ if sys.platform == "win32":
+ # replace msys-style paths with proper Windows paths
+ m = re.match("^\/\w\/", extensionDir)
+ if m:
+ extensionDir = "%s:/%s" % (m.group(0)[1:2], extensionDir[3:])
+ extensionDir = extensionDir.replace("/", "\\")
+ if sys.platform == "darwin":
+ # Needed to avoid tab crashes on mac due to level 3 sandboxing
+ sourceRoot = os.path.join(extensionDir, "..", "..", "..", "..")
+ os.environ["MOZ_DEVELOPER_REPO_DIR"] = os.path.abspath(sourceRoot)
+
+ TPS = TPSTestRunner(
+ extensionDir,
+ binary=options.binary,
+ config=config,
+ debug=options.debug,
+ ignore_unused_engines=options.ignore_unused_engines,
+ logfile=options.logfile,
+ mobile=options.mobile,
+ resultfile=options.resultfile,
+ rlock=rlock,
+ testfile=testfile,
+ stop_on_error=options.stop_on_error,
+ )
+ TPS.run_tests()
+
+ if TPS.numfailed > 0 or TPS.numpassed == 0:
+ sys.exit(1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/testing/tps/tps/firefoxrunner.py b/testing/tps/tps/firefoxrunner.py
new file mode 100644
index 0000000000..e82392c6c0
--- /dev/null
+++ b/testing/tps/tps/firefoxrunner.py
@@ -0,0 +1,84 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+
+import httplib2
+import mozfile
+import mozinstall
+from mozprofile import Profile
+from mozrunner import FirefoxRunner
+
+
+class TPSFirefoxRunner(object):
+
+ PROCESS_TIMEOUT = 240
+
+ def __init__(self, binary):
+ if binary is not None and ("http://" in binary or "ftp://" in binary):
+ self.url = binary
+ self.binary = None
+ else:
+ self.url = None
+ self.binary = binary
+
+ self.installdir = None
+
+ def __del__(self):
+ if self.installdir:
+ mozfile.remove(self.installdir, True)
+
+ def download_url(self, url, dest=None):
+ h = httplib2.Http()
+ resp, content = h.request(url, "GET")
+ if dest is None:
+ dest = os.path.basename(url)
+
+ local = open(dest, "wb")
+ local.write(content)
+ local.close()
+ return dest
+
+ def download_build(self, installdir="downloadedbuild", appname="firefox"):
+ self.installdir = os.path.abspath(installdir)
+ buildName = os.path.basename(self.url)
+ pathToBuild = os.path.join(
+ os.path.dirname(os.path.abspath(__file__)), buildName
+ )
+
+ # delete the build if it already exists
+ if os.access(pathToBuild, os.F_OK):
+ os.remove(pathToBuild)
+
+ # download the build
+ print("downloading build")
+ self.download_url(self.url, pathToBuild)
+
+ # install the build
+ print("installing {}".format(pathToBuild))
+ mozfile.remove(self.installdir, True)
+ binary = mozinstall.install(src=pathToBuild, dest=self.installdir)
+
+ # remove the downloaded archive
+ os.remove(pathToBuild)
+
+ return binary
+
+ def run(self, profile=None, timeout=PROCESS_TIMEOUT, env=None, args=None):
+ """Runs the given FirefoxRunner with the given Profile, waits
+ for completion, then returns the process exit code
+ """
+ if profile is None:
+ profile = Profile()
+ self.profile = profile
+
+ if self.binary is None and self.url:
+ self.binary = self.download_build()
+
+ runner = FirefoxRunner(
+ profile=self.profile, binary=self.binary, env=env, cmdargs=args
+ )
+
+ runner.start(timeout=timeout)
+ return runner.wait()
diff --git a/testing/tps/tps/phase.py b/testing/tps/tps/phase.py
new file mode 100644
index 0000000000..f51cf9e4c6
--- /dev/null
+++ b/testing/tps/tps/phase.py
@@ -0,0 +1,88 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os.path
+import re
+
+
+class TPSTestPhase(object):
+
+ lineRe = re.compile(
+ r"^(.*?)test phase (?P<matchphase>[^\s]+): (?P<matchstatus>.*)$"
+ )
+
+ def __init__(
+ self,
+ phase,
+ profile,
+ testname,
+ testpath,
+ logfile,
+ env,
+ firefoxRunner,
+ logfn,
+ ignore_unused_engines=False,
+ ):
+ self.phase = phase
+ self.profile = profile
+ self.testname = str(testname) # this might be passed in as unicode
+ self.testpath = testpath
+ self.logfile = logfile
+ self.env = env
+ self.firefoxRunner = firefoxRunner
+ self.log = logfn
+ self.ignore_unused_engines = ignore_unused_engines
+ self._status = None
+ self.errline = ""
+
+ @property
+ def status(self):
+ return self._status if self._status else "unknown"
+
+ def run(self):
+ # launch Firefox
+
+ prefs = {
+ "testing.tps.testFile": os.path.abspath(self.testpath),
+ "testing.tps.testPhase": self.phase,
+ "testing.tps.logFile": self.logfile,
+ "testing.tps.ignoreUnusedEngines": self.ignore_unused_engines,
+ }
+
+ self.profile.set_preferences(prefs)
+
+ self.log(
+ "\nLaunching Firefox for phase %s with prefs %s\n"
+ % (self.phase, str(prefs))
+ )
+
+ self.firefoxRunner.run(env=self.env, args=[], profile=self.profile)
+
+ # parse the logfile and look for results from the current test phase
+ found_test = False
+ f = open(self.logfile, "r")
+ for line in f:
+
+ # skip to the part of the log file that deals with the test we're running
+ if not found_test:
+ if line.find("Running test %s" % self.testname) > -1:
+ found_test = True
+ else:
+ continue
+
+ # look for the status of the current phase
+ match = self.lineRe.match(line)
+ if match:
+ if match.group("matchphase") == self.phase:
+ self._status = match.group("matchstatus")
+ break
+
+ # set the status to FAIL if there is TPS error
+ if line.find("CROSSWEAVE ERROR: ") > -1 and not self._status:
+ self._status = "FAIL"
+ self.errline = line[
+ line.find("CROSSWEAVE ERROR: ") + len("CROSSWEAVE ERROR: ") :
+ ]
+
+ f.close()
diff --git a/testing/tps/tps/testrunner.py b/testing/tps/tps/testrunner.py
new file mode 100644
index 0000000000..f18c89fe90
--- /dev/null
+++ b/testing/tps/tps/testrunner.py
@@ -0,0 +1,517 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import json
+import os
+import re
+import tempfile
+import time
+import traceback
+
+import mozinfo
+import mozversion
+import yaml
+from mozprofile import Profile
+from wptserve import server
+
+from .firefoxrunner import TPSFirefoxRunner
+from .phase import TPSTestPhase
+
+
+class TempFile(object):
+ """Class for temporary files that delete themselves when garbage-collected."""
+
+ def __init__(self, prefix=None):
+ self.fd, self.filename = self.tmpfile = tempfile.mkstemp(prefix=prefix)
+
+ def write(self, data):
+ if self.fd:
+ os.write(self.fd, data)
+
+ def close(self):
+ if self.fd:
+ os.close(self.fd)
+ self.fd = None
+
+ def cleanup(self):
+ if self.fd:
+ self.close()
+ if os.access(self.filename, os.F_OK):
+ os.remove(self.filename)
+
+ __del__ = cleanup
+
+
+class TPSTestRunner(object):
+
+ extra_env = {
+ "MOZ_CRASHREPORTER_DISABLE": "1",
+ "GNOME_DISABLE_CRASH_DIALOG": "1",
+ "XRE_NO_WINDOWS_CRASH_DIALOG": "1",
+ "MOZ_NO_REMOTE": "1",
+ "XPCOM_DEBUG_BREAK": "warn",
+ }
+
+ default_preferences = {
+ "app.update.checkInstallTime": False,
+ "app.update.disabledForTesting": True,
+ "security.turn_off_all_security_so_that_viruses_can_take_over_this_computer": True,
+ "browser.dom.window.dump.enabled": True,
+ "devtools.console.stdout.chrome": True,
+ "browser.sessionstore.resume_from_crash": False,
+ "browser.shell.checkDefaultBrowser": False,
+ "browser.tabs.warnOnClose": False,
+ "browser.warnOnQuit": False,
+ # Allow installing extensions dropped into the profile folder
+ "extensions.autoDisableScopes": 10,
+ "extensions.getAddons.get.url": "http://127.0.0.1:4567/addons/api/%IDS%.json",
+ # Our pretend addons server doesn't support metadata...
+ "extensions.getAddons.cache.enabled": False,
+ "extensions.install.requireSecureOrigin": False,
+ "extensions.update.enabled": False,
+ # Don't open a dialog to show available add-on updates
+ "extensions.update.notifyUser": False,
+ "services.sync.firstSync": "notReady",
+ "services.sync.lastversion": "1.0",
+ "toolkit.startup.max_resumed_crashes": -1,
+ # hrm - not sure what the release/beta channels will do?
+ "xpinstall.signatures.required": False,
+ "services.sync.testing.tps": True,
+ # removed data: schema restriction for easier testing of tabs
+ "services.sync.engine.tabs.filteredSchemes": "about|resource|chrome|file|blob|moz-extension",
+ "engine.bookmarks.repair.enabled": False,
+ "extensions.experiments.enabled": True,
+ "webextensions.storage.sync.kinto": False,
+ }
+
+ debug_preferences = {
+ "services.sync.log.appender.console": "Trace",
+ "services.sync.log.appender.dump": "Trace",
+ "services.sync.log.appender.file.level": "Trace",
+ "services.sync.log.appender.file.logOnSuccess": True,
+ "services.sync.log.logger": "Trace",
+ "services.sync.log.logger.engine": "Trace",
+ }
+
+ syncVerRe = re.compile(r"Sync version: (?P<syncversion>.*)\n")
+ ffVerRe = re.compile(r"Firefox version: (?P<ffver>.*)\n")
+ ffBuildIDRe = re.compile(r"Firefox buildid: (?P<ffbuildid>.*)\n")
+
+ def __init__(
+ self,
+ extensionDir,
+ binary=None,
+ config=None,
+ debug=False,
+ ignore_unused_engines=False,
+ logfile="tps.log",
+ mobile=False,
+ rlock=None,
+ resultfile="tps_result.json",
+ testfile=None,
+ stop_on_error=False,
+ ):
+ self.binary = binary
+ self.config = config if config else {}
+ self.debug = debug
+ self.extensions = []
+ self.ignore_unused_engines = ignore_unused_engines
+ self.logfile = os.path.abspath(logfile)
+ self.mobile = mobile
+ self.rlock = rlock
+ self.resultfile = resultfile
+ self.testfile = testfile
+ self.stop_on_error = stop_on_error
+
+ self.addonversion = None
+ self.branch = None
+ self.changeset = None
+ self.errorlogs = {}
+ self.extensionDir = extensionDir
+ self.firefoxRunner = None
+ self.nightly = False
+ self.numfailed = 0
+ self.numpassed = 0
+ self.postdata = {}
+ self.productversion = None
+ self.repo = None
+ self.tpsxpi = None
+
+ @property
+ def mobile(self):
+ return self._mobile
+
+ @mobile.setter
+ def mobile(self, value):
+ self._mobile = value
+ self.synctype = "desktop" if not self._mobile else "mobile"
+
+ def log(self, msg, printToConsole=False):
+ """Appends a string to the logfile"""
+
+ f = open(self.logfile, "a")
+ f.write(msg)
+ f.close()
+ if printToConsole:
+ print(msg)
+
+ def writeToResultFile(self, postdata, body=None, sendTo=["crossweave@mozilla.com"]):
+ """Writes results to test file"""
+
+ results = {"results": []}
+
+ if os.access(self.resultfile, os.F_OK):
+ f = open(self.resultfile, "r")
+ results = json.loads(f.read())
+ f.close()
+
+ f = open(self.resultfile, "w")
+ if body is not None:
+ postdata["body"] = body
+ if self.numpassed is not None:
+ postdata["numpassed"] = self.numpassed
+ if self.numfailed is not None:
+ postdata["numfailed"] = self.numfailed
+ if self.firefoxRunner and self.firefoxRunner.url:
+ postdata["firefoxrunnerurl"] = self.firefoxRunner.url
+
+ postdata["sendTo"] = sendTo
+ results["results"].append(postdata)
+ f.write(json.dumps(results, indent=2))
+ f.close()
+
+ def _zip_add_file(self, zip, file, rootDir):
+ zip.write(os.path.join(rootDir, file), file)
+
+ def _zip_add_dir(self, zip, dir, rootDir):
+ try:
+ zip.write(os.path.join(rootDir, dir), dir)
+ except Exception:
+ # on some OS's, adding directory entries doesn't seem to work
+ pass
+ for root, dirs, files in os.walk(os.path.join(rootDir, dir)):
+ for f in files:
+ zip.write(os.path.join(root, f), os.path.join(dir, f))
+
+ def handle_phase_failure(self, profiles):
+ for profile in profiles:
+ self.log("\nDumping sync log for profile %s\n" % profiles[profile].profile)
+ for root, dirs, files in os.walk(
+ os.path.join(profiles[profile].profile, "weave", "logs")
+ ):
+ for f in files:
+ weavelog = os.path.join(
+ profiles[profile].profile, "weave", "logs", f
+ )
+ if os.access(weavelog, os.F_OK):
+ with open(weavelog, "r") as fh:
+ for line in fh:
+ possible_time = line[0:13]
+ if len(possible_time) == 13 and possible_time.isdigit():
+ time_ms = int(possible_time)
+ # pylint --py3k W1619
+ formatted = time.strftime(
+ "%Y-%m-%d %H:%M:%S",
+ time.localtime(time_ms / 1000),
+ )
+ self.log(
+ "%s.%03d %s"
+ % (formatted, time_ms % 1000, line[14:])
+ )
+ else:
+ self.log(line)
+
+ def run_single_test(self, testdir, testname):
+ testpath = os.path.join(testdir, testname)
+ self.log("Running test %s\n" % testname, True)
+
+ # Read and parse the test file, merge it with the contents of the config
+ # file, and write the combined output to a temporary file.
+ f = open(testpath, "r")
+ testcontent = f.read()
+ f.close()
+ # We use yaml to parse the tests because it is a superset of json
+ # but tolerates things like property names not being quoted, trailing
+ # commas, etc.
+ try:
+ test = yaml.safe_load(testcontent)
+ except Exception:
+ test = yaml.safe_load(
+ testcontent[testcontent.find("{") : testcontent.find("}") + 1]
+ )
+
+ self.preferences["tps.seconds_since_epoch"] = int(time.time())
+
+ # generate the profiles defined in the test, and a list of test phases
+ profiles = {}
+ phaselist = []
+ for phase in test:
+ profilename = test[phase]
+
+ # create the profile if necessary
+ if profilename not in profiles:
+ profiles[profilename] = Profile(
+ preferences=self.preferences.copy(), addons=self.extensions
+ )
+
+ # create the test phase
+ phaselist.append(
+ TPSTestPhase(
+ phase,
+ profiles[profilename],
+ testname,
+ testpath,
+ self.logfile,
+ self.env,
+ self.firefoxRunner,
+ self.log,
+ ignore_unused_engines=self.ignore_unused_engines,
+ )
+ )
+
+ # sort the phase list by name
+ phaselist = sorted(phaselist, key=lambda phase: phase.phase)
+
+ # run each phase in sequence, aborting at the first failure
+ failed = False
+ for phase in phaselist:
+ phase.run()
+ if phase.status != "PASS":
+ failed = True
+ break
+
+ for profilename in profiles:
+ print("### Cleanup Profile ", profilename)
+ cleanup_phase = TPSTestPhase(
+ "cleanup-" + profilename,
+ profiles[profilename],
+ testname,
+ testpath,
+ self.logfile,
+ self.env,
+ self.firefoxRunner,
+ self.log,
+ )
+
+ cleanup_phase.run()
+ if cleanup_phase.status != "PASS":
+ failed = True
+ # Keep going to run the remaining cleanup phases.
+
+ if failed:
+ self.handle_phase_failure(profiles)
+
+ # grep the log for FF and sync versions
+ f = open(self.logfile)
+ logdata = f.read()
+ match = self.syncVerRe.search(logdata)
+ sync_version = match.group("syncversion") if match else "unknown"
+ match = self.ffVerRe.search(logdata)
+ firefox_version = match.group("ffver") if match else "unknown"
+ match = self.ffBuildIDRe.search(logdata)
+ firefox_buildid = match.group("ffbuildid") if match else "unknown"
+ f.close()
+ if phase.status == "PASS":
+ logdata = ""
+ else:
+ # we only care about the log data for this specific test
+ logdata = logdata[logdata.find("Running test %s" % (str(testname))) :]
+
+ result = {
+ "PASS": lambda x: ("TEST-PASS", ""),
+ "FAIL": lambda x: ("TEST-UNEXPECTED-FAIL", x.rstrip()),
+ "unknown": lambda x: ("TEST-UNEXPECTED-FAIL", "test did not complete"),
+ }[phase.status](phase.errline)
+ logstr = "\n%s | %s%s\n" % (
+ result[0],
+ testname,
+ (" | %s" % result[1] if result[1] else ""),
+ )
+
+ try:
+ repoinfo = mozversion.get_version(self.binary)
+ except Exception:
+ repoinfo = {}
+ apprepo = repoinfo.get("application_repository", "")
+ appchangeset = repoinfo.get("application_changeset", "")
+
+ # save logdata to a temporary file for posting to the db
+ tmplogfile = None
+ if logdata:
+ tmplogfile = TempFile(prefix="tps_log_")
+ tmplogfile.write(logdata.encode("utf-8"))
+ tmplogfile.close()
+ self.errorlogs[testname] = tmplogfile
+
+ resultdata = {
+ "productversion": {
+ "version": firefox_version,
+ "buildid": firefox_buildid,
+ "builddate": firefox_buildid[0:8],
+ "product": "Firefox",
+ "repository": apprepo,
+ "changeset": appchangeset,
+ },
+ "addonversion": {"version": sync_version, "product": "Firefox Sync"},
+ "name": testname,
+ "message": result[1],
+ "state": result[0],
+ "logdata": logdata,
+ }
+
+ self.log(logstr, True)
+ for phase in phaselist:
+ print("\t{}: {}".format(phase.phase, phase.status))
+
+ return resultdata
+
+ def update_preferences(self):
+ self.preferences = self.default_preferences.copy()
+
+ if self.mobile:
+ self.preferences.update({"services.sync.client.type": "mobile"})
+
+ # If we are using legacy Sync, then set a dummy username to force the
+ # correct authentication type. Without this pref set to a value
+ # without an '@' character, Sync will initialize for FxA.
+ if self.config.get("auth_type", "fx_account") != "fx_account":
+ self.preferences.update({"services.sync.username": "dummy"})
+
+ if self.debug:
+ self.preferences.update(self.debug_preferences)
+
+ if "preferences" in self.config:
+ self.preferences.update(self.config["preferences"])
+
+ self.preferences["tps.config"] = json.dumps(self.config)
+
+ def run_tests(self):
+ # delete the logfile if it already exists
+ if os.access(self.logfile, os.F_OK):
+ os.remove(self.logfile)
+
+ # Copy the system env variables, and update them for custom settings
+ self.env = os.environ.copy()
+ self.env.update(self.extra_env)
+
+ # Update preferences for custom settings
+ self.update_preferences()
+
+ # Acquire a lock to make sure no other threads are running tests
+ # at the same time.
+ if self.rlock:
+ self.rlock.acquire()
+
+ try:
+ # Create the Firefox runner, which will download and install the
+ # build, as needed.
+ if not self.firefoxRunner:
+ self.firefoxRunner = TPSFirefoxRunner(self.binary)
+
+ # now, run the test group
+ self.run_test_group()
+
+ except Exception:
+ traceback.print_exc()
+ self.numpassed = 0
+ self.numfailed = 1
+ try:
+ self.writeToResultFile(
+ self.postdata, "<pre>%s</pre>" % traceback.format_exc()
+ )
+ except Exception:
+ traceback.print_exc()
+ else:
+ try:
+
+ self.writeToResultFile(self.postdata)
+ except Exception:
+ traceback.print_exc()
+ try:
+ self.writeToResultFile(
+ self.postdata, "<pre>%s</pre>" % traceback.format_exc()
+ )
+ except Exception:
+ traceback.print_exc()
+
+ # release our lock
+ if self.rlock:
+ self.rlock.release()
+
+ # dump out a summary of test results
+ print("Test Summary\n")
+ for test in self.postdata.get("tests", {}):
+ print("{} | {} | {}".format(test["state"], test["name"], test["message"]))
+
+ def run_test_group(self):
+ self.results = []
+
+ # reset number of passed/failed tests
+ self.numpassed = 0
+ self.numfailed = 0
+
+ # build our tps.xpi extension
+ self.extensions = []
+ self.extensions.append(os.path.join(self.extensionDir, "tps"))
+
+ # build the test list
+ try:
+ f = open(self.testfile)
+ jsondata = f.read()
+ f.close()
+ testfiles = json.loads(jsondata)
+ testlist = []
+ for (filename, meta) in testfiles["tests"].items():
+ skip_reason = meta.get("disabled")
+ if skip_reason:
+ print("Skipping test {} - {}".format(filename, skip_reason))
+ else:
+ testlist.append(filename)
+ except ValueError:
+ testlist = [os.path.basename(self.testfile)]
+ testdir = os.path.dirname(self.testfile)
+
+ self.server = server.WebTestHttpd(port=4567, doc_root=testdir)
+ self.server.start()
+
+ # run each test, and save the results
+ for test in testlist:
+ result = self.run_single_test(testdir, test)
+
+ if not self.productversion:
+ self.productversion = result["productversion"]
+ if not self.addonversion:
+ self.addonversion = result["addonversion"]
+
+ self.results.append(
+ {
+ "state": result["state"],
+ "name": result["name"],
+ "message": result["message"],
+ "logdata": result["logdata"],
+ }
+ )
+ if result["state"] == "TEST-PASS":
+ self.numpassed += 1
+ else:
+ self.numfailed += 1
+ if self.stop_on_error:
+ print(
+ "\nTest failed with --stop-on-error specified; "
+ "not running any more tests.\n"
+ )
+ break
+
+ self.server.stop()
+
+ # generate the postdata we'll use to post the results to the db
+ self.postdata = {
+ "tests": self.results,
+ "os": "%s %sbit" % (mozinfo.version, mozinfo.bits),
+ "testtype": "crossweave",
+ "productversion": self.productversion,
+ "addonversion": self.addonversion,
+ "synctype": self.synctype,
+ }
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-02 12:57:25 +0000