diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 09:22:09 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 09:22:09 +0000 |
commit | 43a97878ce14b72f0981164f87f2e35e14151312 (patch) | |
tree | 620249daf56c0258faa40cbdcf9cfba06de2a846 /testing/tps | |
parent | Initial commit. (diff) | |
download | firefox-upstream.tar.xz firefox-upstream.zip |
Adding upstream version 110.0.1.upstream/110.0.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r-- | testing/tps/.gitignore | 4 | ||||
-rw-r--r-- | testing/tps/README | 58 | ||||
-rw-r--r-- | testing/tps/config/config.json.in | 13 | ||||
-rwxr-xr-x | testing/tps/create_venv.py | 204 | ||||
-rw-r--r-- | testing/tps/mach_commands.py | 35 | ||||
-rw-r--r-- | testing/tps/moz.build | 8 | ||||
-rw-r--r-- | testing/tps/pages/microsummary1.txt | 1 | ||||
-rw-r--r-- | testing/tps/pages/microsummary2.txt | 1 | ||||
-rw-r--r-- | testing/tps/pages/microsummary3.txt | 1 | ||||
-rw-r--r-- | testing/tps/pages/page1.html | 14 | ||||
-rw-r--r-- | testing/tps/pages/page2.html | 14 | ||||
-rw-r--r-- | testing/tps/pages/page3.html | 14 | ||||
-rw-r--r-- | testing/tps/pages/page4.html | 14 | ||||
-rw-r--r-- | testing/tps/pages/page5.html | 14 | ||||
-rw-r--r-- | testing/tps/setup.py | 50 | ||||
-rw-r--r-- | testing/tps/tps/__init__.py | 9 | ||||
-rw-r--r-- | testing/tps/tps/cli.py | 157 | ||||
-rw-r--r-- | testing/tps/tps/firefoxrunner.py | 84 | ||||
-rw-r--r-- | testing/tps/tps/phase.py | 88 | ||||
-rw-r--r-- | testing/tps/tps/testrunner.py | 515 |
20 files changed, 1298 insertions, 0 deletions
diff --git a/testing/tps/.gitignore b/testing/tps/.gitignore new file mode 100644 index 0000000000..65f8b6e053 --- /dev/null +++ b/testing/tps/.gitignore @@ -0,0 +1,4 @@ +# These files are added by running the TPS test suite. +build/ +dist/ +tps.egg-info/ diff --git a/testing/tps/README b/testing/tps/README new file mode 100644 index 0000000000..0612bea4c6 --- /dev/null +++ b/testing/tps/README @@ -0,0 +1,58 @@ +TPS is a test automation framework for Firefox Sync. See +https://developer.mozilla.org/en/TPS for documentation. + +Installation +============ + +TPS requires several packages to operate properly. To install TPS and +required packages, use the INSTALL.sh script, provided: + + python3 create_venv.py /path/to/create/virtualenv + +This script will create a virtalenv and install TPS into it. + +You must then activate the virtualenv by executing: + +- (mac/linux): source /path/to/virtualenv/Scripts/activate +- (win): /path/to/virtualenv/Scripts/activate.bat + +TPS can then be run by executing: + + runtps --binary=/path/to/firefox + +> Note: You can run the tps tests in headless mode by using `MOZ_HEADLESS=1`. This will make +> your computer somewhat useable while the tests are running. + +When you are done with TPS, you can deactivate the virtualenv by executing +`deactivate` + +Configuration +============= +To edit the TPS configuration, do not edit config/config.json.in in the tree. +Instead, edit config.json inside your virtualenv; it will be located at the +top level of where you specified the virtualenv be created - eg, for the +example above, it will be `/path/to/create/virtualenv/config.json` + +Setting Up Test Accounts +======================== + +Firefox Accounts +---------------- +To create a test account for using the Firefox Account authentication perform the +following steps: + +> Note: Currently, the TPS tests rely on how restmail returns the verification code +> You should use restmail or something very similar. +> Gmail and other providers might give a `The request was blocked for security reasons` + +1. Go to the URL: http://restmail.net/mail/%account_prefix%@restmail.net + - Replace `%account_prefix%` with your own test name +2. Go to https://accounts.firefox.com/signup?service=sync&context=fx_desktop_v1 +3. Sign in with the previous chosen email address and a password +4. Go back to the Restmail URL, reload the page +5. Search for the verification link and open that page + +Now you will be able to use this account for TPS. Note that these +steps can be done in either a test profile or in a private browsing window - you +might want to avoid doing that in a "real" profile that's already connected to +Sync. diff --git a/testing/tps/config/config.json.in b/testing/tps/config/config.json.in new file mode 100644 index 0000000000..38cf374628 --- /dev/null +++ b/testing/tps/config/config.json.in @@ -0,0 +1,13 @@ +{ + "fx_account": { + "username": "__FX_ACCOUNT_USERNAME__", + "password": "__FX_ACCOUNT_PASSWORD__" + }, + "auth_type": "fx_account", + "es": "localhost:9200", + "os": "Ubuntu", + "platform": "linux64", + "serverURL": null, + "extensiondir": "__EXTENSIONDIR__", + "testdir": "__TESTDIR__" +} diff --git a/testing/tps/create_venv.py b/testing/tps/create_venv.py new file mode 100755 index 0000000000..3d938ffc78 --- /dev/null +++ b/testing/tps/create_venv.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +This scripts sets up a virtualenv and installs TPS into it. +It's probably best to specify a path NOT inside the repo, otherwise +all the virtualenv files will show up in e.g. hg status. +""" + +import optparse +import os +import subprocess +import sys +import venv + +here = os.path.dirname(os.path.abspath(__file__)) +usage_message = """ +*********************************************************************** + +To run TPS, activate the virtualenv using: + source {TARGET}/{BIN_NAME} + +To change your TPS config, please edit the file: + {TARGET}/config.json + +To execute tps use: + runtps --binary=/path/to/firefox + +See runtps --help for all options + +*********************************************************************** +""" + +if sys.platform == "win32": + bin_name = os.path.join("Scripts", "activate.bat") + python_env = os.path.join("Scripts", "python.exe") +else: + bin_name = os.path.join("bin", "activate") + python_env = os.path.join("bin", "python") + + +def setup_virtualenv(target): + print("Creating new virtual environment:", os.path.abspath(target)) + # system_site_packages=True so we have access to setuptools. + venv.create(target, system_site_packages=True) + + +def update_configfile(source, target, replacements): + lines = [] + + with open(source) as config: + for line in config: + for source_string, target_string in replacements.items(): + if target_string: + line = line.replace(source_string, target_string) + lines.append(line) + + with open(target, "w") as config: + for line in lines: + config.write(line) + + +def main(): + parser = optparse.OptionParser("Usage: %prog [options] path_to_venv") + parser.add_option( + "--keep-config", + dest="keep_config", + action="store_true", + help="Keep the existing config file.", + ) + parser.add_option( + "--password", + type="string", + dest="password", + metavar="FX_ACCOUNT_PASSWORD", + default=None, + help="The Firefox Account password.", + ) + parser.add_option( + "-p", + "--python", + type="string", + dest="python", + metavar="PYTHON_BIN", + default=None, + help="The Python interpreter to use.", + ) + parser.add_option( + "--sync-passphrase", + type="string", + dest="sync_passphrase", + metavar="SYNC_ACCOUNT_PASSPHRASE", + default=None, + help="The old Firefox Sync account passphrase.", + ) + parser.add_option( + "--sync-password", + type="string", + dest="sync_password", + metavar="SYNC_ACCOUNT_PASSWORD", + default=None, + help="The old Firefox Sync account password.", + ) + parser.add_option( + "--sync-username", + type="string", + dest="sync_username", + metavar="SYNC_ACCOUNT_USERNAME", + default=None, + help="The old Firefox Sync account username.", + ) + parser.add_option( + "--username", + type="string", + dest="username", + metavar="FX_ACCOUNT_USERNAME", + default=None, + help="The Firefox Account username.", + ) + + (options, args) = parser.parse_args(args=None, values=None) + + if len(args) != 1: + parser.error("Path to the environment has to be specified") + target = args[0] + assert target + + setup_virtualenv(target) + + # Activate tps environment + activate(target) + + # Install TPS in environment + subprocess.check_call( + [os.path.join(target, python_env), os.path.join(here, "setup.py"), "install"] + ) + + # Get the path to tests and extensions directory by checking check where + # the tests and extensions directories are located + sync_dir = os.path.abspath(os.path.join(here, "..", "..", "services", "sync")) + if os.path.exists(sync_dir): + testdir = os.path.join(sync_dir, "tests", "tps") + extdir = os.path.join(sync_dir, "tps", "extensions") + else: + testdir = os.path.join(here, "tests") + extdir = os.path.join(here, "extensions") + + if not options.keep_config: + update_configfile( + os.path.join(here, "config", "config.json.in"), + os.path.join(target, "config.json"), + replacements={ + "__TESTDIR__": testdir.replace("\\", "/"), + "__EXTENSIONDIR__": extdir.replace("\\", "/"), + "__FX_ACCOUNT_USERNAME__": options.username, + "__FX_ACCOUNT_PASSWORD__": options.password, + "__SYNC_ACCOUNT_USERNAME__": options.sync_username, + "__SYNC_ACCOUNT_PASSWORD__": options.sync_password, + "__SYNC_ACCOUNT_PASSPHRASE__": options.sync_passphrase, + }, + ) + + if not (options.username and options.password): + print("\nFirefox Account credentials not specified.") + if not (options.sync_username and options.sync_password and options.passphrase): + print("\nFirefox Sync account credentials not specified.") + + # Print the user instructions + print(usage_message.format(TARGET=target, BIN_NAME=bin_name)) + + +def activate(target): + # This is a lightly modified copy of `activate_this.py`, which existed when + # venv was an external package, but doesn't come with the builtin venv support. + old_os_path = os.environ.get("PATH", "") + os.environ["PATH"] = ( + os.path.dirname(os.path.abspath(__file__)) + os.pathsep + old_os_path + ) + base = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + if sys.platform == "win32": + site_packages = os.path.join(base, "Lib", "site-packages") + else: + site_packages = os.path.join( + base, "lib", "python%s" % sys.version[:3], "site-packages" + ) + prev_sys_path = list(sys.path) + import site + + site.addsitedir(site_packages) + sys.real_prefix = sys.prefix + sys.prefix = base + # Move the added items to the front of the path: + new_sys_path = [] + for item in list(sys.path): + if item not in prev_sys_path: + new_sys_path.append(item) + sys.path.remove(item) + sys.path[:0] = new_sys_path + + +if __name__ == "__main__": + main() diff --git a/testing/tps/mach_commands.py b/testing/tps/mach_commands.py new file mode 100644 index 0000000000..34b20ff799 --- /dev/null +++ b/testing/tps/mach_commands.py @@ -0,0 +1,35 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os + +from mach.decorators import Command, CommandArgument +from mozpack.copier import Jarrer +from mozpack.files import FileFinder + + +@Command("tps-build", category="testing", description="Build TPS add-on.") +@CommandArgument("--dest", default=None, help="Where to write add-on.") +def build(command_context, dest): + src = os.path.join( + command_context.topsrcdir, "services", "sync", "tps", "extensions", "tps" + ) + dest = os.path.join( + dest or os.path.join(command_context.topobjdir, "services", "sync"), + "tps.xpi", + ) + + if not os.path.exists(os.path.dirname(dest)): + os.makedirs(os.path.dirname(dest)) + + if os.path.isfile(dest): + os.unlink(dest) + + jarrer = Jarrer() + for p, f in FileFinder(src).find("*"): + jarrer.add(p, f) + jarrer.copy(dest) + + print("Built TPS add-on as %s" % dest) diff --git a/testing/tps/moz.build b/testing/tps/moz.build new file mode 100644 index 0000000000..06a74e1bfc --- /dev/null +++ b/testing/tps/moz.build @@ -0,0 +1,8 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +with Files("**"): + BUG_COMPONENT = ("Testing", "General") diff --git a/testing/tps/pages/microsummary1.txt b/testing/tps/pages/microsummary1.txt new file mode 100644 index 0000000000..9451d4b5c0 --- /dev/null +++ b/testing/tps/pages/microsummary1.txt @@ -0,0 +1 @@ +Static microsummary #1
diff --git a/testing/tps/pages/microsummary2.txt b/testing/tps/pages/microsummary2.txt new file mode 100644 index 0000000000..7e32085044 --- /dev/null +++ b/testing/tps/pages/microsummary2.txt @@ -0,0 +1 @@ +Static microsummary #2
diff --git a/testing/tps/pages/microsummary3.txt b/testing/tps/pages/microsummary3.txt new file mode 100644 index 0000000000..00ac62bbf2 --- /dev/null +++ b/testing/tps/pages/microsummary3.txt @@ -0,0 +1 @@ +Static microsummary #3
diff --git a/testing/tps/pages/page1.html b/testing/tps/pages/page1.html new file mode 100644 index 0000000000..5178bae13a --- /dev/null +++ b/testing/tps/pages/page1.html @@ -0,0 +1,14 @@ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<html> +<head> +<title>Crossweave Test Page 1</title> +</head> +<body> +<p> +Crossweave Test Page 1 +</p> +</body> +</html> diff --git a/testing/tps/pages/page2.html b/testing/tps/pages/page2.html new file mode 100644 index 0000000000..da117bcc8e --- /dev/null +++ b/testing/tps/pages/page2.html @@ -0,0 +1,14 @@ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<html> +<head> +<title>Crossweave Test Page 2</title> +</head> +<body> +<p> +Crossweave Test Page 2 +</p> +</body> +</html> diff --git a/testing/tps/pages/page3.html b/testing/tps/pages/page3.html new file mode 100644 index 0000000000..001762be83 --- /dev/null +++ b/testing/tps/pages/page3.html @@ -0,0 +1,14 @@ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<html> +<head> +<title>Crossweave Test Page 3</title> +</head> +<body> +<p> +Crossweave Test Page 3 +</p> +</body> +</html> diff --git a/testing/tps/pages/page4.html b/testing/tps/pages/page4.html new file mode 100644 index 0000000000..5234089679 --- /dev/null +++ b/testing/tps/pages/page4.html @@ -0,0 +1,14 @@ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<html> +<head> +<title>Crossweave Test Page 4</title> +</head> +<body> +<p> +Crossweave Test Page 4 +</p> +</body> +</html> diff --git a/testing/tps/pages/page5.html b/testing/tps/pages/page5.html new file mode 100644 index 0000000000..34adbb6827 --- /dev/null +++ b/testing/tps/pages/page5.html @@ -0,0 +1,14 @@ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<html> +<head> +<title>Crossweave Test Page 5</title> +</head> +<body> +<p> +Crossweave Test Page 5 +</p> +</body> +</html> diff --git a/testing/tps/setup.py b/testing/tps/setup.py new file mode 100644 index 0000000000..0bcf4387f0 --- /dev/null +++ b/testing/tps/setup.py @@ -0,0 +1,50 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from setuptools import find_packages, setup + +version = "0.6" + +deps = [ + "httplib2 == 0.9.2", + "mozfile >= 1.2", + "wptserve >= 3.0", + "mozinfo >= 1.2", + "mozinstall == 2.0.1", + "mozprocess == 1.3", + "mozprofile ~= 2.1", + "mozrunner ~= 8.2", + "mozversion == 2.3", + "PyYAML >= 4.0", +] + +setup( + name="tps", + version=version, + description="run automated multi-profile sync tests", + long_description="""\ +""", + classifiers=[ + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 2 :: Only", + ], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + keywords="", + author="Mozilla Automation and Tools team", + author_email="tools@lists.mozilla.org", + url="https://developer.mozilla.org/en-US/docs/TPS", + license="MPL 2.0", + packages=find_packages(exclude=["ez_setup", "examples", "tests"]), + include_package_data=True, + zip_safe=False, + install_requires=deps, + entry_points=""" + # -*- Entry points: -*- + [console_scripts] + runtps = tps.cli:main + """, + data_files=[ + ("tps", ["config/config.json.in"]), + ], +) diff --git a/testing/tps/tps/__init__.py b/testing/tps/tps/__init__.py new file mode 100644 index 0000000000..a4c384c1ce --- /dev/null +++ b/testing/tps/tps/__init__.py @@ -0,0 +1,9 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# flake8: noqa + + +from .firefoxrunner import TPSFirefoxRunner +from .testrunner import TPSTestRunner diff --git a/testing/tps/tps/cli.py b/testing/tps/tps/cli.py new file mode 100644 index 0000000000..9ef6186b4d --- /dev/null +++ b/testing/tps/tps/cli.py @@ -0,0 +1,157 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import optparse +import os +import re +import sys +from threading import RLock + +from tps import TPSTestRunner + + +def main(): + parser = optparse.OptionParser() + parser.add_option( + "--binary", + action="store", + type="string", + dest="binary", + default=None, + help="path to the Firefox binary, specified either as " + "a local file or a url; if omitted, the PATH " + "will be searched;", + ) + parser.add_option( + "--configfile", + action="store", + type="string", + dest="configfile", + default=None, + help="path to the config file to use default: %default]", + ) + parser.add_option( + "--debug", + action="store_true", + dest="debug", + default=False, + help="run in debug mode", + ) + parser.add_option( + "--ignore-unused-engines", + default=False, + action="store_true", + dest="ignore_unused_engines", + help="If defined, do not load unused engines in individual tests." + " Has no effect for pulse monitor.", + ) + parser.add_option( + "--logfile", + action="store", + type="string", + dest="logfile", + default="tps.log", + help="path to the log file [default: %default]", + ) + parser.add_option( + "--mobile", + action="store_true", + dest="mobile", + default=False, + help="run with mobile settings", + ) + parser.add_option( + "--pulsefile", + action="store", + type="string", + dest="pulsefile", + default=None, + help="path to file containing a pulse message in " + "json format that you want to inject into the monitor", + ) + parser.add_option( + "--resultfile", + action="store", + type="string", + dest="resultfile", + default="tps_result.json", + help="path to the result file [default: %default]", + ) + parser.add_option( + "--testfile", + action="store", + type="string", + dest="testfile", + default="all_tests.json", + help="path to the test file to run [default: %default]", + ) + parser.add_option( + "--stop-on-error", + action="store_true", + dest="stop_on_error", + help="stop running tests after the first failure", + ) + (options, args) = parser.parse_args() + + configfile = options.configfile + if configfile is None: + virtual_env = os.environ.get("VIRTUAL_ENV") + if virtual_env: + configfile = os.path.join(virtual_env, "config.json") + if configfile is None or not os.access(configfile, os.F_OK): + raise Exception( + "Unable to find config.json in a VIRTUAL_ENV; you must " + "specify a config file using the --configfile option" + ) + + # load the config file + f = open(configfile, "r") + configcontent = f.read() + f.close() + config = json.loads(configcontent) + testfile = os.path.join(config.get("testdir", ""), options.testfile) + + rlock = RLock() + + print("using result file", options.resultfile) + + extensionDir = config.get("extensiondir") + if not extensionDir or extensionDir == "__EXTENSIONDIR__": + extensionDir = os.path.join( + os.getcwd(), "..", "..", "services", "sync", "tps", "extensions" + ) + else: + if sys.platform == "win32": + # replace msys-style paths with proper Windows paths + m = re.match("^\/\w\/", extensionDir) + if m: + extensionDir = "%s:/%s" % (m.group(0)[1:2], extensionDir[3:]) + extensionDir = extensionDir.replace("/", "\\") + if sys.platform == "darwin": + # Needed to avoid tab crashes on mac due to level 3 sandboxing + sourceRoot = os.path.join(extensionDir, "..", "..", "..", "..") + os.environ["MOZ_DEVELOPER_REPO_DIR"] = os.path.abspath(sourceRoot) + + TPS = TPSTestRunner( + extensionDir, + binary=options.binary, + config=config, + debug=options.debug, + ignore_unused_engines=options.ignore_unused_engines, + logfile=options.logfile, + mobile=options.mobile, + resultfile=options.resultfile, + rlock=rlock, + testfile=testfile, + stop_on_error=options.stop_on_error, + ) + TPS.run_tests() + + if TPS.numfailed > 0 or TPS.numpassed == 0: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/testing/tps/tps/firefoxrunner.py b/testing/tps/tps/firefoxrunner.py new file mode 100644 index 0000000000..e82392c6c0 --- /dev/null +++ b/testing/tps/tps/firefoxrunner.py @@ -0,0 +1,84 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +import httplib2 +import mozfile +import mozinstall +from mozprofile import Profile +from mozrunner import FirefoxRunner + + +class TPSFirefoxRunner(object): + + PROCESS_TIMEOUT = 240 + + def __init__(self, binary): + if binary is not None and ("http://" in binary or "ftp://" in binary): + self.url = binary + self.binary = None + else: + self.url = None + self.binary = binary + + self.installdir = None + + def __del__(self): + if self.installdir: + mozfile.remove(self.installdir, True) + + def download_url(self, url, dest=None): + h = httplib2.Http() + resp, content = h.request(url, "GET") + if dest is None: + dest = os.path.basename(url) + + local = open(dest, "wb") + local.write(content) + local.close() + return dest + + def download_build(self, installdir="downloadedbuild", appname="firefox"): + self.installdir = os.path.abspath(installdir) + buildName = os.path.basename(self.url) + pathToBuild = os.path.join( + os.path.dirname(os.path.abspath(__file__)), buildName + ) + + # delete the build if it already exists + if os.access(pathToBuild, os.F_OK): + os.remove(pathToBuild) + + # download the build + print("downloading build") + self.download_url(self.url, pathToBuild) + + # install the build + print("installing {}".format(pathToBuild)) + mozfile.remove(self.installdir, True) + binary = mozinstall.install(src=pathToBuild, dest=self.installdir) + + # remove the downloaded archive + os.remove(pathToBuild) + + return binary + + def run(self, profile=None, timeout=PROCESS_TIMEOUT, env=None, args=None): + """Runs the given FirefoxRunner with the given Profile, waits + for completion, then returns the process exit code + """ + if profile is None: + profile = Profile() + self.profile = profile + + if self.binary is None and self.url: + self.binary = self.download_build() + + runner = FirefoxRunner( + profile=self.profile, binary=self.binary, env=env, cmdargs=args + ) + + runner.start(timeout=timeout) + return runner.wait() diff --git a/testing/tps/tps/phase.py b/testing/tps/tps/phase.py new file mode 100644 index 0000000000..f51cf9e4c6 --- /dev/null +++ b/testing/tps/tps/phase.py @@ -0,0 +1,88 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os.path +import re + + +class TPSTestPhase(object): + + lineRe = re.compile( + r"^(.*?)test phase (?P<matchphase>[^\s]+): (?P<matchstatus>.*)$" + ) + + def __init__( + self, + phase, + profile, + testname, + testpath, + logfile, + env, + firefoxRunner, + logfn, + ignore_unused_engines=False, + ): + self.phase = phase + self.profile = profile + self.testname = str(testname) # this might be passed in as unicode + self.testpath = testpath + self.logfile = logfile + self.env = env + self.firefoxRunner = firefoxRunner + self.log = logfn + self.ignore_unused_engines = ignore_unused_engines + self._status = None + self.errline = "" + + @property + def status(self): + return self._status if self._status else "unknown" + + def run(self): + # launch Firefox + + prefs = { + "testing.tps.testFile": os.path.abspath(self.testpath), + "testing.tps.testPhase": self.phase, + "testing.tps.logFile": self.logfile, + "testing.tps.ignoreUnusedEngines": self.ignore_unused_engines, + } + + self.profile.set_preferences(prefs) + + self.log( + "\nLaunching Firefox for phase %s with prefs %s\n" + % (self.phase, str(prefs)) + ) + + self.firefoxRunner.run(env=self.env, args=[], profile=self.profile) + + # parse the logfile and look for results from the current test phase + found_test = False + f = open(self.logfile, "r") + for line in f: + + # skip to the part of the log file that deals with the test we're running + if not found_test: + if line.find("Running test %s" % self.testname) > -1: + found_test = True + else: + continue + + # look for the status of the current phase + match = self.lineRe.match(line) + if match: + if match.group("matchphase") == self.phase: + self._status = match.group("matchstatus") + break + + # set the status to FAIL if there is TPS error + if line.find("CROSSWEAVE ERROR: ") > -1 and not self._status: + self._status = "FAIL" + self.errline = line[ + line.find("CROSSWEAVE ERROR: ") + len("CROSSWEAVE ERROR: ") : + ] + + f.close() diff --git a/testing/tps/tps/testrunner.py b/testing/tps/tps/testrunner.py new file mode 100644 index 0000000000..66a68b3580 --- /dev/null +++ b/testing/tps/tps/testrunner.py @@ -0,0 +1,515 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os +import re +import tempfile +import time +import traceback + +import mozinfo +import mozversion +import yaml +from mozprofile import Profile +from wptserve import server + +from .firefoxrunner import TPSFirefoxRunner +from .phase import TPSTestPhase + + +class TempFile(object): + """Class for temporary files that delete themselves when garbage-collected.""" + + def __init__(self, prefix=None): + self.fd, self.filename = self.tmpfile = tempfile.mkstemp(prefix=prefix) + + def write(self, data): + if self.fd: + os.write(self.fd, data) + + def close(self): + if self.fd: + os.close(self.fd) + self.fd = None + + def cleanup(self): + if self.fd: + self.close() + if os.access(self.filename, os.F_OK): + os.remove(self.filename) + + __del__ = cleanup + + +class TPSTestRunner(object): + + extra_env = { + "MOZ_CRASHREPORTER_DISABLE": "1", + "GNOME_DISABLE_CRASH_DIALOG": "1", + "XRE_NO_WINDOWS_CRASH_DIALOG": "1", + "MOZ_NO_REMOTE": "1", + "XPCOM_DEBUG_BREAK": "warn", + } + + default_preferences = { + "app.update.checkInstallTime": False, + "app.update.disabledForTesting": True, + "security.turn_off_all_security_so_that_viruses_can_take_over_this_computer": True, + "browser.dom.window.dump.enabled": True, + "devtools.console.stdout.chrome": True, + "browser.sessionstore.resume_from_crash": False, + "browser.shell.checkDefaultBrowser": False, + "browser.tabs.warnOnClose": False, + "browser.warnOnQuit": False, + # Allow installing extensions dropped into the profile folder + "extensions.autoDisableScopes": 10, + "extensions.getAddons.get.url": "http://127.0.0.1:4567/addons/api/%IDS%.json", + # Our pretend addons server doesn't support metadata... + "extensions.getAddons.cache.enabled": False, + "extensions.install.requireSecureOrigin": False, + "extensions.update.enabled": False, + # Don't open a dialog to show available add-on updates + "extensions.update.notifyUser": False, + "services.sync.firstSync": "notReady", + "services.sync.lastversion": "1.0", + "toolkit.startup.max_resumed_crashes": -1, + # hrm - not sure what the release/beta channels will do? + "xpinstall.signatures.required": False, + "services.sync.testing.tps": True, + "engine.bookmarks.repair.enabled": False, + "extensions.experiments.enabled": True, + "webextensions.storage.sync.kinto": False, + } + + debug_preferences = { + "services.sync.log.appender.console": "Trace", + "services.sync.log.appender.dump": "Trace", + "services.sync.log.appender.file.level": "Trace", + "services.sync.log.appender.file.logOnSuccess": True, + "services.sync.log.logger": "Trace", + "services.sync.log.logger.engine": "Trace", + } + + syncVerRe = re.compile(r"Sync version: (?P<syncversion>.*)\n") + ffVerRe = re.compile(r"Firefox version: (?P<ffver>.*)\n") + ffBuildIDRe = re.compile(r"Firefox buildid: (?P<ffbuildid>.*)\n") + + def __init__( + self, + extensionDir, + binary=None, + config=None, + debug=False, + ignore_unused_engines=False, + logfile="tps.log", + mobile=False, + rlock=None, + resultfile="tps_result.json", + testfile=None, + stop_on_error=False, + ): + self.binary = binary + self.config = config if config else {} + self.debug = debug + self.extensions = [] + self.ignore_unused_engines = ignore_unused_engines + self.logfile = os.path.abspath(logfile) + self.mobile = mobile + self.rlock = rlock + self.resultfile = resultfile + self.testfile = testfile + self.stop_on_error = stop_on_error + + self.addonversion = None + self.branch = None + self.changeset = None + self.errorlogs = {} + self.extensionDir = extensionDir + self.firefoxRunner = None + self.nightly = False + self.numfailed = 0 + self.numpassed = 0 + self.postdata = {} + self.productversion = None + self.repo = None + self.tpsxpi = None + + @property + def mobile(self): + return self._mobile + + @mobile.setter + def mobile(self, value): + self._mobile = value + self.synctype = "desktop" if not self._mobile else "mobile" + + def log(self, msg, printToConsole=False): + """Appends a string to the logfile""" + + f = open(self.logfile, "a") + f.write(msg) + f.close() + if printToConsole: + print(msg) + + def writeToResultFile(self, postdata, body=None, sendTo=["crossweave@mozilla.com"]): + """Writes results to test file""" + + results = {"results": []} + + if os.access(self.resultfile, os.F_OK): + f = open(self.resultfile, "r") + results = json.loads(f.read()) + f.close() + + f = open(self.resultfile, "w") + if body is not None: + postdata["body"] = body + if self.numpassed is not None: + postdata["numpassed"] = self.numpassed + if self.numfailed is not None: + postdata["numfailed"] = self.numfailed + if self.firefoxRunner and self.firefoxRunner.url: + postdata["firefoxrunnerurl"] = self.firefoxRunner.url + + postdata["sendTo"] = sendTo + results["results"].append(postdata) + f.write(json.dumps(results, indent=2)) + f.close() + + def _zip_add_file(self, zip, file, rootDir): + zip.write(os.path.join(rootDir, file), file) + + def _zip_add_dir(self, zip, dir, rootDir): + try: + zip.write(os.path.join(rootDir, dir), dir) + except Exception: + # on some OS's, adding directory entries doesn't seem to work + pass + for root, dirs, files in os.walk(os.path.join(rootDir, dir)): + for f in files: + zip.write(os.path.join(root, f), os.path.join(dir, f)) + + def handle_phase_failure(self, profiles): + for profile in profiles: + self.log("\nDumping sync log for profile %s\n" % profiles[profile].profile) + for root, dirs, files in os.walk( + os.path.join(profiles[profile].profile, "weave", "logs") + ): + for f in files: + weavelog = os.path.join( + profiles[profile].profile, "weave", "logs", f + ) + if os.access(weavelog, os.F_OK): + with open(weavelog, "r") as fh: + for line in fh: + possible_time = line[0:13] + if len(possible_time) == 13 and possible_time.isdigit(): + time_ms = int(possible_time) + # pylint --py3k W1619 + formatted = time.strftime( + "%Y-%m-%d %H:%M:%S", + time.localtime(time_ms / 1000), + ) + self.log( + "%s.%03d %s" + % (formatted, time_ms % 1000, line[14:]) + ) + else: + self.log(line) + + def run_single_test(self, testdir, testname): + testpath = os.path.join(testdir, testname) + self.log("Running test %s\n" % testname, True) + + # Read and parse the test file, merge it with the contents of the config + # file, and write the combined output to a temporary file. + f = open(testpath, "r") + testcontent = f.read() + f.close() + # We use yaml to parse the tests because it is a superset of json + # but tolerates things like property names not being quoted, trailing + # commas, etc. + try: + test = yaml.safe_load(testcontent) + except Exception: + test = yaml.safe_load( + testcontent[testcontent.find("{") : testcontent.find("}") + 1] + ) + + self.preferences["tps.seconds_since_epoch"] = int(time.time()) + + # generate the profiles defined in the test, and a list of test phases + profiles = {} + phaselist = [] + for phase in test: + profilename = test[phase] + + # create the profile if necessary + if profilename not in profiles: + profiles[profilename] = Profile( + preferences=self.preferences.copy(), addons=self.extensions + ) + + # create the test phase + phaselist.append( + TPSTestPhase( + phase, + profiles[profilename], + testname, + testpath, + self.logfile, + self.env, + self.firefoxRunner, + self.log, + ignore_unused_engines=self.ignore_unused_engines, + ) + ) + + # sort the phase list by name + phaselist = sorted(phaselist, key=lambda phase: phase.phase) + + # run each phase in sequence, aborting at the first failure + failed = False + for phase in phaselist: + phase.run() + if phase.status != "PASS": + failed = True + break + + for profilename in profiles: + print("### Cleanup Profile ", profilename) + cleanup_phase = TPSTestPhase( + "cleanup-" + profilename, + profiles[profilename], + testname, + testpath, + self.logfile, + self.env, + self.firefoxRunner, + self.log, + ) + + cleanup_phase.run() + if cleanup_phase.status != "PASS": + failed = True + # Keep going to run the remaining cleanup phases. + + if failed: + self.handle_phase_failure(profiles) + + # grep the log for FF and sync versions + f = open(self.logfile) + logdata = f.read() + match = self.syncVerRe.search(logdata) + sync_version = match.group("syncversion") if match else "unknown" + match = self.ffVerRe.search(logdata) + firefox_version = match.group("ffver") if match else "unknown" + match = self.ffBuildIDRe.search(logdata) + firefox_buildid = match.group("ffbuildid") if match else "unknown" + f.close() + if phase.status == "PASS": + logdata = "" + else: + # we only care about the log data for this specific test + logdata = logdata[logdata.find("Running test %s" % (str(testname))) :] + + result = { + "PASS": lambda x: ("TEST-PASS", ""), + "FAIL": lambda x: ("TEST-UNEXPECTED-FAIL", x.rstrip()), + "unknown": lambda x: ("TEST-UNEXPECTED-FAIL", "test did not complete"), + }[phase.status](phase.errline) + logstr = "\n%s | %s%s\n" % ( + result[0], + testname, + (" | %s" % result[1] if result[1] else ""), + ) + + try: + repoinfo = mozversion.get_version(self.binary) + except Exception: + repoinfo = {} + apprepo = repoinfo.get("application_repository", "") + appchangeset = repoinfo.get("application_changeset", "") + + # save logdata to a temporary file for posting to the db + tmplogfile = None + if logdata: + tmplogfile = TempFile(prefix="tps_log_") + tmplogfile.write(logdata.encode("utf-8")) + tmplogfile.close() + self.errorlogs[testname] = tmplogfile + + resultdata = { + "productversion": { + "version": firefox_version, + "buildid": firefox_buildid, + "builddate": firefox_buildid[0:8], + "product": "Firefox", + "repository": apprepo, + "changeset": appchangeset, + }, + "addonversion": {"version": sync_version, "product": "Firefox Sync"}, + "name": testname, + "message": result[1], + "state": result[0], + "logdata": logdata, + } + + self.log(logstr, True) + for phase in phaselist: + print("\t{}: {}".format(phase.phase, phase.status)) + + return resultdata + + def update_preferences(self): + self.preferences = self.default_preferences.copy() + + if self.mobile: + self.preferences.update({"services.sync.client.type": "mobile"}) + + # If we are using legacy Sync, then set a dummy username to force the + # correct authentication type. Without this pref set to a value + # without an '@' character, Sync will initialize for FxA. + if self.config.get("auth_type", "fx_account") != "fx_account": + self.preferences.update({"services.sync.username": "dummy"}) + + if self.debug: + self.preferences.update(self.debug_preferences) + + if "preferences" in self.config: + self.preferences.update(self.config["preferences"]) + + self.preferences["tps.config"] = json.dumps(self.config) + + def run_tests(self): + # delete the logfile if it already exists + if os.access(self.logfile, os.F_OK): + os.remove(self.logfile) + + # Copy the system env variables, and update them for custom settings + self.env = os.environ.copy() + self.env.update(self.extra_env) + + # Update preferences for custom settings + self.update_preferences() + + # Acquire a lock to make sure no other threads are running tests + # at the same time. + if self.rlock: + self.rlock.acquire() + + try: + # Create the Firefox runner, which will download and install the + # build, as needed. + if not self.firefoxRunner: + self.firefoxRunner = TPSFirefoxRunner(self.binary) + + # now, run the test group + self.run_test_group() + + except Exception: + traceback.print_exc() + self.numpassed = 0 + self.numfailed = 1 + try: + self.writeToResultFile( + self.postdata, "<pre>%s</pre>" % traceback.format_exc() + ) + except Exception: + traceback.print_exc() + else: + try: + + self.writeToResultFile(self.postdata) + except Exception: + traceback.print_exc() + try: + self.writeToResultFile( + self.postdata, "<pre>%s</pre>" % traceback.format_exc() + ) + except Exception: + traceback.print_exc() + + # release our lock + if self.rlock: + self.rlock.release() + + # dump out a summary of test results + print("Test Summary\n") + for test in self.postdata.get("tests", {}): + print("{} | {} | {}".format(test["state"], test["name"], test["message"])) + + def run_test_group(self): + self.results = [] + + # reset number of passed/failed tests + self.numpassed = 0 + self.numfailed = 0 + + # build our tps.xpi extension + self.extensions = [] + self.extensions.append(os.path.join(self.extensionDir, "tps")) + + # build the test list + try: + f = open(self.testfile) + jsondata = f.read() + f.close() + testfiles = json.loads(jsondata) + testlist = [] + for (filename, meta) in testfiles["tests"].items(): + skip_reason = meta.get("disabled") + if skip_reason: + print("Skipping test {} - {}".format(filename, skip_reason)) + else: + testlist.append(filename) + except ValueError: + testlist = [os.path.basename(self.testfile)] + testdir = os.path.dirname(self.testfile) + + self.server = server.WebTestHttpd(port=4567, doc_root=testdir) + self.server.start() + + # run each test, and save the results + for test in testlist: + result = self.run_single_test(testdir, test) + + if not self.productversion: + self.productversion = result["productversion"] + if not self.addonversion: + self.addonversion = result["addonversion"] + + self.results.append( + { + "state": result["state"], + "name": result["name"], + "message": result["message"], + "logdata": result["logdata"], + } + ) + if result["state"] == "TEST-PASS": + self.numpassed += 1 + else: + self.numfailed += 1 + if self.stop_on_error: + print( + "\nTest failed with --stop-on-error specified; " + "not running any more tests.\n" + ) + break + + self.server.stop() + + # generate the postdata we'll use to post the results to the db + self.postdata = { + "tests": self.results, + "os": "%s %sbit" % (mozinfo.version, mozinfo.bits), + "testtype": "crossweave", + "productversion": self.productversion, + "addonversion": self.addonversion, + "synctype": self.synctype, + } |