# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import, print_function, unicode_literals from argparse import Namespace from collections import defaultdict import functools import logging import os import six import sys import warnings from mozbuild.base import ( MachCommandBase, MachCommandConditions as conditions, MozbuildObject, ) from mach.decorators import ( CommandArgument, CommandProvider, Command, ) here = os.path.abspath(os.path.dirname(__file__)) ENG_BUILD_REQUIRED = """ The mochitest command requires an engineering build. It may be the case that VARIANT=user or PRODUCTION=1 were set. Try re-building with VARIANT=eng: $ VARIANT=eng ./build.sh There should be an app called 'test-container.gaiamobile.org' located in {}. """.lstrip() SUPPORTED_TESTS_NOT_FOUND = """ The mochitest command could not find any supported tests to run! The following flavors and subsuites were found, but are either not supported on {} builds, or were excluded on the command line: {} Double check the command line you used, and make sure you are running in context of the proper build. To switch build contexts, either run |mach| from the appropriate objdir, or export the correct mozconfig: $ export MOZCONFIG=path/to/mozconfig """.lstrip() TESTS_NOT_FOUND = """ The mochitest command could not find any mochitests under the following test path(s): {} Please check spelling and make sure there are mochitests living there. """.lstrip() SUPPORTED_APPS = ["firefox", "android", "thunderbird"] parser = None class MochitestRunner(MozbuildObject): """Easily run mochitests. This currently contains just the basics for running mochitests. We may want to hook up result parsing, etc. """ def __init__(self, *args, **kwargs): MozbuildObject.__init__(self, *args, **kwargs) # TODO Bug 794506 remove once mach integrates with virtualenv. build_path = os.path.join(self.topobjdir, "build") if build_path not in sys.path: sys.path.append(build_path) self.tests_dir = os.path.join(self.topobjdir, "_tests") self.mochitest_dir = os.path.join(self.tests_dir, "testing", "mochitest") self.bin_dir = os.path.join(self.topobjdir, "dist", "bin") def resolve_tests(self, test_paths, test_objects=None, cwd=None): if test_objects: return test_objects from moztest.resolve import TestResolver resolver = self._spawn(TestResolver) tests = list(resolver.resolve_tests(paths=test_paths, cwd=cwd)) return tests def run_desktop_test(self, context, tests=None, **kwargs): """Runs a mochitest.""" # runtests.py is ambiguous, so we load the file/module manually. if "mochitest" not in sys.modules: import imp path = os.path.join(self.mochitest_dir, "runtests.py") with open(path, "r") as fh: imp.load_module("mochitest", fh, path, (".py", "r", imp.PY_SOURCE)) import mochitest # This is required to make other components happy. Sad, isn't it? os.chdir(self.topobjdir) # Automation installs its own stream handler to stdout. Since we want # all logging to go through us, we just remove their handler. remove_handlers = [ l for l in logging.getLogger().handlers if isinstance(l, logging.StreamHandler) ] for handler in remove_handlers: logging.getLogger().removeHandler(handler) options = Namespace(**kwargs) options.topsrcdir = self.topsrcdir options.topobjdir = self.topobjdir from manifestparser import TestManifest if tests and not options.manifestFile: manifest = TestManifest() manifest.tests.extend(tests) options.manifestFile = manifest # When developing mochitest-plain tests, it's often useful to be able to # refresh the page to pick up modifications. Therefore leave the browser # open if only running a single mochitest-plain test. This behaviour can # be overridden by passing in --keep-open=false. if ( len(tests) == 1 and options.keep_open is None and not options.headless and getattr(options, "flavor", "plain") == "plain" ): options.keep_open = True # We need this to enable colorization of output. self.log_manager.enable_unstructured() result = mochitest.run_test_harness(parser, options) self.log_manager.disable_unstructured() return result def run_android_test(self, context, tests, **kwargs): host_ret = verify_host_bin() if host_ret != 0: return host_ret import imp path = os.path.join(self.mochitest_dir, "runtestsremote.py") with open(path, "r") as fh: imp.load_module("runtestsremote", fh, path, (".py", "r", imp.PY_SOURCE)) import runtestsremote from mozrunner.devices.android_device import get_adb_path if not kwargs["adbPath"]: kwargs["adbPath"] = get_adb_path(self) options = Namespace(**kwargs) from manifestparser import TestManifest if tests and not options.manifestFile: manifest = TestManifest() manifest.tests.extend(tests) options.manifestFile = manifest # Firefox for Android doesn't use e10s if options.app is not None and "geckoview" not in options.app: options.e10s = False print("using e10s=False for non-geckoview app") return runtestsremote.run_test_harness(parser, options) def run_geckoview_junit_test(self, context, **kwargs): host_ret = verify_host_bin() if host_ret != 0: return host_ret import runjunit options = Namespace(**kwargs) return runjunit.run_test_harness(parser, options) # parser def setup_argument_parser(): build_obj = MozbuildObject.from_environment(cwd=here) build_path = os.path.join(build_obj.topobjdir, "build") if build_path not in sys.path: sys.path.append(build_path) mochitest_dir = os.path.join(build_obj.topobjdir, "_tests", "testing", "mochitest") with warnings.catch_warnings(): warnings.simplefilter("ignore") import imp path = os.path.join(build_obj.topobjdir, mochitest_dir, "runtests.py") if not os.path.exists(path): path = os.path.join(here, "runtests.py") with open(path, "r") as fh: imp.load_module("mochitest", fh, path, (".py", "r", imp.PY_SOURCE)) from mochitest_options import MochitestArgumentParser if conditions.is_android(build_obj): # On Android, check for a connected device (and offer to start an # emulator if appropriate) before running tests. This check must # be done in this admittedly awkward place because # MochitestArgumentParser initialization fails if no device is found. from mozrunner.devices.android_device import ( verify_android_device, InstallIntent, ) # verify device and xre verify_android_device(build_obj, install=InstallIntent.NO, xre=True) global parser parser = MochitestArgumentParser() return parser def setup_junit_argument_parser(): build_obj = MozbuildObject.from_environment(cwd=here) build_path = os.path.join(build_obj.topobjdir, "build") if build_path not in sys.path: sys.path.append(build_path) mochitest_dir = os.path.join(build_obj.topobjdir, "_tests", "testing", "mochitest") with warnings.catch_warnings(): warnings.simplefilter("ignore") # runtests.py contains MochitestDesktop, required by runjunit import imp path = os.path.join(build_obj.topobjdir, mochitest_dir, "runtests.py") if not os.path.exists(path): path = os.path.join(here, "runtests.py") with open(path, "r") as fh: imp.load_module("mochitest", fh, path, (".py", "r", imp.PY_SOURCE)) import runjunit from mozrunner.devices.android_device import ( verify_android_device, InstallIntent, ) verify_android_device( build_obj, install=InstallIntent.NO, xre=True, network=True ) global parser parser = runjunit.JunitArgumentParser() return parser def verify_host_bin(): # validate MOZ_HOST_BIN environment variables for Android tests xpcshell_binary = "xpcshell" if os.name == "nt": xpcshell_binary = "xpcshell.exe" MOZ_HOST_BIN = os.environ.get("MOZ_HOST_BIN") if not MOZ_HOST_BIN: print( "environment variable MOZ_HOST_BIN must be set to a directory containing host " "%s" % xpcshell_binary ) return 1 elif not os.path.isdir(MOZ_HOST_BIN): print("$MOZ_HOST_BIN does not specify a directory") return 1 elif not os.path.isfile(os.path.join(MOZ_HOST_BIN, xpcshell_binary)): print("$MOZ_HOST_BIN/%s does not exist" % xpcshell_binary) return 1 return 0 @CommandProvider class MachCommands(MachCommandBase): @Command( "mochitest", category="testing", conditions=[functools.partial(conditions.is_buildapp_in, apps=SUPPORTED_APPS)], description="Run any flavor of mochitest (integration test).", parser=setup_argument_parser, ) def run_mochitest_general( self, flavor=None, test_objects=None, resolve_tests=True, **kwargs ): from mochitest_options import ALL_FLAVORS from mozlog.commandline import setup_logging from mozlog.handlers import StreamHandler from moztest.resolve import get_suite_definition # TODO: This is only strictly necessary while mochitest is using Python # 2 and can be removed once the command is migrated to Python 3. self.activate_virtualenv() buildapp = None for app in SUPPORTED_APPS: if conditions.is_buildapp_in(self, apps=[app]): buildapp = app break flavors = None if flavor: for fname, fobj in six.iteritems(ALL_FLAVORS): if flavor in fobj["aliases"]: if buildapp not in fobj["enabled_apps"]: continue flavors = [fname] break else: flavors = [ f for f, v in six.iteritems(ALL_FLAVORS) if buildapp in v["enabled_apps"] ] from mozbuild.controller.building import BuildDriver self._ensure_state_subdir_exists(".") test_paths = kwargs["test_paths"] kwargs["test_paths"] = [] if kwargs.get("debugger", None): import mozdebug if not mozdebug.get_debugger_info(kwargs.get("debugger")): sys.exit(1) mochitest = self._spawn(MochitestRunner) tests = [] if resolve_tests: tests = mochitest.resolve_tests( test_paths, test_objects, cwd=self._mach_context.cwd ) if not kwargs.get("log"): # Create shared logger format_args = {"level": self._mach_context.settings["test"]["level"]} if len(tests) == 1: format_args["verbose"] = True format_args["compact"] = False default_format = self._mach_context.settings["test"]["format"] kwargs["log"] = setup_logging( "mach-mochitest", kwargs, {default_format: sys.stdout}, format_args ) for handler in kwargs["log"].handlers: if isinstance(handler, StreamHandler): handler.formatter.inner.summary_on_shutdown = True driver = self._spawn(BuildDriver) driver.install_tests() subsuite = kwargs.get("subsuite") if subsuite == "default": kwargs["subsuite"] = None suites = defaultdict(list) is_webrtc_tag_present = False unsupported = set() for test in tests: # Check if we're running a webrtc test so we can enable webrtc # specific test logic later if needed. if "webrtc" in test.get("tags", ""): is_webrtc_tag_present = True # Filter out non-mochitests and unsupported flavors. if test["flavor"] not in ALL_FLAVORS: continue key = (test["flavor"], test.get("subsuite", "")) if test["flavor"] not in flavors: unsupported.add(key) continue if subsuite == "default": # "--subsuite default" means only run tests that don't have a subsuite if test.get("subsuite"): unsupported.add(key) continue elif subsuite and test.get("subsuite", "") != subsuite: unsupported.add(key) continue suites[key].append(test) # Only webrtc mochitests in the media suite need the websocketprocessbridge. if ("mochitest", "media") in suites and is_webrtc_tag_present: req = os.path.join( "testing", "tools", "websocketprocessbridge", "websocketprocessbridge_requirements_3.txt", ) self.virtualenv_manager.activate() self.virtualenv_manager.install_pip_requirements(req, require_hashes=False) # sys.executable is used to start the websocketprocessbridge, though for some # reason it doesn't get set when calling `activate_this.py` in the virtualenv. sys.executable = self.virtualenv_manager.python_path # This is a hack to introduce an option in mach to not send # filtered tests to the mochitest harness. Mochitest harness will read # the master manifest in that case. if not resolve_tests: for flavor in flavors: key = (flavor, kwargs.get("subsuite")) suites[key] = [] if not suites: # Make it very clear why no tests were found if not unsupported: print( TESTS_NOT_FOUND.format( "\n".join(sorted(list(test_paths or test_objects))) ) ) return 1 msg = [] for f, s in unsupported: fobj = ALL_FLAVORS[f] apps = fobj["enabled_apps"] name = fobj["aliases"][0] if s: name = "{} --subsuite {}".format(name, s) if buildapp not in apps: reason = "requires {}".format(" or ".join(apps)) else: reason = "excluded by the command line" msg.append(" mochitest -f {} ({})".format(name, reason)) print(SUPPORTED_TESTS_NOT_FOUND.format(buildapp, "\n".join(sorted(msg)))) return 1 if buildapp == "android": from mozrunner.devices.android_device import ( verify_android_device, InstallIntent, ) app = kwargs.get("app") if not app: app = "org.mozilla.geckoview.test" device_serial = kwargs.get("deviceSerial") install = ( InstallIntent.NO if kwargs.get("no_install") else InstallIntent.YES ) # verify installation verify_android_device( self, install=install, xre=False, network=True, app=app, device_serial=device_serial, ) run_mochitest = mochitest.run_android_test else: run_mochitest = mochitest.run_desktop_test overall = None for (flavor, subsuite), tests in sorted(suites.items()): suite_name, suite = get_suite_definition(flavor, subsuite) if "test_paths" in suite["kwargs"]: del suite["kwargs"]["test_paths"] harness_args = kwargs.copy() harness_args.update(suite["kwargs"]) # Pass in the full suite name as defined in moztest/resolve.py in case # chunk-by-runtime is called, in which case runtime information for # specific mochitest suite has to be loaded. See Bug 1637463. harness_args.update({"suite_name": suite_name}) result = run_mochitest(self._mach_context, tests=tests, **harness_args) if result: overall = result # Halt tests on keyboard interrupt if result == -1: break # Only shutdown the logger if we created it if kwargs["log"].name == "mach-mochitest": kwargs["log"].shutdown() return overall @CommandProvider class GeckoviewJunitCommands(MachCommandBase): @Command( "geckoview-junit", category="testing", conditions=[conditions.is_android], description="Run remote geckoview junit tests.", parser=setup_junit_argument_parser, ) @CommandArgument( "--no-install", help="Do not try to install application on device before " + "running (default: False)", action="store_true", default=False, ) def run_junit(self, no_install, **kwargs): self._ensure_state_subdir_exists(".") from mozrunner.devices.android_device import ( get_adb_path, verify_android_device, InstallIntent, ) # verify installation app = kwargs.get("app") device_serial = kwargs.get("deviceSerial") verify_android_device( self, install=InstallIntent.NO if no_install else InstallIntent.YES, xre=False, app=app, device_serial=device_serial, ) if not kwargs.get("adbPath"): kwargs["adbPath"] = get_adb_path(self) if not kwargs.get("log"): from mozlog.commandline import setup_logging format_args = {"level": self._mach_context.settings["test"]["level"]} default_format = self._mach_context.settings["test"]["format"] kwargs["log"] = setup_logging( "mach-mochitest", kwargs, {default_format: sys.stdout}, format_args ) mochitest = self._spawn(MochitestRunner) return mochitest.run_geckoview_junit_test(self._mach_context, **kwargs)