diff options
Diffstat (limited to 'third_party/libwebrtc/build/android/test_runner.py')
| -rwxr-xr-x | third_party/libwebrtc/build/android/test_runner.py | 1194 |
1 files changed, 1194 insertions, 0 deletions
diff --git a/third_party/libwebrtc/build/android/test_runner.py b/third_party/libwebrtc/build/android/test_runner.py new file mode 100755 index 0000000000..1e312ba0b8 --- /dev/null +++ b/third_party/libwebrtc/build/android/test_runner.py @@ -0,0 +1,1194 @@ +#!/usr/bin/env vpython3 +# +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Runs all types of tests from one unified interface.""" + +from __future__ import absolute_import +import argparse +import collections +import contextlib +import itertools +import logging +import os +import re +import shutil +import signal +import sys +import tempfile +import threading +import traceback +import unittest + +# Import _strptime before threaded code. datetime.datetime.strptime is +# threadsafe except for the initial import of the _strptime module. +# See http://crbug.com/724524 and https://bugs.python.org/issue7980. +import _strptime # pylint: disable=unused-import + +# pylint: disable=ungrouped-imports +from pylib.constants import host_paths + +if host_paths.DEVIL_PATH not in sys.path: + sys.path.append(host_paths.DEVIL_PATH) + +from devil import base_error +from devil.utils import reraiser_thread +from devil.utils import run_tests_helper + +from pylib import constants +from pylib.base import base_test_result +from pylib.base import environment_factory +from pylib.base import output_manager +from pylib.base import output_manager_factory +from pylib.base import test_instance_factory +from pylib.base import test_run_factory +from pylib.results import json_results +from pylib.results import report_results +from pylib.results.presentation import test_results_presentation +from pylib.utils import local_utils +from pylib.utils import logdog_helper +from pylib.utils import logging_utils +from pylib.utils import test_filter + +from py_utils import contextlib_ext + +from lib.results import result_sink # pylint: disable=import-error + +_DEVIL_STATIC_CONFIG_FILE = os.path.abspath(os.path.join( + host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'devil_config.json')) + + +def _RealPath(arg): + if arg.startswith('//'): + arg = os.path.abspath(os.path.join(host_paths.DIR_SOURCE_ROOT, + arg[2:].replace('/', os.sep))) + return os.path.realpath(arg) + + +def AddTestLauncherOptions(parser): + """Adds arguments mirroring //base/test/launcher. + + Args: + parser: The parser to which arguments should be added. + Returns: + The given parser. + """ + parser.add_argument( + '--test-launcher-retry-limit', + '--test_launcher_retry_limit', + '--num_retries', '--num-retries', + '--isolated-script-test-launcher-retry-limit', + dest='num_retries', type=int, default=2, + help='Number of retries for a test before ' + 'giving up (default: %(default)s).') + parser.add_argument( + '--test-launcher-summary-output', + '--json-results-file', + dest='json_results_file', type=os.path.realpath, + help='If set, will dump results in JSON form to the specified file. ' + 'Note that this will also trigger saving per-test logcats to ' + 'logdog.') + parser.add_argument( + '--test-launcher-shard-index', + type=int, default=os.environ.get('GTEST_SHARD_INDEX', 0), + help='Index of the external shard to run.') + parser.add_argument( + '--test-launcher-total-shards', + type=int, default=os.environ.get('GTEST_TOTAL_SHARDS', 1), + help='Total number of external shards.') + + test_filter.AddFilterOptions(parser) + + return parser + + +def AddCommandLineOptions(parser): + """Adds arguments to support passing command-line flags to the device.""" + parser.add_argument( + '--device-flags-file', + type=os.path.realpath, + help='The relative filepath to a file containing ' + 'command-line flags to set on the device') + parser.add_argument( + '--use-apk-under-test-flags-file', + action='store_true', + help='Wether to use the flags file for the apk under test. If set, ' + "the filename will be looked up in the APK's PackageInfo.") + parser.set_defaults(allow_unknown=True) + parser.set_defaults(command_line_flags=None) + + +def AddTracingOptions(parser): + # TODO(shenghuazhang): Move this into AddCommonOptions once it's supported + # for all test types. + parser.add_argument( + '--trace-output', + metavar='FILENAME', type=os.path.realpath, + help='Path to save test_runner trace json output to.') + + parser.add_argument( + '--trace-all', + action='store_true', + help='Whether to trace all function calls.') + + +def AddCommonOptions(parser): + """Adds all common options to |parser|.""" + + default_build_type = os.environ.get('BUILDTYPE', 'Debug') + + debug_or_release_group = parser.add_mutually_exclusive_group() + debug_or_release_group.add_argument( + '--debug', + action='store_const', const='Debug', dest='build_type', + default=default_build_type, + help='If set, run test suites under out/Debug. ' + 'Default is env var BUILDTYPE or Debug.') + debug_or_release_group.add_argument( + '--release', + action='store_const', const='Release', dest='build_type', + help='If set, run test suites under out/Release. ' + 'Default is env var BUILDTYPE or Debug.') + + parser.add_argument( + '--break-on-failure', '--break_on_failure', + dest='break_on_failure', action='store_true', + help='Whether to break on failure.') + + # TODO(jbudorick): Remove this once everything has switched to platform + # mode. + parser.add_argument( + '--enable-platform-mode', + action='store_true', + help='Run the test scripts in platform mode, which ' + 'conceptually separates the test runner from the ' + '"device" (local or remote, real or emulated) on ' + 'which the tests are running. [experimental]') + + parser.add_argument( + '-e', '--environment', + default='local', choices=constants.VALID_ENVIRONMENTS, + help='Test environment to run in (default: %(default)s).') + + parser.add_argument( + '--local-output', + action='store_true', + help='Whether to archive test output locally and generate ' + 'a local results detail page.') + + class FastLocalDevAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + namespace.enable_concurrent_adb = True + namespace.enable_device_cache = True + namespace.extract_test_list_from_filter = True + namespace.local_output = True + namespace.num_retries = 0 + namespace.skip_clear_data = True + + parser.add_argument( + '--fast-local-dev', + type=bool, + nargs=0, + action=FastLocalDevAction, + help='Alias for: --num-retries=0 --enable-device-cache ' + '--enable-concurrent-adb --skip-clear-data ' + '--extract-test-list-from-filter --local-output') + + # TODO(jbudorick): Remove this once downstream bots have switched to + # api.test_results. + parser.add_argument( + '--flakiness-dashboard-server', + dest='flakiness_dashboard_server', + help=argparse.SUPPRESS) + parser.add_argument( + '--gs-results-bucket', + help='Google Storage bucket to upload results to.') + + parser.add_argument( + '--output-directory', + dest='output_directory', type=os.path.realpath, + help='Path to the directory in which build files are' + ' located (must include build type). This will take' + ' precedence over --debug and --release') + parser.add_argument( + '-v', '--verbose', + dest='verbose_count', default=0, action='count', + help='Verbose level (multiple times for more)') + + parser.add_argument( + '--repeat', '--gtest_repeat', '--gtest-repeat', + '--isolated-script-test-repeat', + dest='repeat', type=int, default=0, + help='Number of times to repeat the specified set of tests.') + + # This is currently only implemented for gtests and instrumentation tests. + parser.add_argument( + '--gtest_also_run_disabled_tests', '--gtest-also-run-disabled-tests', + '--isolated-script-test-also-run-disabled-tests', + dest='run_disabled', action='store_true', + help='Also run disabled tests if applicable.') + + # These are currently only implemented for gtests. + parser.add_argument('--isolated-script-test-output', + help='If present, store test results on this path.') + parser.add_argument('--isolated-script-test-perf-output', + help='If present, store chartjson results on this path.') + + AddTestLauncherOptions(parser) + + +def ProcessCommonOptions(args): + """Processes and handles all common options.""" + run_tests_helper.SetLogLevel(args.verbose_count, add_handler=False) + # pylint: disable=redefined-variable-type + if args.verbose_count > 0: + handler = logging_utils.ColorStreamHandler() + else: + handler = logging.StreamHandler(sys.stdout) + # pylint: enable=redefined-variable-type + handler.setFormatter(run_tests_helper.CustomFormatter()) + logging.getLogger().addHandler(handler) + + constants.SetBuildType(args.build_type) + if args.output_directory: + constants.SetOutputDirectory(args.output_directory) + + +def AddDeviceOptions(parser): + """Adds device options to |parser|.""" + + parser = parser.add_argument_group('device arguments') + + parser.add_argument( + '--adb-path', + type=os.path.realpath, + help='Specify the absolute path of the adb binary that ' + 'should be used.') + parser.add_argument('--denylist-file', + type=os.path.realpath, + help='Device denylist file.') + parser.add_argument( + '-d', '--device', nargs='+', + dest='test_devices', + help='Target device(s) for the test suite to run on.') + parser.add_argument( + '--enable-concurrent-adb', + action='store_true', + help='Run multiple adb commands at the same time, even ' + 'for the same device.') + parser.add_argument( + '--enable-device-cache', + action='store_true', + help='Cache device state to disk between runs') + parser.add_argument( + '--skip-clear-data', + action='store_true', + help='Do not wipe app data between tests. Use this to ' + 'speed up local development and never on bots ' + '(increases flakiness)') + parser.add_argument( + '--recover-devices', + action='store_true', + help='Attempt to recover devices prior to the final retry. Warning: ' + 'this will cause all devices to reboot.') + parser.add_argument( + '--tool', + dest='tool', + help='Run the test under a tool ' + '(use --tool help to list them)') + + parser.add_argument( + '--upload-logcats-file', + action='store_true', + dest='upload_logcats_file', + help='Whether to upload logcat file to logdog.') + + logcat_output_group = parser.add_mutually_exclusive_group() + logcat_output_group.add_argument( + '--logcat-output-dir', type=os.path.realpath, + help='If set, will dump logcats recorded during test run to directory. ' + 'File names will be the device ids with timestamps.') + logcat_output_group.add_argument( + '--logcat-output-file', type=os.path.realpath, + help='If set, will merge logcats recorded during test run and dump them ' + 'to the specified file.') + + +def AddEmulatorOptions(parser): + """Adds emulator-specific options to |parser|.""" + parser = parser.add_argument_group('emulator arguments') + + parser.add_argument( + '--avd-config', + type=os.path.realpath, + help='Path to the avd config textpb. ' + '(See //tools/android/avd/proto/ for message definition' + ' and existing textpb files.)') + parser.add_argument( + '--emulator-count', + type=int, + default=1, + help='Number of emulators to use.') + parser.add_argument( + '--emulator-window', + action='store_true', + default=False, + help='Enable graphical window display on the emulator.') + + +def AddGTestOptions(parser): + """Adds gtest options to |parser|.""" + + parser = parser.add_argument_group('gtest arguments') + + parser.add_argument( + '--app-data-file', + action='append', dest='app_data_files', + help='A file path relative to the app data directory ' + 'that should be saved to the host.') + parser.add_argument( + '--app-data-file-dir', + help='Host directory to which app data files will be' + ' saved. Used with --app-data-file.') + parser.add_argument( + '--enable-xml-result-parsing', + action='store_true', help=argparse.SUPPRESS) + parser.add_argument( + '--executable-dist-dir', + type=os.path.realpath, + help="Path to executable's dist directory for native" + " (non-apk) tests.") + parser.add_argument( + '--extract-test-list-from-filter', + action='store_true', + help='When a test filter is specified, and the list of ' + 'tests can be determined from it, skip querying the ' + 'device for the list of all tests. Speeds up local ' + 'development, but is not safe to use on bots (' + 'http://crbug.com/549214') + parser.add_argument( + '--gs-test-artifacts-bucket', + help=('If present, test artifacts will be uploaded to this Google ' + 'Storage bucket.')) + parser.add_argument( + '--render-test-output-dir', + help='If present, store rendering artifacts in this path.') + parser.add_argument( + '--runtime-deps-path', + dest='runtime_deps_path', type=os.path.realpath, + help='Runtime data dependency file from GN.') + parser.add_argument( + '-t', '--shard-timeout', + dest='shard_timeout', type=int, default=120, + help='Timeout to wait for each test (default: %(default)s).') + parser.add_argument( + '--store-tombstones', + dest='store_tombstones', action='store_true', + help='Add tombstones in results if crash.') + parser.add_argument( + '-s', '--suite', + dest='suite_name', nargs='+', metavar='SUITE_NAME', required=True, + help='Executable name of the test suite to run.') + parser.add_argument( + '--test-apk-incremental-install-json', + type=os.path.realpath, + help='Path to install json for the test apk.') + parser.add_argument('--test-launcher-batch-limit', + dest='test_launcher_batch_limit', + type=int, + help='The max number of tests to run in a shard. ' + 'Ignores non-positive ints and those greater than ' + 'MAX_SHARDS') + parser.add_argument( + '-w', '--wait-for-java-debugger', action='store_true', + help='Wait for java debugger to attach before running any application ' + 'code. Also disables test timeouts and sets retries=0.') + parser.add_argument( + '--coverage-dir', + type=os.path.realpath, + help='Directory in which to place all generated coverage files.') + parser.add_argument( + '--use-existing-test-data', + action='store_true', + help='Do not push new files to the device, instead using existing APK ' + 'and test data. Only use when running the same test for multiple ' + 'iterations.') + + +def AddInstrumentationTestOptions(parser): + """Adds Instrumentation test options to |parser|.""" + + parser = parser.add_argument_group('instrumentation arguments') + + parser.add_argument( + '--additional-apk', + action='append', dest='additional_apks', default=[], + type=_RealPath, + help='Additional apk that must be installed on ' + 'the device when the tests are run') + parser.add_argument( + '-A', '--annotation', + dest='annotation_str', + help='Comma-separated list of annotations. Run only tests with any of ' + 'the given annotations. An annotation can be either a key or a ' + 'key-values pair. A test that has no annotation is considered ' + '"SmallTest".') + # TODO(jbudorick): Remove support for name-style APK specification once + # bots are no longer doing it. + parser.add_argument( + '--apk-under-test', + help='Path or name of the apk under test.') + parser.add_argument( + '--module', + action='append', + dest='modules', + help='Specify Android App Bundle modules to install in addition to the ' + 'base module.') + parser.add_argument( + '--fake-module', + action='append', + dest='fake_modules', + help='Specify Android App Bundle modules to fake install in addition to ' + 'the real modules.') + parser.add_argument( + '--additional-locale', + action='append', + dest='additional_locales', + help='Specify locales in addition to the device locale to install splits ' + 'for when --apk-under-test is an Android App Bundle.') + parser.add_argument( + '--coverage-dir', + type=os.path.realpath, + help='Directory in which to place all generated ' + 'Jacoco coverage files.') + parser.add_argument( + '--disable-dalvik-asserts', + dest='set_asserts', action='store_false', default=True, + help='Removes the dalvik.vm.enableassertions property') + parser.add_argument( + '--enable-java-deobfuscation', + action='store_true', + help='Deobfuscate java stack traces in test output and logcat.') + parser.add_argument( + '-E', '--exclude-annotation', + dest='exclude_annotation_str', + help='Comma-separated list of annotations. Exclude tests with these ' + 'annotations.') + parser.add_argument( + '--enable-breakpad-dump', + action='store_true', + help='Stores any breakpad dumps till the end of the test.') + + def package_replacement(arg): + split_arg = arg.split(',') + if len(split_arg) != 2: + raise argparse.ArgumentError( + arg, + 'Expected two comma-separated strings for --replace-system-package, ' + 'received %d' % len(split_arg)) + PackageReplacement = collections.namedtuple('PackageReplacement', + ['package', 'replacement_apk']) + return PackageReplacement(package=split_arg[0], + replacement_apk=_RealPath(split_arg[1])) + parser.add_argument( + '--replace-system-package', + type=package_replacement, default=None, + help='Specifies a system package to replace with a given APK for the ' + 'duration of the test. Given as a comma-separated pair of strings, ' + 'the first element being the package and the second the path to the ' + 'replacement APK. Only supports replacing one package. Example: ' + '--replace-system-package com.example.app,path/to/some.apk') + parser.add_argument( + '--remove-system-package', + default=[], + action='append', + dest='system_packages_to_remove', + help='Specifies a system package to remove before testing if it exists ' + 'on the system. WARNING: THIS WILL PERMANENTLY REMOVE THE SYSTEM APP. ' + 'Unlike --replace-system-package, the app will not be restored after ' + 'tests are finished.') + + parser.add_argument( + '--use-webview-provider', + type=_RealPath, default=None, + help='Use this apk as the webview provider during test. ' + 'The original provider will be restored if possible, ' + "on Nougat the provider can't be determined and so " + 'the system will choose the default provider.') + parser.add_argument( + '--runtime-deps-path', + dest='runtime_deps_path', type=os.path.realpath, + help='Runtime data dependency file from GN.') + parser.add_argument( + '--screenshot-directory', + dest='screenshot_dir', type=os.path.realpath, + help='Capture screenshots of test failures') + parser.add_argument( + '--shared-prefs-file', + dest='shared_prefs_file', type=_RealPath, + help='The relative path to a file containing JSON list of shared ' + 'preference files to edit and how to do so. Example list: ' + '[{' + ' "package": "com.package.example",' + ' "filename": "ExampleSettings.xml",' + ' "set": {' + ' "boolean_key_in_xml": true,' + ' "string_key_in_xml": "string_value"' + ' },' + ' "remove": [' + ' "key_in_xml_to_remove"' + ' ]' + '}]') + parser.add_argument( + '--store-tombstones', + action='store_true', dest='store_tombstones', + help='Add tombstones in results if crash.') + parser.add_argument( + '--strict-mode', + dest='strict_mode', default='testing', + help='StrictMode command-line flag set on the device, ' + 'death/testing to kill the process, off to stop ' + 'checking, flash to flash only. (default: %(default)s)') + parser.add_argument( + '--test-apk', + required=True, + help='Path or name of the apk containing the tests.') + parser.add_argument( + '--test-jar', + help='Path of jar containing test java files.') + parser.add_argument( + '--test-launcher-batch-limit', + dest='test_launcher_batch_limit', + type=int, + help=('Not actually used for instrumentation tests, but can be used as ' + 'a proxy for determining if the current run is a retry without ' + 'patch.')) + parser.add_argument( + '--timeout-scale', + type=float, + help='Factor by which timeouts should be scaled.') + parser.add_argument( + '-w', '--wait-for-java-debugger', action='store_true', + help='Wait for java debugger to attach before running any application ' + 'code. Also disables test timeouts and sets retries=0.') + + # WPR record mode. + parser.add_argument('--wpr-enable-record', + action='store_true', + default=False, + help='If true, WPR server runs in record mode.' + 'otherwise, runs in replay mode.') + + # These arguments are suppressed from the help text because they should + # only ever be specified by an intermediate script. + parser.add_argument( + '--apk-under-test-incremental-install-json', + help=argparse.SUPPRESS) + parser.add_argument( + '--test-apk-incremental-install-json', + type=os.path.realpath, + help=argparse.SUPPRESS) + + +def AddSkiaGoldTestOptions(parser): + """Adds Skia Gold test options to |parser|.""" + parser = parser.add_argument_group("Skia Gold arguments") + parser.add_argument( + '--code-review-system', + help='A non-default code review system to pass to pass to Gold, if ' + 'applicable') + parser.add_argument( + '--continuous-integration-system', + help='A non-default continuous integration system to pass to Gold, if ' + 'applicable') + parser.add_argument( + '--git-revision', help='The git commit currently being tested.') + parser.add_argument( + '--gerrit-issue', + help='The Gerrit issue this test is being run on, if applicable.') + parser.add_argument( + '--gerrit-patchset', + help='The Gerrit patchset this test is being run on, if applicable.') + parser.add_argument( + '--buildbucket-id', + help='The Buildbucket build ID that this test was triggered from, if ' + 'applicable.') + local_group = parser.add_mutually_exclusive_group() + local_group.add_argument( + '--local-pixel-tests', + action='store_true', + default=None, + help='Specifies to run the Skia Gold pixel tests in local mode. When run ' + 'in local mode, uploading to Gold is disabled and traditional ' + 'generated/golden/diff images are output instead of triage links. ' + 'Running in local mode also implies --no-luci-auth. If both this ' + 'and --no-local-pixel-tests are left unset, the test harness will ' + 'attempt to detect whether it is running on a workstation or not ' + 'and set the options accordingly.') + local_group.add_argument( + '--no-local-pixel-tests', + action='store_false', + dest='local_pixel_tests', + help='Specifies to run the Skia Gold pixel tests in non-local (bot) ' + 'mode. When run in this mode, data is actually uploaded to Gold and ' + 'triage links are generated. If both this and --local-pixel-tests ' + 'are left unset, the test harness will attempt to detect whether ' + 'it is running on a workstation or not and set the options ' + 'accordingly.') + parser.add_argument( + '--no-luci-auth', + action='store_true', + default=False, + help="Don't use the serve account provided by LUCI for authentication " + 'with Skia Gold, instead relying on gsutil to be pre-authenticated. ' + 'Meant for testing locally instead of on the bots.') + parser.add_argument( + '--bypass-skia-gold-functionality', + action='store_true', + default=False, + help='Bypass all interaction with Skia Gold, effectively disabling the ' + 'image comparison portion of any tests that use Gold. Only meant to be ' + 'used in case a Gold outage occurs and cannot be fixed quickly.') + + +def AddJUnitTestOptions(parser): + """Adds junit test options to |parser|.""" + + parser = parser.add_argument_group('junit arguments') + + parser.add_argument( + '--coverage-on-the-fly', + action='store_true', + help='Generate coverage data by Jacoco on-the-fly instrumentation.') + parser.add_argument( + '--coverage-dir', type=os.path.realpath, + help='Directory to store coverage info.') + parser.add_argument( + '--package-filter', + help='Filters tests by package.') + parser.add_argument( + '--runner-filter', + help='Filters tests by runner class. Must be fully qualified.') + parser.add_argument( + '--shards', + default=-1, + type=int, + help='Number of shards to run junit tests in parallel on. Only 1 shard ' + 'is supported when test-filter is specified. Values less than 1 will ' + 'use auto select.') + parser.add_argument( + '-s', '--test-suite', required=True, + help='JUnit test suite to run.') + debug_group = parser.add_mutually_exclusive_group() + debug_group.add_argument( + '-w', '--wait-for-java-debugger', action='store_const', const='8701', + dest='debug_socket', help='Alias for --debug-socket=8701') + debug_group.add_argument( + '--debug-socket', + help='Wait for java debugger to attach at specified socket address ' + 'before running any application code. Also disables test timeouts ' + 'and sets retries=0.') + + # These arguments are for Android Robolectric tests. + parser.add_argument( + '--robolectric-runtime-deps-dir', + help='Path to runtime deps for Robolectric.') + parser.add_argument( + '--resource-apk', + required=True, + help='Path to .ap_ containing binary resources for Robolectric.') + + +def AddLinkerTestOptions(parser): + + parser = parser.add_argument_group('linker arguments') + + parser.add_argument( + '--test-apk', + type=os.path.realpath, + help='Path to the linker test APK.') + + +def AddMonkeyTestOptions(parser): + """Adds monkey test options to |parser|.""" + + parser = parser.add_argument_group('monkey arguments') + + parser.add_argument('--browser', + required=True, + choices=list(constants.PACKAGE_INFO.keys()), + metavar='BROWSER', + help='Browser under test.') + parser.add_argument( + '--category', + nargs='*', dest='categories', default=[], + help='A list of allowed categories. Monkey will only visit activities ' + 'that are listed with one of the specified categories.') + parser.add_argument( + '--event-count', + default=10000, type=int, + help='Number of events to generate (default: %(default)s).') + parser.add_argument( + '--seed', + type=int, + help='Seed value for pseudo-random generator. Same seed value generates ' + 'the same sequence of events. Seed is randomized by default.') + parser.add_argument( + '--throttle', + default=100, type=int, + help='Delay between events (ms) (default: %(default)s). ') + + +def AddPythonTestOptions(parser): + + parser = parser.add_argument_group('python arguments') + + parser.add_argument('-s', + '--suite', + dest='suite_name', + metavar='SUITE_NAME', + choices=list(constants.PYTHON_UNIT_TEST_SUITES.keys()), + help='Name of the test suite to run.') + + +def _CreateClassToFileNameDict(test_apk): + """Creates a dict mapping classes to file names from size-info apk.""" + constants.CheckOutputDirectory() + test_apk_size_info = os.path.join(constants.GetOutDirectory(), 'size-info', + os.path.basename(test_apk) + '.jar.info') + + class_to_file_dict = {} + # Some tests such as webview_cts_tests use a separately downloaded apk to run + # tests. This means the apk may not have been built by the system and hence + # no size info file exists. + if not os.path.exists(test_apk_size_info): + logging.debug('Apk size file not found. %s', test_apk_size_info) + return class_to_file_dict + + with open(test_apk_size_info, 'r') as f: + for line in f: + file_class, file_name = line.rstrip().split(',', 1) + # Only want files that are not prebuilt. + if file_name.startswith('../../'): + class_to_file_dict[file_class] = str( + file_name.replace('../../', '//', 1)) + + return class_to_file_dict + + +def _RunPythonTests(args): + """Subcommand of RunTestsCommand which runs python unit tests.""" + suite_vars = constants.PYTHON_UNIT_TEST_SUITES[args.suite_name] + suite_path = suite_vars['path'] + suite_test_modules = suite_vars['test_modules'] + + sys.path = [suite_path] + sys.path + try: + suite = unittest.TestSuite() + suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m) + for m in suite_test_modules) + runner = unittest.TextTestRunner(verbosity=1+args.verbose_count) + return 0 if runner.run(suite).wasSuccessful() else 1 + finally: + sys.path = sys.path[1:] + + +_DEFAULT_PLATFORM_MODE_TESTS = [ + 'gtest', 'instrumentation', 'junit', 'linker', 'monkey' +] + + +def RunTestsCommand(args, result_sink_client=None): + """Checks test type and dispatches to the appropriate function. + + Args: + args: argparse.Namespace object. + result_sink_client: A ResultSinkClient object. + + Returns: + Integer indicated exit code. + + Raises: + Exception: Unknown command name passed in, or an exception from an + individual test runner. + """ + command = args.command + + ProcessCommonOptions(args) + logging.info('command: %s', ' '.join(sys.argv)) + if args.enable_platform_mode or command in _DEFAULT_PLATFORM_MODE_TESTS: + return RunTestsInPlatformMode(args, result_sink_client) + + if command == 'python': + return _RunPythonTests(args) + else: + raise Exception('Unknown test type.') + + +_SUPPORTED_IN_PLATFORM_MODE = [ + # TODO(jbudorick): Add support for more test types. + 'gtest', + 'instrumentation', + 'junit', + 'linker', + 'monkey', +] + + +def RunTestsInPlatformMode(args, result_sink_client=None): + + def infra_error(message): + logging.fatal(message) + sys.exit(constants.INFRA_EXIT_CODE) + + if args.command not in _SUPPORTED_IN_PLATFORM_MODE: + infra_error('%s is not yet supported in platform mode' % args.command) + + ### Set up sigterm handler. + + contexts_to_notify_on_sigterm = [] + def unexpected_sigterm(_signum, _frame): + msg = [ + 'Received SIGTERM. Shutting down.', + ] + for live_thread in threading.enumerate(): + # pylint: disable=protected-access + thread_stack = ''.join(traceback.format_stack( + sys._current_frames()[live_thread.ident])) + msg.extend([ + 'Thread "%s" (ident: %s) is currently running:' % ( + live_thread.name, live_thread.ident), + thread_stack]) + + for context in contexts_to_notify_on_sigterm: + context.ReceivedSigterm() + + infra_error('\n'.join(msg)) + + signal.signal(signal.SIGTERM, unexpected_sigterm) + + ### Set up results handling. + # TODO(jbudorick): Rewrite results handling. + + # all_raw_results is a list of lists of + # base_test_result.TestRunResults objects. Each instance of + # TestRunResults contains all test results produced by a single try, + # while each list of TestRunResults contains all tries in a single + # iteration. + all_raw_results = [] + + # all_iteration_results is a list of base_test_result.TestRunResults + # objects. Each instance of TestRunResults contains the last test + # result for each test run in that iteration. + all_iteration_results = [] + + global_results_tags = set() + + json_file = tempfile.NamedTemporaryFile(delete=False) + json_file.close() + + @contextlib.contextmanager + def json_finalizer(): + try: + yield + finally: + if args.json_results_file and os.path.exists(json_file.name): + shutil.move(json_file.name, args.json_results_file) + elif args.isolated_script_test_output and os.path.exists(json_file.name): + shutil.move(json_file.name, args.isolated_script_test_output) + else: + os.remove(json_file.name) + + @contextlib.contextmanager + def json_writer(): + try: + yield + except Exception: + global_results_tags.add('UNRELIABLE_RESULTS') + raise + finally: + if args.isolated_script_test_output: + interrupted = 'UNRELIABLE_RESULTS' in global_results_tags + json_results.GenerateJsonTestResultFormatFile(all_raw_results, + interrupted, + json_file.name, + indent=2) + else: + json_results.GenerateJsonResultsFile( + all_raw_results, + json_file.name, + global_tags=list(global_results_tags), + indent=2) + + test_class_to_file_name_dict = {} + # Test Location is only supported for instrumentation tests as it + # requires the size-info file. + if test_instance.TestType() == 'instrumentation': + test_class_to_file_name_dict = _CreateClassToFileNameDict(args.test_apk) + + if result_sink_client: + for run in all_raw_results: + for results in run: + for r in results.GetAll(): + # Matches chrome.page_info.PageInfoViewTest#testChromePage + match = re.search(r'^(.+\..+)#', r.GetName()) + test_file_name = test_class_to_file_name_dict.get( + match.group(1)) if match else None + # Some tests put in non utf-8 char as part of the test + # which breaks uploads, so need to decode and re-encode. + log_decoded = r.GetLog() + if isinstance(log_decoded, bytes): + log_decoded = log_decoded.decode('utf-8', 'replace') + result_sink_client.Post(r.GetName(), + r.GetType(), + r.GetDuration(), + log_decoded.encode('utf-8'), + test_file_name, + failure_reason=r.GetFailureReason()) + + @contextlib.contextmanager + def upload_logcats_file(): + try: + yield + finally: + if not args.logcat_output_file: + logging.critical('Cannot upload logcat file: no file specified.') + elif not os.path.exists(args.logcat_output_file): + logging.critical("Cannot upload logcat file: file doesn't exist.") + else: + with open(args.logcat_output_file) as src: + dst = logdog_helper.open_text('unified_logcats') + if dst: + shutil.copyfileobj(src, dst) + dst.close() + logging.critical( + 'Logcat: %s', logdog_helper.get_viewer_url('unified_logcats')) + + + logcats_uploader = contextlib_ext.Optional( + upload_logcats_file(), + 'upload_logcats_file' in args and args.upload_logcats_file) + + ### Set up test objects. + + out_manager = output_manager_factory.CreateOutputManager(args) + env = environment_factory.CreateEnvironment( + args, out_manager, infra_error) + test_instance = test_instance_factory.CreateTestInstance(args, infra_error) + test_run = test_run_factory.CreateTestRun(env, test_instance, infra_error) + + contexts_to_notify_on_sigterm.append(env) + contexts_to_notify_on_sigterm.append(test_run) + + ### Run. + with out_manager, json_finalizer(): + with json_writer(), logcats_uploader, env, test_instance, test_run: + + repetitions = (range(args.repeat + + 1) if args.repeat >= 0 else itertools.count()) + result_counts = collections.defaultdict( + lambda: collections.defaultdict(int)) + iteration_count = 0 + for _ in repetitions: + # raw_results will be populated with base_test_result.TestRunResults by + # test_run.RunTests(). It is immediately added to all_raw_results so + # that in the event of an exception, all_raw_results will already have + # the up-to-date results and those can be written to disk. + raw_results = [] + all_raw_results.append(raw_results) + + test_run.RunTests(raw_results) + if not raw_results: + all_raw_results.pop() + continue + + iteration_results = base_test_result.TestRunResults() + for r in reversed(raw_results): + iteration_results.AddTestRunResults(r) + all_iteration_results.append(iteration_results) + iteration_count += 1 + + for r in iteration_results.GetAll(): + result_counts[r.GetName()][r.GetType()] += 1 + + report_results.LogFull( + results=iteration_results, + test_type=test_instance.TestType(), + test_package=test_run.TestPackage(), + annotation=getattr(args, 'annotations', None), + flakiness_server=getattr(args, 'flakiness_dashboard_server', + None)) + if args.break_on_failure and not iteration_results.DidRunPass(): + break + + if iteration_count > 1: + # display summary results + # only display results for a test if at least one test did not pass + all_pass = 0 + tot_tests = 0 + for test_name in result_counts: + tot_tests += 1 + if any(result_counts[test_name][x] for x in ( + base_test_result.ResultType.FAIL, + base_test_result.ResultType.CRASH, + base_test_result.ResultType.TIMEOUT, + base_test_result.ResultType.UNKNOWN)): + logging.critical( + '%s: %s', + test_name, + ', '.join('%s %s' % (str(result_counts[test_name][i]), i) + for i in base_test_result.ResultType.GetTypes())) + else: + all_pass += 1 + + logging.critical('%s of %s tests passed in all %s runs', + str(all_pass), + str(tot_tests), + str(iteration_count)) + + if (args.local_output or not local_utils.IsOnSwarming() + ) and not args.isolated_script_test_output: + with out_manager.ArchivedTempfile( + 'test_results_presentation.html', + 'test_results_presentation', + output_manager.Datatype.HTML) as results_detail_file: + result_html_string, _, _ = test_results_presentation.result_details( + json_path=json_file.name, + test_name=args.command, + cs_base_url='http://cs.chromium.org', + local_output=True) + results_detail_file.write(result_html_string.encode('utf-8')) + results_detail_file.flush() + logging.critical('TEST RESULTS: %s', results_detail_file.Link()) + + ui_screenshots = test_results_presentation.ui_screenshot_set( + json_file.name) + if ui_screenshots: + with out_manager.ArchivedTempfile( + 'ui_screenshots.json', + 'ui_capture', + output_manager.Datatype.JSON) as ui_screenshot_file: + ui_screenshot_file.write(ui_screenshots) + logging.critical('UI Screenshots: %s', ui_screenshot_file.Link()) + + return (0 if all(r.DidRunPass() for r in all_iteration_results) + else constants.ERROR_EXIT_CODE) + + +def DumpThreadStacks(_signal, _frame): + for thread in threading.enumerate(): + reraiser_thread.LogThreadStack(thread) + + +def main(): + signal.signal(signal.SIGUSR1, DumpThreadStacks) + + parser = argparse.ArgumentParser() + command_parsers = parser.add_subparsers( + title='test types', dest='command') + + subp = command_parsers.add_parser( + 'gtest', + help='googletest-based C++ tests') + AddCommonOptions(subp) + AddDeviceOptions(subp) + AddEmulatorOptions(subp) + AddGTestOptions(subp) + AddTracingOptions(subp) + AddCommandLineOptions(subp) + + subp = command_parsers.add_parser( + 'instrumentation', + help='InstrumentationTestCase-based Java tests') + AddCommonOptions(subp) + AddDeviceOptions(subp) + AddEmulatorOptions(subp) + AddInstrumentationTestOptions(subp) + AddSkiaGoldTestOptions(subp) + AddTracingOptions(subp) + AddCommandLineOptions(subp) + + subp = command_parsers.add_parser( + 'junit', + help='JUnit4-based Java tests') + AddCommonOptions(subp) + AddJUnitTestOptions(subp) + + subp = command_parsers.add_parser( + 'linker', + help='linker tests') + AddCommonOptions(subp) + AddDeviceOptions(subp) + AddEmulatorOptions(subp) + AddLinkerTestOptions(subp) + + subp = command_parsers.add_parser( + 'monkey', + help="tests based on Android's monkey command") + AddCommonOptions(subp) + AddDeviceOptions(subp) + AddEmulatorOptions(subp) + AddMonkeyTestOptions(subp) + + subp = command_parsers.add_parser( + 'python', + help='python tests based on unittest.TestCase') + AddCommonOptions(subp) + AddPythonTestOptions(subp) + + args, unknown_args = parser.parse_known_args() + if unknown_args: + if hasattr(args, 'allow_unknown') and args.allow_unknown: + args.command_line_flags = unknown_args + else: + parser.error('unrecognized arguments: %s' % ' '.join(unknown_args)) + + # --replace-system-package/--remove-system-package has the potential to cause + # issues if --enable-concurrent-adb is set, so disallow that combination. + concurrent_adb_enabled = (hasattr(args, 'enable_concurrent_adb') + and args.enable_concurrent_adb) + replacing_system_packages = (hasattr(args, 'replace_system_package') + and args.replace_system_package) + removing_system_packages = (hasattr(args, 'system_packages_to_remove') + and args.system_packages_to_remove) + if (concurrent_adb_enabled + and (replacing_system_packages or removing_system_packages)): + parser.error('--enable-concurrent-adb cannot be used with either ' + '--replace-system-package or --remove-system-package') + + # --use-webview-provider has the potential to cause issues if + # --enable-concurrent-adb is set, so disallow that combination + if (hasattr(args, 'use_webview_provider') and + hasattr(args, 'enable_concurrent_adb') and args.use_webview_provider and + args.enable_concurrent_adb): + parser.error('--use-webview-provider and --enable-concurrent-adb cannot ' + 'be used together') + + if (getattr(args, 'coverage_on_the_fly', False) + and not getattr(args, 'coverage_dir', '')): + parser.error('--coverage-on-the-fly requires --coverage-dir') + + if (hasattr(args, 'debug_socket') or + (hasattr(args, 'wait_for_java_debugger') and + args.wait_for_java_debugger)): + args.num_retries = 0 + + # Result-sink may not exist in the environment if rdb stream is not enabled. + result_sink_client = result_sink.TryInitClient() + + try: + return RunTestsCommand(args, result_sink_client) + except base_error.BaseError as e: + logging.exception('Error occurred.') + if e.is_infra_error: + return constants.INFRA_EXIT_CODE + return constants.ERROR_EXIT_CODE + except: # pylint: disable=W0702 + logging.exception('Unrecognized error occurred.') + return constants.ERROR_EXIT_CODE + + +if __name__ == '__main__': + sys.exit(main()) |