path: root/third_party/libwebrtc/build/android/test_runner.py

#!/usr/bin/env vpython3
#
# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Runs all types of tests from one unified interface."""

from __future__ import absolute_import
import argparse
import collections
import contextlib
import itertools
import logging
import os
import re
import shutil
import signal
import sys
import tempfile
import threading
import traceback
import unittest

# Import _strptime before threaded code. datetime.datetime.strptime is
# threadsafe except for the initial import of the _strptime module.
# See http://crbug.com/724524 and https://bugs.python.org/issue7980.
import _strptime  # pylint: disable=unused-import

# pylint: disable=ungrouped-imports
from pylib.constants import host_paths

if host_paths.DEVIL_PATH not in sys.path:
  sys.path.append(host_paths.DEVIL_PATH)

from devil import base_error
from devil.utils import reraiser_thread
from devil.utils import run_tests_helper

from pylib import constants
from pylib.base import base_test_result
from pylib.base import environment_factory
from pylib.base import output_manager
from pylib.base import output_manager_factory
from pylib.base import test_instance_factory
from pylib.base import test_run_factory
from pylib.results import json_results
from pylib.results import report_results
from pylib.results.presentation import test_results_presentation
from pylib.utils import local_utils
from pylib.utils import logdog_helper
from pylib.utils import logging_utils
from pylib.utils import test_filter

from py_utils import contextlib_ext

from lib.results import result_sink  # pylint: disable=import-error

_DEVIL_STATIC_CONFIG_FILE = os.path.abspath(os.path.join(
    host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'devil_config.json'))


def _RealPath(arg):
  if arg.startswith('//'):
    arg = os.path.abspath(os.path.join(host_paths.DIR_SOURCE_ROOT,
                                       arg[2:].replace('/', os.sep)))
  return os.path.realpath(arg)


def AddTestLauncherOptions(parser):
  """Adds arguments mirroring //base/test/launcher.

  Args:
    parser: The parser to which arguments should be added.
  Returns:
    The given parser.
  """
  parser.add_argument(
      '--test-launcher-retry-limit',
      '--test_launcher_retry_limit',
      '--num_retries', '--num-retries',
      '--isolated-script-test-launcher-retry-limit',
      dest='num_retries', type=int, default=2,
      help='Number of retries for a test before '
           'giving up (default: %(default)s).')
  parser.add_argument(
      '--test-launcher-summary-output',
      '--json-results-file',
      dest='json_results_file', type=os.path.realpath,
      help='If set, will dump results in JSON form to the specified file. '
           'Note that this will also trigger saving per-test logcats to '
           'logdog.')
  parser.add_argument(
      '--test-launcher-shard-index',
      type=int, default=os.environ.get('GTEST_SHARD_INDEX', 0),
      help='Index of the external shard to run.')
  parser.add_argument(
      '--test-launcher-total-shards',
      type=int, default=os.environ.get('GTEST_TOTAL_SHARDS', 1),
      help='Total number of external shards.')

  test_filter.AddFilterOptions(parser)

  return parser


def AddCommandLineOptions(parser):
  """Adds arguments to support passing command-line flags to the device."""
  parser.add_argument(
      '--device-flags-file',
      type=os.path.realpath,
      help='The relative filepath to a file containing '
           'command-line flags to set on the device')
  parser.add_argument(
      '--use-apk-under-test-flags-file',
      action='store_true',
      help='Wether to use the flags file for the apk under test. If set, '
           "the filename will be looked up in the APK's PackageInfo.")
  parser.set_defaults(allow_unknown=True)
  parser.set_defaults(command_line_flags=None)


def AddTracingOptions(parser):
  # TODO(shenghuazhang): Move this into AddCommonOptions once it's supported
  # for all test types.
  parser.add_argument(
      '--trace-output',
      metavar='FILENAME', type=os.path.realpath,
      help='Path to save test_runner trace json output to.')

  parser.add_argument(
      '--trace-all',
      action='store_true',
      help='Whether to trace all function calls.')


def AddCommonOptions(parser):
  """Adds all common options to |parser|."""

  default_build_type = os.environ.get('BUILDTYPE', 'Debug')

  debug_or_release_group = parser.add_mutually_exclusive_group()
  debug_or_release_group.add_argument(
      '--debug',
      action='store_const', const='Debug', dest='build_type',
      default=default_build_type,
      help='If set, run test suites under out/Debug. '
           'Default is env var BUILDTYPE or Debug.')
  debug_or_release_group.add_argument(
      '--release',
      action='store_const', const='Release', dest='build_type',
      help='If set, run test suites under out/Release. '
           'Default is env var BUILDTYPE or Debug.')

  parser.add_argument(
      '--break-on-failure', '--break_on_failure',
      dest='break_on_failure', action='store_true',
      help='Whether to break on failure.')

  # TODO(jbudorick): Remove this once everything has switched to platform
  # mode.
  parser.add_argument(
      '--enable-platform-mode',
      action='store_true',
      help='Run the test scripts in platform mode, which '
           'conceptually separates the test runner from the '
           '"device" (local or remote, real or emulated) on '
           'which the tests are running. [experimental]')

  parser.add_argument(
      '-e', '--environment',
      default='local', choices=constants.VALID_ENVIRONMENTS,
      help='Test environment to run in (default: %(default)s).')

  parser.add_argument(
      '--local-output',
      action='store_true',
      help='Whether to archive test output locally and generate '
           'a local results detail page.')

  class FastLocalDevAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
      namespace.enable_concurrent_adb = True
      namespace.enable_device_cache = True
      namespace.extract_test_list_from_filter = True
      namespace.local_output = True
      namespace.num_retries = 0
      namespace.skip_clear_data = True

  parser.add_argument(
      '--fast-local-dev',
      type=bool,
      nargs=0,
      action=FastLocalDevAction,
      help='Alias for: --num-retries=0 --enable-device-cache '
      '--enable-concurrent-adb --skip-clear-data '
      '--extract-test-list-from-filter --local-output')

  # TODO(jbudorick): Remove this once downstream bots have switched to
  # api.test_results.
  parser.add_argument(
      '--flakiness-dashboard-server',
      dest='flakiness_dashboard_server',
      help=argparse.SUPPRESS)
  parser.add_argument(
      '--gs-results-bucket',
      help='Google Storage bucket to upload results to.')

  parser.add_argument(
      '--output-directory',
      dest='output_directory', type=os.path.realpath,
      help='Path to the directory in which build files are'
           ' located (must include build type). This will take'
           ' precedence over --debug and --release')
  parser.add_argument(
      '-v', '--verbose',
      dest='verbose_count', default=0, action='count',
      help='Verbose level (multiple times for more)')

  parser.add_argument(
      '--repeat', '--gtest_repeat', '--gtest-repeat',
      '--isolated-script-test-repeat',
      dest='repeat', type=int, default=0,
      help='Number of times to repeat the specified set of tests.')

  # This is currently only implemented for gtests and instrumentation tests.
  parser.add_argument(
      '--gtest_also_run_disabled_tests', '--gtest-also-run-disabled-tests',
      '--isolated-script-test-also-run-disabled-tests',
      dest='run_disabled', action='store_true',
      help='Also run disabled tests if applicable.')

  # These are currently only implemented for gtests.
  parser.add_argument('--isolated-script-test-output',
                      help='If present, store test results on this path.')
  parser.add_argument('--isolated-script-test-perf-output',
                      help='If present, store chartjson results on this path.')

  AddTestLauncherOptions(parser)


def ProcessCommonOptions(args):
  """Processes and handles all common options."""
  run_tests_helper.SetLogLevel(args.verbose_count, add_handler=False)
  # pylint: disable=redefined-variable-type
  if args.verbose_count > 0:
    handler = logging_utils.ColorStreamHandler()
  else:
    handler = logging.StreamHandler(sys.stdout)
  # pylint: enable=redefined-variable-type
  handler.setFormatter(run_tests_helper.CustomFormatter())
  logging.getLogger().addHandler(handler)

  constants.SetBuildType(args.build_type)
  if args.output_directory:
    constants.SetOutputDirectory(args.output_directory)


def AddDeviceOptions(parser):
  """Adds device options to |parser|."""

  parser = parser.add_argument_group('device arguments')

  parser.add_argument(
      '--adb-path',
      type=os.path.realpath,
      help='Specify the absolute path of the adb binary that '
           'should be used.')
  parser.add_argument('--denylist-file',
                      type=os.path.realpath,
                      help='Device denylist file.')
  parser.add_argument(
      '-d', '--device', nargs='+',
      dest='test_devices',
      help='Target device(s) for the test suite to run on.')
  parser.add_argument(
      '--enable-concurrent-adb',
      action='store_true',
      help='Run multiple adb commands at the same time, even '
           'for the same device.')
  parser.add_argument(
      '--enable-device-cache',
      action='store_true',
      help='Cache device state to disk between runs')
  parser.add_argument(
      '--skip-clear-data',
      action='store_true',
      help='Do not wipe app data between tests. Use this to '
           'speed up local development and never on bots '
                     '(increases flakiness)')
  parser.add_argument(
      '--recover-devices',
      action='store_true',
      help='Attempt to recover devices prior to the final retry. Warning: '
           'this will cause all devices to reboot.')
  parser.add_argument(
      '--tool',
      dest='tool',
      help='Run the test under a tool '
           '(use --tool help to list them)')

  parser.add_argument(
      '--upload-logcats-file',
      action='store_true',
      dest='upload_logcats_file',
      help='Whether to upload logcat file to logdog.')

  logcat_output_group = parser.add_mutually_exclusive_group()
  logcat_output_group.add_argument(
      '--logcat-output-dir', type=os.path.realpath,
      help='If set, will dump logcats recorded during test run to directory. '
           'File names will be the device ids with timestamps.')
  logcat_output_group.add_argument(
      '--logcat-output-file', type=os.path.realpath,
      help='If set, will merge logcats recorded during test run and dump them '
           'to the specified file.')


def AddEmulatorOptions(parser):
  """Adds emulator-specific options to |parser|."""
  parser = parser.add_argument_group('emulator arguments')

  parser.add_argument(
      '--avd-config',
      type=os.path.realpath,
      help='Path to the avd config textpb. '
      '(See //tools/android/avd/proto/ for message definition'
      ' and existing textpb files.)')
  parser.add_argument(
      '--emulator-count',
      type=int,
      default=1,
      help='Number of emulators to use.')
  parser.add_argument(
      '--emulator-window',
      action='store_true',
      default=False,
      help='Enable graphical window display on the emulator.')


def AddGTestOptions(parser):
  """Adds gtest options to |parser|."""

  parser = parser.add_argument_group('gtest arguments')

  parser.add_argument(
      '--app-data-file',
      action='append', dest='app_data_files',
      help='A file path relative to the app data directory '
           'that should be saved to the host.')
  parser.add_argument(
      '--app-data-file-dir',
      help='Host directory to which app data files will be'
           ' saved. Used with --app-data-file.')
  parser.add_argument(
      '--enable-xml-result-parsing',
      action='store_true', help=argparse.SUPPRESS)
  parser.add_argument(
      '--executable-dist-dir',
      type=os.path.realpath,
      help="Path to executable's dist directory for native"
           " (non-apk) tests.")
  parser.add_argument(
      '--extract-test-list-from-filter',
      action='store_true',
      help='When a test filter is specified, and the list of '
           'tests can be determined from it, skip querying the '
           'device for the list of all tests. Speeds up local '
           'development, but is not safe to use on bots ('
           'http://crbug.com/549214')
  parser.add_argument(
      '--gs-test-artifacts-bucket',
      help=('If present, test artifacts will be uploaded to this Google '
            'Storage bucket.'))
  parser.add_argument(
      '--render-test-output-dir',
      help='If present, store rendering artifacts in this path.')
  parser.add_argument(
      '--runtime-deps-path',
      dest='runtime_deps_path', type=os.path.realpath,
      help='Runtime data dependency file from GN.')
  parser.add_argument(
      '-t', '--shard-timeout',
      dest='shard_timeout', type=int, default=120,
      help='Timeout to wait for each test (default: %(default)s).')
  parser.add_argument(
      '--store-tombstones',
      dest='store_tombstones', action='store_true',
      help='Add tombstones in results if crash.')
  parser.add_argument(
      '-s', '--suite',
      dest='suite_name', nargs='+', metavar='SUITE_NAME', required=True,
      help='Executable name of the test suite to run.')
  parser.add_argument(
      '--test-apk-incremental-install-json',
      type=os.path.realpath,
      help='Path to install json for the test apk.')
  parser.add_argument('--test-launcher-batch-limit',
                      dest='test_launcher_batch_limit',
                      type=int,
                      help='The max number of tests to run in a shard. '
                      'Ignores non-positive ints and those greater than '
                      'MAX_SHARDS')
  parser.add_argument(
      '-w', '--wait-for-java-debugger', action='store_true',
      help='Wait for java debugger to attach before running any application '
           'code. Also disables test timeouts and sets retries=0.')
  parser.add_argument(
      '--coverage-dir',
      type=os.path.realpath,
      help='Directory in which to place all generated coverage files.')
  parser.add_argument(
      '--use-existing-test-data',
      action='store_true',
      help='Do not push new files to the device, instead using existing APK '
      'and test data. Only use when running the same test for multiple '
      'iterations.')


def AddInstrumentationTestOptions(parser):
  """Adds Instrumentation test options to |parser|."""

  parser = parser.add_argument_group('instrumentation arguments')

  parser.add_argument(
      '--additional-apk',
      action='append', dest='additional_apks', default=[],
      type=_RealPath,
      help='Additional apk that must be installed on '
           'the device when the tests are run')
  parser.add_argument(
      '-A', '--annotation',
      dest='annotation_str',
      help='Comma-separated list of annotations. Run only tests with any of '
           'the given annotations. An annotation can be either a key or a '
           'key-values pair. A test that has no annotation is considered '
           '"SmallTest".')
  # TODO(jbudorick): Remove support for name-style APK specification once
  # bots are no longer doing it.
  parser.add_argument(
      '--apk-under-test',
      help='Path or name of the apk under test.')
  parser.add_argument(
      '--module',
      action='append',
      dest='modules',
      help='Specify Android App Bundle modules to install in addition to the '
      'base module.')
  parser.add_argument(
      '--fake-module',
      action='append',
      dest='fake_modules',
      help='Specify Android App Bundle modules to fake install in addition to '
      'the real modules.')
  parser.add_argument(
      '--additional-locale',
      action='append',
      dest='additional_locales',
      help='Specify locales in addition to the device locale to install splits '
      'for when --apk-under-test is an Android App Bundle.')
  parser.add_argument(
      '--coverage-dir',
      type=os.path.realpath,
      help='Directory in which to place all generated '
      'Jacoco coverage files.')
  parser.add_argument(
      '--disable-dalvik-asserts',
      dest='set_asserts', action='store_false', default=True,
      help='Removes the dalvik.vm.enableassertions property')
  parser.add_argument(
      '--enable-java-deobfuscation',
      action='store_true',
      help='Deobfuscate java stack traces in test output and logcat.')
  parser.add_argument(
      '-E', '--exclude-annotation',
      dest='exclude_annotation_str',
      help='Comma-separated list of annotations. Exclude tests with these '
           'annotations.')
  parser.add_argument(
      '--enable-breakpad-dump',
      action='store_true',
      help='Stores any breakpad dumps till the end of the test.')

  def package_replacement(arg):
    split_arg = arg.split(',')
    if len(split_arg) != 2:
      raise argparse.ArgumentError(
          arg,
          'Expected two comma-separated strings for --replace-system-package, '
          'received %d' % len(split_arg))
    PackageReplacement = collections.namedtuple('PackageReplacement',
                                                ['package', 'replacement_apk'])
    return PackageReplacement(package=split_arg[0],
                              replacement_apk=_RealPath(split_arg[1]))
  parser.add_argument(
      '--replace-system-package',
      type=package_replacement, default=None,
      help='Specifies a system package to replace with a given APK for the '
           'duration of the test. Given as a comma-separated pair of strings, '
           'the first element being the package and the second the path to the '
           'replacement APK. Only supports replacing one package. Example: '
           '--replace-system-package com.example.app,path/to/some.apk')
  parser.add_argument(
      '--remove-system-package',
      default=[],
      action='append',
      dest='system_packages_to_remove',
      help='Specifies a system package to remove before testing if it exists '
      'on the system. WARNING: THIS WILL PERMANENTLY REMOVE THE SYSTEM APP. '
      'Unlike --replace-system-package, the app will not be restored after '
      'tests are finished.')

  parser.add_argument(
      '--use-webview-provider',
      type=_RealPath, default=None,
      help='Use this apk as the webview provider during test. '
           'The original provider will be restored if possible, '
           "on Nougat the provider can't be determined and so "
           'the system will choose the default provider.')
  parser.add_argument(
      '--runtime-deps-path',
      dest='runtime_deps_path', type=os.path.realpath,
      help='Runtime data dependency file from GN.')
  parser.add_argument(
      '--screenshot-directory',
      dest='screenshot_dir', type=os.path.realpath,
      help='Capture screenshots of test failures')
  parser.add_argument(
      '--shared-prefs-file',
      dest='shared_prefs_file', type=_RealPath,
      help='The relative path to a file containing JSON list of shared '
           'preference files to edit and how to do so. Example list: '
           '[{'
           '  "package": "com.package.example",'
           '  "filename": "ExampleSettings.xml",'
           '  "set": {'
           '    "boolean_key_in_xml": true,'
           '    "string_key_in_xml": "string_value"'
           '  },'
           '  "remove": ['
           '    "key_in_xml_to_remove"'
           '  ]'
           '}]')
  parser.add_argument(
      '--store-tombstones',
      action='store_true', dest='store_tombstones',
      help='Add tombstones in results if crash.')
  parser.add_argument(
      '--strict-mode',
      dest='strict_mode', default='testing',
      help='StrictMode command-line flag set on the device, '
           'death/testing to kill the process, off to stop '
           'checking, flash to flash only. (default: %(default)s)')
  parser.add_argument(
      '--test-apk',
      required=True,
      help='Path or name of the apk containing the tests.')
  parser.add_argument(
      '--test-jar',
      help='Path of jar containing test java files.')
  parser.add_argument(
      '--test-launcher-batch-limit',
      dest='test_launcher_batch_limit',
      type=int,
      help=('Not actually used for instrumentation tests, but can be used as '
            'a proxy for determining if the current run is a retry without '
            'patch.'))
  parser.add_argument(
      '--timeout-scale',
      type=float,
      help='Factor by which timeouts should be scaled.')
  parser.add_argument(
      '-w', '--wait-for-java-debugger', action='store_true',
      help='Wait for java debugger to attach before running any application '
           'code. Also disables test timeouts and sets retries=0.')

  # WPR record mode.
  parser.add_argument('--wpr-enable-record',
                      action='store_true',
                      default=False,
                      help='If true, WPR server runs in record mode.'
                      'otherwise, runs in replay mode.')

  # These arguments are suppressed from the help text because they should
  # only ever be specified by an intermediate script.
  parser.add_argument(
      '--apk-under-test-incremental-install-json',
      help=argparse.SUPPRESS)
  parser.add_argument(
      '--test-apk-incremental-install-json',
      type=os.path.realpath,
      help=argparse.SUPPRESS)


def AddSkiaGoldTestOptions(parser):
  """Adds Skia Gold test options to |parser|."""
  parser = parser.add_argument_group("Skia Gold arguments")
  parser.add_argument(
      '--code-review-system',
      help='A non-default code review system to pass to pass to Gold, if '
      'applicable')
  parser.add_argument(
      '--continuous-integration-system',
      help='A non-default continuous integration system to pass to Gold, if '
      'applicable')
  parser.add_argument(
      '--git-revision', help='The git commit currently being tested.')
  parser.add_argument(
      '--gerrit-issue',
      help='The Gerrit issue this test is being run on, if applicable.')
  parser.add_argument(
      '--gerrit-patchset',
      help='The Gerrit patchset this test is being run on, if applicable.')
  parser.add_argument(
      '--buildbucket-id',
      help='The Buildbucket build ID that this test was triggered from, if '
      'applicable.')
  local_group = parser.add_mutually_exclusive_group()
  local_group.add_argument(
      '--local-pixel-tests',
      action='store_true',
      default=None,
      help='Specifies to run the Skia Gold pixel tests in local mode. When run '
      'in local mode, uploading to Gold is disabled and traditional '
      'generated/golden/diff images are output instead of triage links. '
      'Running in local mode also implies --no-luci-auth. If both this '
      'and --no-local-pixel-tests are left unset, the test harness will '
      'attempt to detect whether it is running on a workstation or not '
      'and set the options accordingly.')
  local_group.add_argument(
      '--no-local-pixel-tests',
      action='store_false',
      dest='local_pixel_tests',
      help='Specifies to run the Skia Gold pixel tests in non-local (bot) '
      'mode. When run in this mode, data is actually uploaded to Gold and '
      'triage links are generated. If both this and --local-pixel-tests '
      'are left unset, the test harness will attempt to detect whether '
      'it is running on a workstation or not and set the options '
      'accordingly.')
  parser.add_argument(
      '--no-luci-auth',
      action='store_true',
      default=False,
      help="Don't use the serve account provided by LUCI for authentication "
      'with Skia Gold, instead relying on gsutil to be pre-authenticated. '
      'Meant for testing locally instead of on the bots.')
  parser.add_argument(
      '--bypass-skia-gold-functionality',
      action='store_true',
      default=False,
      help='Bypass all interaction with Skia Gold, effectively disabling the '
      'image comparison portion of any tests that use Gold. Only meant to be '
      'used in case a Gold outage occurs and cannot be fixed quickly.')


def AddJUnitTestOptions(parser):
  """Adds junit test options to |parser|."""

  parser = parser.add_argument_group('junit arguments')

  parser.add_argument(
      '--coverage-on-the-fly',
      action='store_true',
      help='Generate coverage data by Jacoco on-the-fly instrumentation.')
  parser.add_argument(
      '--coverage-dir', type=os.path.realpath,
      help='Directory to store coverage info.')
  parser.add_argument(
      '--package-filter',
      help='Filters tests by package.')
  parser.add_argument(
      '--runner-filter',
      help='Filters tests by runner class. Must be fully qualified.')
  parser.add_argument(
      '--shards',
      default=-1,
      type=int,
      help='Number of shards to run junit tests in parallel on. Only 1 shard '
      'is supported when test-filter is specified. Values less than 1 will '
      'use auto select.')
  parser.add_argument(
      '-s', '--test-suite', required=True,
      help='JUnit test suite to run.')
  debug_group = parser.add_mutually_exclusive_group()
  debug_group.add_argument(
      '-w', '--wait-for-java-debugger', action='store_const', const='8701',
      dest='debug_socket', help='Alias for --debug-socket=8701')
  debug_group.add_argument(
      '--debug-socket',
      help='Wait for java debugger to attach at specified socket address '
           'before running any application code. Also disables test timeouts '
           'and sets retries=0.')

  # These arguments are for Android Robolectric tests.
  parser.add_argument(
      '--robolectric-runtime-deps-dir',
      help='Path to runtime deps for Robolectric.')
  parser.add_argument(
      '--resource-apk',
      required=True,
      help='Path to .ap_ containing binary resources for Robolectric.')


def AddLinkerTestOptions(parser):

  parser = parser.add_argument_group('linker arguments')

  parser.add_argument(
      '--test-apk',
      type=os.path.realpath,
      help='Path to the linker test APK.')


def AddMonkeyTestOptions(parser):
  """Adds monkey test options to |parser|."""

  parser = parser.add_argument_group('monkey arguments')

  parser.add_argument('--browser',
                      required=True,
                      choices=list(constants.PACKAGE_INFO.keys()),
                      metavar='BROWSER',
                      help='Browser under test.')
  parser.add_argument(
      '--category',
      nargs='*', dest='categories', default=[],
      help='A list of allowed categories. Monkey will only visit activities '
           'that are listed with one of the specified categories.')
  parser.add_argument(
      '--event-count',
      default=10000, type=int,
      help='Number of events to generate (default: %(default)s).')
  parser.add_argument(
      '--seed',
      type=int,
      help='Seed value for pseudo-random generator. Same seed value generates '
           'the same sequence of events. Seed is randomized by default.')
  parser.add_argument(
      '--throttle',
      default=100, type=int,
      help='Delay between events (ms) (default: %(default)s). ')


def AddPythonTestOptions(parser):

  parser = parser.add_argument_group('python arguments')

  parser.add_argument('-s',
                      '--suite',
                      dest='suite_name',
                      metavar='SUITE_NAME',
                      choices=list(constants.PYTHON_UNIT_TEST_SUITES.keys()),
                      help='Name of the test suite to run.')


def _CreateClassToFileNameDict(test_apk):
  """Creates a dict mapping classes to file names from size-info apk."""
  constants.CheckOutputDirectory()
  test_apk_size_info = os.path.join(constants.GetOutDirectory(), 'size-info',
                                    os.path.basename(test_apk) + '.jar.info')

  class_to_file_dict = {}
  # Some tests such as webview_cts_tests use a separately downloaded apk to run
  # tests. This means the apk may not have been built by the system and hence
  # no size info file exists.
  if not os.path.exists(test_apk_size_info):
    logging.debug('Apk size file not found. %s', test_apk_size_info)
    return class_to_file_dict

  with open(test_apk_size_info, 'r') as f:
    for line in f:
      file_class, file_name = line.rstrip().split(',', 1)
      # Only want files that are not prebuilt.
      if file_name.startswith('../../'):
        class_to_file_dict[file_class] = str(
            file_name.replace('../../', '//', 1))

  return class_to_file_dict


def _RunPythonTests(args):
  """Subcommand of RunTestsCommand which runs python unit tests."""
  suite_vars = constants.PYTHON_UNIT_TEST_SUITES[args.suite_name]
  suite_path = suite_vars['path']
  suite_test_modules = suite_vars['test_modules']

  sys.path = [suite_path] + sys.path
  try:
    suite = unittest.TestSuite()
    suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m)
                   for m in suite_test_modules)
    runner = unittest.TextTestRunner(verbosity=1+args.verbose_count)
    return 0 if runner.run(suite).wasSuccessful() else 1
  finally:
    sys.path = sys.path[1:]


_DEFAULT_PLATFORM_MODE_TESTS = [
    'gtest', 'instrumentation', 'junit', 'linker', 'monkey'
]


def RunTestsCommand(args, result_sink_client=None):
  """Checks test type and dispatches to the appropriate function.

  Args:
    args: argparse.Namespace object.
    result_sink_client: A ResultSinkClient object.

  Returns:
    Integer indicated exit code.

  Raises:
    Exception: Unknown command name passed in, or an exception from an
        individual test runner.
  """
  command = args.command

  ProcessCommonOptions(args)
  logging.info('command: %s', ' '.join(sys.argv))
  if args.enable_platform_mode or command in _DEFAULT_PLATFORM_MODE_TESTS:
    return RunTestsInPlatformMode(args, result_sink_client)

  if command == 'python':
    return _RunPythonTests(args)
  else:
    raise Exception('Unknown test type.')


_SUPPORTED_IN_PLATFORM_MODE = [
  # TODO(jbudorick): Add support for more test types.
  'gtest',
  'instrumentation',
  'junit',
  'linker',
  'monkey',
]


def RunTestsInPlatformMode(args, result_sink_client=None):

  def infra_error(message):
    logging.fatal(message)
    sys.exit(constants.INFRA_EXIT_CODE)

  if args.command not in _SUPPORTED_IN_PLATFORM_MODE:
    infra_error('%s is not yet supported in platform mode' % args.command)

  ### Set up sigterm handler.

  contexts_to_notify_on_sigterm = []
  def unexpected_sigterm(_signum, _frame):
    msg = [
      'Received SIGTERM. Shutting down.',
    ]
    for live_thread in threading.enumerate():
      # pylint: disable=protected-access
      thread_stack = ''.join(traceback.format_stack(
          sys._current_frames()[live_thread.ident]))
      msg.extend([
        'Thread "%s" (ident: %s) is currently running:' % (
            live_thread.name, live_thread.ident),
        thread_stack])

    for context in contexts_to_notify_on_sigterm:
      context.ReceivedSigterm()

    infra_error('\n'.join(msg))

  signal.signal(signal.SIGTERM, unexpected_sigterm)

  ### Set up results handling.
  # TODO(jbudorick): Rewrite results handling.

  # all_raw_results is a list of lists of
  # base_test_result.TestRunResults objects. Each instance of
  # TestRunResults contains all test results produced by a single try,
  # while each list of TestRunResults contains all tries in a single
  # iteration.
  all_raw_results = []

  # all_iteration_results is a list of base_test_result.TestRunResults
  # objects. Each instance of TestRunResults contains the last test
  # result for each test run in that iteration.
  all_iteration_results = []

  global_results_tags = set()

  json_file = tempfile.NamedTemporaryFile(delete=False)
  json_file.close()

  @contextlib.contextmanager
  def json_finalizer():
    try:
      yield
    finally:
      if args.json_results_file and os.path.exists(json_file.name):
        shutil.move(json_file.name, args.json_results_file)
      elif args.isolated_script_test_output and os.path.exists(json_file.name):
        shutil.move(json_file.name, args.isolated_script_test_output)
      else:
        os.remove(json_file.name)

  @contextlib.contextmanager
  def json_writer():
    try:
      yield
    except Exception:
      global_results_tags.add('UNRELIABLE_RESULTS')
      raise
    finally:
      if args.isolated_script_test_output:
        interrupted = 'UNRELIABLE_RESULTS' in global_results_tags
        json_results.GenerateJsonTestResultFormatFile(all_raw_results,
                                                      interrupted,
                                                      json_file.name,
                                                      indent=2)
      else:
        json_results.GenerateJsonResultsFile(
            all_raw_results,
            json_file.name,
            global_tags=list(global_results_tags),
            indent=2)

      test_class_to_file_name_dict = {}
      # Test Location is only supported for instrumentation tests as it
      # requires the size-info file.
      if test_instance.TestType() == 'instrumentation':
        test_class_to_file_name_dict = _CreateClassToFileNameDict(args.test_apk)

      if result_sink_client:
        for run in all_raw_results:
          for results in run:
            for r in results.GetAll():
              # Matches chrome.page_info.PageInfoViewTest#testChromePage
              match = re.search(r'^(.+\..+)#', r.GetName())
              test_file_name = test_class_to_file_name_dict.get(
                  match.group(1)) if match else None
              # Some tests put in non utf-8 char as part of the test
              # which breaks uploads, so need to decode and re-encode.
              log_decoded = r.GetLog()
              if isinstance(log_decoded, bytes):
                log_decoded = log_decoded.decode('utf-8', 'replace')
              result_sink_client.Post(r.GetName(),
                                      r.GetType(),
                                      r.GetDuration(),
                                      log_decoded.encode('utf-8'),
                                      test_file_name,
                                      failure_reason=r.GetFailureReason())

  @contextlib.contextmanager
  def upload_logcats_file():
    try:
      yield
    finally:
      if not args.logcat_output_file:
        logging.critical('Cannot upload logcat file: no file specified.')
      elif not os.path.exists(args.logcat_output_file):
        logging.critical("Cannot upload logcat file: file doesn't exist.")
      else:
        with open(args.logcat_output_file) as src:
          dst = logdog_helper.open_text('unified_logcats')
          if dst:
            shutil.copyfileobj(src, dst)
            dst.close()
            logging.critical(
                'Logcat: %s', logdog_helper.get_viewer_url('unified_logcats'))


  logcats_uploader = contextlib_ext.Optional(
      upload_logcats_file(),
      'upload_logcats_file' in args and args.upload_logcats_file)

  ### Set up test objects.

  out_manager = output_manager_factory.CreateOutputManager(args)
  env = environment_factory.CreateEnvironment(
      args, out_manager, infra_error)
  test_instance = test_instance_factory.CreateTestInstance(args, infra_error)
  test_run = test_run_factory.CreateTestRun(env, test_instance, infra_error)

  contexts_to_notify_on_sigterm.append(env)
  contexts_to_notify_on_sigterm.append(test_run)

  ### Run.
  with out_manager, json_finalizer():
    with json_writer(), logcats_uploader, env, test_instance, test_run:

      repetitions = (range(args.repeat +
                           1) if args.repeat >= 0 else itertools.count())
      result_counts = collections.defaultdict(
          lambda: collections.defaultdict(int))
      iteration_count = 0
      for _ in repetitions:
        # raw_results will be populated with base_test_result.TestRunResults by
        # test_run.RunTests(). It is immediately added to all_raw_results so
        # that in the event of an exception, all_raw_results will already have
        # the up-to-date results and those can be written to disk.
        raw_results = []
        all_raw_results.append(raw_results)

        test_run.RunTests(raw_results)
        if not raw_results:
          all_raw_results.pop()
          continue

        iteration_results = base_test_result.TestRunResults()
        for r in reversed(raw_results):
          iteration_results.AddTestRunResults(r)
        all_iteration_results.append(iteration_results)
        iteration_count += 1

        for r in iteration_results.GetAll():
          result_counts[r.GetName()][r.GetType()] += 1

        report_results.LogFull(
            results=iteration_results,
            test_type=test_instance.TestType(),
            test_package=test_run.TestPackage(),
            annotation=getattr(args, 'annotations', None),
            flakiness_server=getattr(args, 'flakiness_dashboard_server',
                                     None))
        if args.break_on_failure and not iteration_results.DidRunPass():
          break

      if iteration_count > 1:
        # display summary results
        # only display results for a test if at least one test did not pass
        all_pass = 0
        tot_tests = 0
        for test_name in result_counts:
          tot_tests += 1
          if any(result_counts[test_name][x] for x in (
              base_test_result.ResultType.FAIL,
              base_test_result.ResultType.CRASH,
              base_test_result.ResultType.TIMEOUT,
              base_test_result.ResultType.UNKNOWN)):
            logging.critical(
                '%s: %s',
                test_name,
                ', '.join('%s %s' % (str(result_counts[test_name][i]), i)
                          for i in base_test_result.ResultType.GetTypes()))
          else:
            all_pass += 1

        logging.critical('%s of %s tests passed in all %s runs',
                         str(all_pass),
                         str(tot_tests),
                         str(iteration_count))

    if (args.local_output or not local_utils.IsOnSwarming()
        ) and not args.isolated_script_test_output:
      with out_manager.ArchivedTempfile(
          'test_results_presentation.html',
          'test_results_presentation',
          output_manager.Datatype.HTML) as results_detail_file:
        result_html_string, _, _ = test_results_presentation.result_details(
            json_path=json_file.name,
            test_name=args.command,
            cs_base_url='http://cs.chromium.org',
            local_output=True)
        results_detail_file.write(result_html_string.encode('utf-8'))
        results_detail_file.flush()
      logging.critical('TEST RESULTS: %s', results_detail_file.Link())

      ui_screenshots = test_results_presentation.ui_screenshot_set(
          json_file.name)
      if ui_screenshots:
        with out_manager.ArchivedTempfile(
            'ui_screenshots.json',
            'ui_capture',
            output_manager.Datatype.JSON) as ui_screenshot_file:
          ui_screenshot_file.write(ui_screenshots)
        logging.critical('UI Screenshots: %s', ui_screenshot_file.Link())

  return (0 if all(r.DidRunPass() for r in all_iteration_results)
          else constants.ERROR_EXIT_CODE)


def DumpThreadStacks(_signal, _frame):
  for thread in threading.enumerate():
    reraiser_thread.LogThreadStack(thread)


def main():
  signal.signal(signal.SIGUSR1, DumpThreadStacks)

  parser = argparse.ArgumentParser()
  command_parsers = parser.add_subparsers(
      title='test types', dest='command')

  subp = command_parsers.add_parser(
      'gtest',
      help='googletest-based C++ tests')
  AddCommonOptions(subp)
  AddDeviceOptions(subp)
  AddEmulatorOptions(subp)
  AddGTestOptions(subp)
  AddTracingOptions(subp)
  AddCommandLineOptions(subp)

  subp = command_parsers.add_parser(
      'instrumentation',
      help='InstrumentationTestCase-based Java tests')
  AddCommonOptions(subp)
  AddDeviceOptions(subp)
  AddEmulatorOptions(subp)
  AddInstrumentationTestOptions(subp)
  AddSkiaGoldTestOptions(subp)
  AddTracingOptions(subp)
  AddCommandLineOptions(subp)

  subp = command_parsers.add_parser(
      'junit',
      help='JUnit4-based Java tests')
  AddCommonOptions(subp)
  AddJUnitTestOptions(subp)

  subp = command_parsers.add_parser(
      'linker',
      help='linker tests')
  AddCommonOptions(subp)
  AddDeviceOptions(subp)
  AddEmulatorOptions(subp)
  AddLinkerTestOptions(subp)

  subp = command_parsers.add_parser(
      'monkey',
      help="tests based on Android's monkey command")
  AddCommonOptions(subp)
  AddDeviceOptions(subp)
  AddEmulatorOptions(subp)
  AddMonkeyTestOptions(subp)

  subp = command_parsers.add_parser(
      'python',
      help='python tests based on unittest.TestCase')
  AddCommonOptions(subp)
  AddPythonTestOptions(subp)

  args, unknown_args = parser.parse_known_args()
  if unknown_args:
    if hasattr(args, 'allow_unknown') and args.allow_unknown:
      args.command_line_flags = unknown_args
    else:
      parser.error('unrecognized arguments: %s' % ' '.join(unknown_args))

  # --replace-system-package/--remove-system-package has the potential to cause
  # issues if --enable-concurrent-adb is set, so disallow that combination.
  concurrent_adb_enabled = (hasattr(args, 'enable_concurrent_adb')
                            and args.enable_concurrent_adb)
  replacing_system_packages = (hasattr(args, 'replace_system_package')
                               and args.replace_system_package)
  removing_system_packages = (hasattr(args, 'system_packages_to_remove')
                              and args.system_packages_to_remove)
  if (concurrent_adb_enabled
      and (replacing_system_packages or removing_system_packages)):
    parser.error('--enable-concurrent-adb cannot be used with either '
                 '--replace-system-package or --remove-system-package')

  # --use-webview-provider has the potential to cause issues if
  # --enable-concurrent-adb is set, so disallow that combination
  if (hasattr(args, 'use_webview_provider') and
      hasattr(args, 'enable_concurrent_adb') and args.use_webview_provider and
      args.enable_concurrent_adb):
    parser.error('--use-webview-provider and --enable-concurrent-adb cannot '
                 'be used together')

  if (getattr(args, 'coverage_on_the_fly', False)
      and not getattr(args, 'coverage_dir', '')):
    parser.error('--coverage-on-the-fly requires --coverage-dir')

  if (hasattr(args, 'debug_socket') or
      (hasattr(args, 'wait_for_java_debugger') and
      args.wait_for_java_debugger)):
    args.num_retries = 0

  # Result-sink may not exist in the environment if rdb stream is not enabled.
  result_sink_client = result_sink.TryInitClient()

  try:
    return RunTestsCommand(args, result_sink_client)
  except base_error.BaseError as e:
    logging.exception('Error occurred.')
    if e.is_infra_error:
      return constants.INFRA_EXIT_CODE
    return constants.ERROR_EXIT_CODE
  except: # pylint: disable=W0702
    logging.exception('Unrecognized error occurred.')
    return constants.ERROR_EXIT_CODE


if __name__ == '__main__':
  sys.exit(main())