diff options
Diffstat (limited to 'third_party/libwebrtc/build/chromeos')
| -rw-r--r-- | third_party/libwebrtc/build/chromeos/.style.yapf | 2 | ||||
| -rw-r--r-- | third_party/libwebrtc/build/chromeos/OWNERS | 1 | ||||
| -rw-r--r-- | third_party/libwebrtc/build/chromeos/PRESUBMIT.py | 34 | ||||
| -rwxr-xr-x | third_party/libwebrtc/build/chromeos/gen_skylab_runner.py | 108 | ||||
| -rw-r--r-- | third_party/libwebrtc/build/chromeos/pylintrc | 15 | ||||
| -rwxr-xr-x | third_party/libwebrtc/build/chromeos/test_runner.py | 1044 | ||||
| -rwxr-xr-x | third_party/libwebrtc/build/chromeos/test_runner_test.py | 352 |
7 files changed, 1556 insertions, 0 deletions
diff --git a/third_party/libwebrtc/build/chromeos/.style.yapf b/third_party/libwebrtc/build/chromeos/.style.yapf new file mode 100644 index 0000000000..de0c6a70f3 --- /dev/null +++ b/third_party/libwebrtc/build/chromeos/.style.yapf @@ -0,0 +1,2 @@ +[style] +based_on_style = chromium diff --git a/third_party/libwebrtc/build/chromeos/OWNERS b/third_party/libwebrtc/build/chromeos/OWNERS new file mode 100644 index 0000000000..e1058c853e --- /dev/null +++ b/third_party/libwebrtc/build/chromeos/OWNERS @@ -0,0 +1 @@ +bpastene@chromium.org diff --git a/third_party/libwebrtc/build/chromeos/PRESUBMIT.py b/third_party/libwebrtc/build/chromeos/PRESUBMIT.py new file mode 100644 index 0000000000..0e829278fb --- /dev/null +++ b/third_party/libwebrtc/build/chromeos/PRESUBMIT.py @@ -0,0 +1,34 @@ +# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Presubmit script for build/chromeos/. + +See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts for +details on the presubmit API built into depot_tools. +""" + + +USE_PYTHON3 = True + + +def CommonChecks(input_api, output_api): + results = [] + results += input_api.canned_checks.RunPylint( + input_api, output_api, pylintrc='pylintrc', version='2.6') + tests = input_api.canned_checks.GetUnitTestsInDirectory( + input_api, + output_api, + '.', [r'^.+_test\.py$'], + run_on_python2=False, + run_on_python3=True, + skip_shebang_check=True) + results += input_api.RunTests(tests) + return results + + +def CheckChangeOnUpload(input_api, output_api): + return CommonChecks(input_api, output_api) + + +def CheckChangeOnCommit(input_api, output_api): + return CommonChecks(input_api, output_api) diff --git a/third_party/libwebrtc/build/chromeos/gen_skylab_runner.py b/third_party/libwebrtc/build/chromeos/gen_skylab_runner.py new file mode 100755 index 0000000000..3a5e0c0445 --- /dev/null +++ b/third_party/libwebrtc/build/chromeos/gen_skylab_runner.py @@ -0,0 +1,108 @@ +#!/usr/bin/env vpython3 +# +# Copyright 2021 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import os +import sys + + +class SkylabClientTestTest: + + # The basic shell script for client test run in Skylab. The arguments listed + # here will be fed by autotest at the run time. + # + # * test-launcher-summary-output: the path for the json result. It will be + # assigned by autotest, who will upload it to GCS upon test completion. + # * test-launcher-shard-index: the index for this test run. + # * test-launcher-total-shards: the total test shards. + # * test_args: arbitrary runtime arguments configured in test_suites.pyl, + # attached after '--'. + BASIC_SHELL_SCRIPT = """ +#!/bin/sh + +while [[ $# -gt 0 ]]; do + case "$1" in + --test-launcher-summary-output) + summary_output=$2 + shift 2 + ;; + + --test-launcher-shard-index) + shard_index=$2 + shift 2 + ;; + + --test-launcher-total-shards) + total_shards=$2 + shift 2 + ;; + + --) + test_args=$2 + break + ;; + + *) + break + ;; + esac +done + +if [ ! -d $(dirname $summary_output) ] ; then + mkdir -p $(dirname $summary_output) +fi + +cd `dirname $0` && cd .. + """ + + def __init__(self, args): + self.test_exe = args.test_exe + self.output = args.output + + @property + def suite_name(self): + return self.test_exe + + def build_test_script(self): + # Build the shell script that will be used on the device to invoke the test. + # Stored here as a list of lines. + device_test_script_contents = self.BASIC_SHELL_SCRIPT.split('\n') + + test_invocation = ('LD_LIBRARY_PATH=./ ./%s ' + ' --test-launcher-summary-output=$summary_output' + ' --test-launcher-shard-index=$shard_index' + ' --test-launcher-total-shards=$total_shards' + ' $test_args' % self.test_exe) + + device_test_script_contents.append(test_invocation) + with open(self.output, 'w') as w: + w.write('\n'.join(device_test_script_contents) + '\n') + os.chmod(self.output, 0o755) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + '--test-exe', + type=str, + required=True, + help='Path to test executable to run inside the device.') + parser.add_argument('--verbose', '-v', action='store_true') + parser.add_argument( + '--output', + required=True, + type=str, + help='Path to create the runner script.') + + args = parser.parse_args() + + test = SkylabClientTestTest(args) + test.build_test_script() + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/third_party/libwebrtc/build/chromeos/pylintrc b/third_party/libwebrtc/build/chromeos/pylintrc new file mode 100644 index 0000000000..2a721bf270 --- /dev/null +++ b/third_party/libwebrtc/build/chromeos/pylintrc @@ -0,0 +1,15 @@ +[FORMAT] + +max-line-length=80 + +[MESSAGES CONTROL] + +disable=abstract-class-not-used,bad-continuation,bad-indentation,duplicate-code,fixme,invalid-name,locally-disabled,locally-enabled,missing-docstring,star-args,too-few-public-methods,too-many-arguments,too-many-branches,too-many-instance-attributes,too-many-lines,too-many-locals,too-many-public-methods,too-many-statements,wrong-import-position + +[REPORTS] + +reports=no + +[VARIABLES] + +dummy-variables-rgx=^_.*$|dummy diff --git a/third_party/libwebrtc/build/chromeos/test_runner.py b/third_party/libwebrtc/build/chromeos/test_runner.py new file mode 100755 index 0000000000..cc8a5bd198 --- /dev/null +++ b/third_party/libwebrtc/build/chromeos/test_runner.py @@ -0,0 +1,1044 @@ +#!/usr/bin/env vpython3 +# +# Copyright 2018 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import collections +import json +import logging +import os +import pipes +import re +import shutil +import signal +import socket +import sys +import tempfile +import six + +# The following non-std imports are fetched via vpython. See the list at +# //.vpython +import dateutil.parser # pylint: disable=import-error +import jsonlines # pylint: disable=import-error +import psutil # pylint: disable=import-error + +CHROMIUM_SRC_PATH = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..')) + +# Use the android test-runner's gtest results support library for generating +# output json ourselves. +sys.path.insert(0, os.path.join(CHROMIUM_SRC_PATH, 'build', 'android')) +from pylib.base import base_test_result # pylint: disable=import-error +from pylib.results import json_results # pylint: disable=import-error + +sys.path.insert(0, os.path.join(CHROMIUM_SRC_PATH, 'build', 'util')) +from lib.results import result_sink # pylint: disable=import-error + +if six.PY2: + import subprocess32 as subprocess # pylint: disable=import-error +else: + import subprocess # pylint: disable=import-error,wrong-import-order + +DEFAULT_CROS_CACHE = os.path.abspath( + os.path.join(CHROMIUM_SRC_PATH, 'build', 'cros_cache')) +CHROMITE_PATH = os.path.abspath( + os.path.join(CHROMIUM_SRC_PATH, 'third_party', 'chromite')) +CROS_RUN_TEST_PATH = os.path.abspath( + os.path.join(CHROMITE_PATH, 'bin', 'cros_run_test')) + +LACROS_LAUNCHER_SCRIPT_PATH = os.path.abspath( + os.path.join(CHROMIUM_SRC_PATH, 'build', 'lacros', + 'mojo_connection_lacros_launcher.py')) + +# This is a special hostname that resolves to a different DUT in the lab +# depending on which lab machine you're on. +LAB_DUT_HOSTNAME = 'variable_chromeos_device_hostname' + +SYSTEM_LOG_LOCATIONS = [ + '/home/chronos/crash/', + '/var/log/chrome/', + '/var/log/messages', + '/var/log/ui/', +] + +TAST_DEBUG_DOC = 'https://bit.ly/2LgvIXz' + + +class TestFormatError(Exception): + pass + + +class RemoteTest: + + # This is a basic shell script that can be appended to in order to invoke the + # test on the device. + BASIC_SHELL_SCRIPT = [ + '#!/bin/sh', + + # /home and /tmp are mounted with "noexec" in the device, but some of our + # tools and tests use those dirs as a workspace (eg: vpython downloads + # python binaries to ~/.vpython-root and /tmp/vpython_bootstrap). + # /usr/local/tmp doesn't have this restriction, so change the location of + # the home and temp dirs for the duration of the test. + 'export HOME=/usr/local/tmp', + 'export TMPDIR=/usr/local/tmp', + ] + + def __init__(self, args, unknown_args): + self._additional_args = unknown_args + self._path_to_outdir = args.path_to_outdir + self._test_launcher_summary_output = args.test_launcher_summary_output + self._logs_dir = args.logs_dir + self._use_vm = args.use_vm + self._rdb_client = result_sink.TryInitClient() + + self._retries = 0 + self._timeout = None + self._test_launcher_shard_index = args.test_launcher_shard_index + self._test_launcher_total_shards = args.test_launcher_total_shards + + # The location on disk of a shell script that can be optionally used to + # invoke the test on the device. If it's not set, we assume self._test_cmd + # contains the test invocation. + self._on_device_script = None + + self._test_cmd = [ + CROS_RUN_TEST_PATH, + '--board', + args.board, + '--cache-dir', + args.cros_cache, + ] + if args.use_vm: + self._test_cmd += [ + '--start', + # Don't persist any filesystem changes after the VM shutsdown. + '--copy-on-write', + ] + else: + self._test_cmd += [ + '--device', args.device if args.device else LAB_DUT_HOSTNAME + ] + if args.logs_dir: + for log in SYSTEM_LOG_LOCATIONS: + self._test_cmd += ['--results-src', log] + self._test_cmd += [ + '--results-dest-dir', + os.path.join(args.logs_dir, 'system_logs') + ] + if args.flash: + self._test_cmd += ['--flash'] + if args.public_image: + self._test_cmd += ['--public-image'] + + # This environment variable is set for tests that have been instrumented + # for code coverage. Its incoming value is expected to be a location + # inside a subdirectory of result_dir above. This is converted to an + # absolute path that the vm is able to write to, and passed in the + # --results-src flag to cros_run_vm_test for copying out of the vm before + # its termination. + self._llvm_profile_var = None + if os.environ.get('LLVM_PROFILE_FILE'): + _, llvm_profile_file = os.path.split(os.environ['LLVM_PROFILE_FILE']) + self._llvm_profile_var = '/tmp/profraw/%s' % llvm_profile_file + + # This should make the vm test runner exfil the profiling data. + self._test_cmd += ['--results-src', '/tmp/profraw'] + + self._test_env = setup_env() + + @property + def suite_name(self): + raise NotImplementedError('Child classes need to define suite name.') + + @property + def test_cmd(self): + return self._test_cmd + + def write_test_script_to_disk(self, script_contents): + # Since we're using an on_device_script to invoke the test, we'll need to + # set cwd. + self._test_cmd += [ + '--remote-cmd', + '--cwd', + os.path.relpath(self._path_to_outdir, CHROMIUM_SRC_PATH), + ] + logging.info('Running the following command on the device:') + logging.info('\n%s', '\n'.join(script_contents)) + fd, tmp_path = tempfile.mkstemp(suffix='.sh', dir=self._path_to_outdir) + os.fchmod(fd, 0o755) + with os.fdopen(fd, 'w') as f: + f.write('\n'.join(script_contents) + '\n') + return tmp_path + + def run_test(self): + # Traps SIGTERM and kills all child processes of cros_run_test when it's + # caught. This will allow us to capture logs from the device if a test hangs + # and gets timeout-killed by swarming. See also: + # https://chromium.googlesource.com/infra/luci/luci-py/+/main/appengine/swarming/doc/Bot.md#graceful-termination_aka-the-sigterm-and-sigkill-dance + test_proc = None + + def _kill_child_procs(trapped_signal, _): + logging.warning('Received signal %d. Killing child processes of test.', + trapped_signal) + if not test_proc or not test_proc.pid: + # This shouldn't happen? + logging.error('Test process not running.') + return + for child in psutil.Process(test_proc.pid).children(): + logging.warning('Killing process %s', child) + child.kill() + + signal.signal(signal.SIGTERM, _kill_child_procs) + + for i in range(self._retries + 1): + logging.info('########################################') + logging.info('Test attempt #%d', i) + logging.info('########################################') + test_proc = subprocess.Popen( + self._test_cmd, + stdout=sys.stdout, + stderr=sys.stderr, + env=self._test_env) + try: + test_proc.wait(timeout=self._timeout) + except subprocess.TimeoutExpired: # pylint: disable=no-member + logging.error('Test timed out. Sending SIGTERM.') + # SIGTERM the proc and wait 10s for it to close. + test_proc.terminate() + try: + test_proc.wait(timeout=10) + except subprocess.TimeoutExpired: # pylint: disable=no-member + # If it hasn't closed in 10s, SIGKILL it. + logging.error('Test did not exit in time. Sending SIGKILL.') + test_proc.kill() + test_proc.wait() + logging.info('Test exitted with %d.', test_proc.returncode) + if test_proc.returncode == 0: + break + + self.post_run(test_proc.returncode) + # Allow post_run to override test proc return code. (Useful when the host + # side Tast bin returns 0 even for failed tests.) + return test_proc.returncode + + def post_run(self, return_code): + if self._on_device_script: + os.remove(self._on_device_script) + # Create a simple json results file for a test run. The results will contain + # only one test (suite_name), and will either be a PASS or FAIL depending on + # return_code. + if self._test_launcher_summary_output: + result = ( + base_test_result.ResultType.FAIL + if return_code else base_test_result.ResultType.PASS) + suite_result = base_test_result.BaseTestResult(self.suite_name, result) + run_results = base_test_result.TestRunResults() + run_results.AddResult(suite_result) + with open(self._test_launcher_summary_output, 'w') as f: + json.dump(json_results.GenerateResultsDict([run_results]), f) + if self._rdb_client: + self._rdb_client.Post(self.suite_name, result, None, None, None) + + @staticmethod + def get_artifacts(path): + """Crawls a given directory for file artifacts to attach to a test. + + Args: + path: Path to a directory to search for artifacts. + Returns: + A dict mapping name of the artifact to its absolute filepath. + """ + artifacts = {} + for dirpath, _, filenames in os.walk(path): + for f in filenames: + artifact_path = os.path.join(dirpath, f) + artifacts[os.path.relpath(artifact_path, path)] = { + 'filePath': artifact_path, + } + return artifacts + + +class TastTest(RemoteTest): + + def __init__(self, args, unknown_args): + super().__init__(args, unknown_args) + + self._suite_name = args.suite_name + self._tast_vars = args.tast_vars + self._tests = args.tests + # The CQ passes in '--gtest_filter' when specifying tests to skip. Store it + # here and parse it later to integrate it into Tast executions. + self._gtest_style_filter = args.gtest_filter + self._attr_expr = args.attr_expr + self._should_strip = args.strip_chrome + self._deploy_lacros = args.deploy_lacros + + if self._deploy_lacros and self._should_strip: + raise TestFormatError( + '--strip-chrome is only applicable to ash-chrome because ' + 'lacros-chrome deployment uses --nostrip by default, so it cannot ' + 'be specificed with --deploy-lacros.') + + if not self._llvm_profile_var and not self._logs_dir: + # The host-side Tast bin returns 0 when tests fail, so we need to capture + # and parse its json results to reliably determine if tests fail. + raise TestFormatError( + 'When using the host-side Tast bin, "--logs-dir" must be passed in ' + 'order to parse its results.') + + # If the first test filter is negative, it should be safe to assume all of + # them are, so just test the first filter. + if self._gtest_style_filter and self._gtest_style_filter[0] == '-': + raise TestFormatError('Negative test filters not supported for Tast.') + + @property + def suite_name(self): + return self._suite_name + + def build_test_command(self): + unsupported_args = [ + '--test-launcher-retry-limit', + '--test-launcher-batch-limit', + '--gtest_repeat', + ] + for unsupported_arg in unsupported_args: + if any(arg.startswith(unsupported_arg) for arg in self._additional_args): + logging.info( + '%s not supported for Tast tests. The arg will be ignored.', + unsupported_arg) + self._additional_args = [ + arg for arg in self._additional_args + if not arg.startswith(unsupported_arg) + ] + + # Lacros deployment mounts itself by default. + self._test_cmd.extend([ + '--deploy-lacros', '--lacros-launcher-script', + LACROS_LAUNCHER_SCRIPT_PATH + ] if self._deploy_lacros else ['--deploy', '--mount']) + self._test_cmd += [ + '--build-dir', + os.path.relpath(self._path_to_outdir, CHROMIUM_SRC_PATH) + ] + self._additional_args + + # Coverage tests require some special pre-test setup, so use an + # on_device_script in that case. For all other tests, use cros_run_test's + # built-in '--tast' option. This gives us much better results reporting. + if self._llvm_profile_var: + # Build the shell script that will be used on the device to invoke the + # test. + device_test_script_contents = self.BASIC_SHELL_SCRIPT[:] + device_test_script_contents += [ + 'echo "LLVM_PROFILE_FILE=%s" >> /etc/chrome_dev.conf' % + (self._llvm_profile_var) + ] + + local_test_runner_cmd = ['local_test_runner', '-waituntilready'] + if self._use_vm: + # If we're running tests in VMs, tell the test runner to skip tests that + # aren't compatible. + local_test_runner_cmd.append('-extrauseflags=tast_vm') + if self._attr_expr: + local_test_runner_cmd.append(pipes.quote(self._attr_expr)) + else: + local_test_runner_cmd.extend(self._tests) + device_test_script_contents.append(' '.join(local_test_runner_cmd)) + + self._on_device_script = self.write_test_script_to_disk( + device_test_script_contents) + + self._test_cmd += [ + '--files', + os.path.relpath(self._on_device_script), '--', + './' + os.path.relpath(self._on_device_script, self._path_to_outdir) + ] + else: + # Capture tast's results in the logs dir as well. + if self._logs_dir: + self._test_cmd += [ + '--results-dir', + self._logs_dir, + ] + self._test_cmd += [ + '--tast-total-shards=%d' % self._test_launcher_total_shards, + '--tast-shard-index=%d' % self._test_launcher_shard_index, + ] + # If we're using a test filter, replace the contents of the Tast + # conditional with a long list of "name:test" expressions, one for each + # test in the filter. + if self._gtest_style_filter: + if self._attr_expr or self._tests: + logging.warning( + 'Presence of --gtest_filter will cause the specified Tast expr' + ' or test list to be ignored.') + names = [] + for test in self._gtest_style_filter.split(':'): + names.append('"name:%s"' % test) + self._attr_expr = '(' + ' || '.join(names) + ')' + + if self._attr_expr: + # Don't use pipes.quote() here. Something funky happens with the arg + # as it gets passed down from cros_run_test to tast. (Tast picks up the + # escaping single quotes and complains that the attribute expression + # "must be within parentheses".) + self._test_cmd.append('--tast=%s' % self._attr_expr) + else: + self._test_cmd.append('--tast') + self._test_cmd.extend(self._tests) + + for v in self._tast_vars or []: + self._test_cmd.extend(['--tast-var', v]) + + # Mounting ash-chrome gives it enough disk space to not need stripping, + # but only for one not instrumented with code coverage. + # Lacros uses --nostrip by default, so there is no need to specify. + if not self._deploy_lacros and not self._should_strip: + self._test_cmd.append('--nostrip') + + def post_run(self, return_code): + # If we don't need to parse the host-side Tast tool's results, fall back to + # the parent method's default behavior. + if self._llvm_profile_var: + return super().post_run(return_code) + + tast_results_path = os.path.join(self._logs_dir, 'streamed_results.jsonl') + if not os.path.exists(tast_results_path): + logging.error( + 'Tast results not found at %s. Falling back to generic result ' + 'reporting.', tast_results_path) + return super().post_run(return_code) + + # See the link below for the format of the results: + # https://godoc.org/chromium.googlesource.com/chromiumos/platform/tast.git/src/chromiumos/cmd/tast/run#TestResult + with jsonlines.open(tast_results_path) as reader: + tast_results = collections.deque(reader) + + suite_results = base_test_result.TestRunResults() + for test in tast_results: + errors = test['errors'] + start, end = test['start'], test['end'] + # Use dateutil to parse the timestamps since datetime can't handle + # nanosecond precision. + duration = dateutil.parser.parse(end) - dateutil.parser.parse(start) + # If the duration is negative, Tast has likely reported an incorrect + # duration. See https://issuetracker.google.com/issues/187973541. Round + # up to 0 in that case to avoid confusing RDB. + duration_ms = max(duration.total_seconds() * 1000, 0) + if bool(test['skipReason']): + result = base_test_result.ResultType.SKIP + elif errors: + result = base_test_result.ResultType.FAIL + else: + result = base_test_result.ResultType.PASS + primary_error_message = None + error_log = '' + if errors: + # See the link below for the format of these errors: + # https://source.chromium.org/chromiumos/chromiumos/codesearch/+/main:src/platform/tast/src/chromiumos/tast/cmd/tast/internal/run/resultsjson/resultsjson.go + primary_error_message = errors[0]['reason'] + for err in errors: + error_log += err['stack'] + '\n' + error_log += ( + "\nIf you're unsure why this test failed, consult the steps " + 'outlined in\n%s\n' % TAST_DEBUG_DOC) + base_result = base_test_result.BaseTestResult( + test['name'], result, duration=duration_ms, log=error_log) + suite_results.AddResult(base_result) + self._maybe_handle_perf_results(test['name']) + + if self._rdb_client: + # Walk the contents of the test's "outDir" and atttach any file found + # inside as an RDB 'artifact'. (This could include system logs, screen + # shots, etc.) + artifacts = self.get_artifacts(test['outDir']) + self._rdb_client.Post( + test['name'], + result, + duration_ms, + error_log, + None, + artifacts=artifacts, + failure_reason=primary_error_message) + + if self._rdb_client and self._logs_dir: + # Attach artifacts from the device that don't apply to a single test. + artifacts = self.get_artifacts( + os.path.join(self._logs_dir, 'system_logs')) + artifacts.update( + self.get_artifacts(os.path.join(self._logs_dir, 'crashes'))) + self._rdb_client.ReportInvocationLevelArtifacts(artifacts) + + if self._test_launcher_summary_output: + with open(self._test_launcher_summary_output, 'w') as f: + json.dump(json_results.GenerateResultsDict([suite_results]), f) + + if not suite_results.DidRunPass(): + return 1 + if return_code: + logging.warning( + 'No failed tests found, but exit code of %d was returned from ' + 'cros_run_test.', return_code) + return return_code + return 0 + + def _maybe_handle_perf_results(self, test_name): + """Prepares any perf results from |test_name| for process_perf_results. + + - process_perf_results looks for top level directories containing a + perf_results.json file and a test_results.json file. The directory names + are used as the benchmark names. + - If a perf_results.json or results-chart.json file exists in the + |test_name| results directory, a top level directory is created and the + perf results file is copied to perf_results.json. + - A trivial test_results.json file is also created to indicate that the test + succeeded (this function would not be called otherwise). + - When process_perf_results is run, it will find the expected files in the + named directory and upload the benchmark results. + """ + + perf_results = os.path.join(self._logs_dir, 'tests', test_name, + 'perf_results.json') + # TODO(stevenjb): Remove check for crosbolt results-chart.json file. + if not os.path.exists(perf_results): + perf_results = os.path.join(self._logs_dir, 'tests', test_name, + 'results-chart.json') + if os.path.exists(perf_results): + benchmark_dir = os.path.join(self._logs_dir, test_name) + if not os.path.isdir(benchmark_dir): + os.makedirs(benchmark_dir) + shutil.copyfile(perf_results, + os.path.join(benchmark_dir, 'perf_results.json')) + # process_perf_results.py expects a test_results.json file. + test_results = {'valid': True, 'failures': []} + with open(os.path.join(benchmark_dir, 'test_results.json'), 'w') as out: + json.dump(test_results, out) + + +class GTestTest(RemoteTest): + + # The following list corresponds to paths that should not be copied over to + # the device during tests. In other words, these files are only ever used on + # the host. + _FILE_IGNORELIST = [ + re.compile(r'.*build/android.*'), + re.compile(r'.*build/chromeos.*'), + re.compile(r'.*build/cros_cache.*'), + # The following matches anything under //testing/ that isn't under + # //testing/buildbot/filters/. + re.compile(r'.*testing/(?!buildbot/filters).*'), + re.compile(r'.*third_party/chromite.*'), + ] + + def __init__(self, args, unknown_args): + super().__init__(args, unknown_args) + + self._test_exe = args.test_exe + self._runtime_deps_path = args.runtime_deps_path + self._vpython_dir = args.vpython_dir + + self._on_device_script = None + self._env_vars = args.env_var + self._stop_ui = args.stop_ui + self._trace_dir = args.trace_dir + + @property + def suite_name(self): + return self._test_exe + + def build_test_command(self): + # To keep things easy for us, ensure both types of output locations are + # the same. + if self._test_launcher_summary_output and self._logs_dir: + json_out_dir = os.path.dirname(self._test_launcher_summary_output) or '.' + if os.path.abspath(json_out_dir) != os.path.abspath(self._logs_dir): + raise TestFormatError( + '--test-launcher-summary-output and --logs-dir must point to ' + 'the same directory.') + + if self._test_launcher_summary_output: + result_dir, result_file = os.path.split( + self._test_launcher_summary_output) + # If args.test_launcher_summary_output is a file in cwd, result_dir will + # be an empty string, so replace it with '.' when this is the case so + # cros_run_test can correctly handle it. + if not result_dir: + result_dir = '.' + device_result_file = '/tmp/%s' % result_file + self._test_cmd += [ + '--results-src', + device_result_file, + '--results-dest-dir', + result_dir, + ] + + if self._trace_dir and self._logs_dir: + trace_path = os.path.dirname(self._trace_dir) or '.' + if os.path.abspath(trace_path) != os.path.abspath(self._logs_dir): + raise TestFormatError( + '--trace-dir and --logs-dir must point to the same directory.') + + if self._trace_dir: + trace_path, trace_dirname = os.path.split(self._trace_dir) + device_trace_dir = '/tmp/%s' % trace_dirname + self._test_cmd += [ + '--results-src', + device_trace_dir, + '--results-dest-dir', + trace_path, + ] + + # Build the shell script that will be used on the device to invoke the test. + # Stored here as a list of lines. + device_test_script_contents = self.BASIC_SHELL_SCRIPT[:] + if self._llvm_profile_var: + device_test_script_contents += [ + 'export LLVM_PROFILE_FILE=%s' % self._llvm_profile_var, + ] + + for var_name, var_val in self._env_vars: + device_test_script_contents += ['export %s=%s' % (var_name, var_val)] + + if self._vpython_dir: + vpython_path = os.path.join(self._path_to_outdir, self._vpython_dir, + 'vpython3') + cpython_path = os.path.join(self._path_to_outdir, self._vpython_dir, + 'bin', 'python3') + if not os.path.exists(vpython_path) or not os.path.exists(cpython_path): + raise TestFormatError( + '--vpython-dir must point to a dir with both ' + 'infra/3pp/tools/cpython3 and infra/tools/luci/vpython installed.') + vpython_spec_path = os.path.relpath( + os.path.join(CHROMIUM_SRC_PATH, '.vpython3'), self._path_to_outdir) + # Initialize the vpython cache. This can take 10-20s, and some tests + # can't afford to wait that long on the first invocation. + device_test_script_contents.extend([ + 'export PATH=$PWD/%s:$PWD/%s/bin/:$PATH' % + (self._vpython_dir, self._vpython_dir), + 'vpython3 -vpython-spec %s -vpython-tool install' % + (vpython_spec_path), + ]) + + test_invocation = ('LD_LIBRARY_PATH=./ ./%s --test-launcher-shard-index=%d ' + '--test-launcher-total-shards=%d' % + (self._test_exe, self._test_launcher_shard_index, + self._test_launcher_total_shards)) + if self._test_launcher_summary_output: + test_invocation += ' --test-launcher-summary-output=%s' % ( + device_result_file) + + if self._trace_dir: + device_test_script_contents.extend([ + 'rm -rf %s' % device_trace_dir, + 'su chronos -c -- "mkdir -p %s"' % device_trace_dir, + ]) + test_invocation += ' --trace-dir=%s' % device_trace_dir + + if self._additional_args: + test_invocation += ' %s' % ' '.join(self._additional_args) + + if self._stop_ui: + device_test_script_contents += [ + 'stop ui', + ] + # The UI service on the device owns the chronos user session, so shutting + # it down as chronos kills the entire execution of the test. So we'll have + # to run as root up until the test invocation. + test_invocation = 'su chronos -c -- "%s"' % test_invocation + # And we'll need to chown everything since cros_run_test's "--as-chronos" + # option normally does that for us. + device_test_script_contents.append('chown -R chronos: ../..') + else: + self._test_cmd += [ + # Some tests fail as root, so run as the less privileged user + # 'chronos'. + '--as-chronos', + ] + + device_test_script_contents.append(test_invocation) + + self._on_device_script = self.write_test_script_to_disk( + device_test_script_contents) + + runtime_files = [os.path.relpath(self._on_device_script)] + runtime_files += self._read_runtime_files() + if self._vpython_dir: + # --vpython-dir is relative to the out dir, but --files expects paths + # relative to src dir, so fix the path up a bit. + runtime_files.append( + os.path.relpath( + os.path.abspath( + os.path.join(self._path_to_outdir, self._vpython_dir)), + CHROMIUM_SRC_PATH)) + # TODO(bpastene): Add the vpython spec to the test's runtime deps instead + # of handling it here. + runtime_files.append('.vpython') + + for f in runtime_files: + self._test_cmd.extend(['--files', f]) + + self._test_cmd += [ + '--', + './' + os.path.relpath(self._on_device_script, self._path_to_outdir) + ] + + def _read_runtime_files(self): + if not self._runtime_deps_path: + return [] + + abs_runtime_deps_path = os.path.abspath( + os.path.join(self._path_to_outdir, self._runtime_deps_path)) + with open(abs_runtime_deps_path) as runtime_deps_file: + files = [l.strip() for l in runtime_deps_file if l] + rel_file_paths = [] + for f in files: + rel_file_path = os.path.relpath( + os.path.abspath(os.path.join(self._path_to_outdir, f))) + if not any(regex.match(rel_file_path) for regex in self._FILE_IGNORELIST): + rel_file_paths.append(rel_file_path) + return rel_file_paths + + def post_run(self, _): + if self._on_device_script: + os.remove(self._on_device_script) + + if self._test_launcher_summary_output and self._rdb_client: + if not os.path.exists(self._test_launcher_summary_output): + logging.error('Unable to locate %s in order to upload results to RDB.', + self._test_launcher_summary_output) + return + with open(self._test_launcher_summary_output) as f: + raw_results = json.load(f) + parsed_results = json_results.ParseResultsFromJson(raw_results) + for r in parsed_results: + self._rdb_client.Post( + r.GetName(), + r.GetType(), + r.GetDuration(), + r.GetLog(), + None, + failure_reason=r.GetFailureReason()) + + +def device_test(args, unknown_args): + # cros_run_test has trouble with relative paths that go up directories, + # so cd to src/, which should be the root of all data deps. + os.chdir(CHROMIUM_SRC_PATH) + + # TODO: Remove the above when depot_tool's pylint is updated to include the + # fix to https://github.com/PyCQA/pylint/issues/710. + if args.test_type == 'tast': + test = TastTest(args, unknown_args) + else: + test = GTestTest(args, unknown_args) + + test.build_test_command() + logging.info('Running the following command on the device:') + logging.info(' '.join(test.test_cmd)) + + return test.run_test() + + +def host_cmd(args, cmd_args): + if not cmd_args: + raise TestFormatError('Must specify command to run on the host.') + if args.deploy_chrome and not args.path_to_outdir: + raise TestFormatError( + '--path-to-outdir must be specified if --deploy-chrome is passed.') + + cros_run_test_cmd = [ + CROS_RUN_TEST_PATH, + '--board', + args.board, + '--cache-dir', + os.path.join(CHROMIUM_SRC_PATH, args.cros_cache), + ] + if args.use_vm: + cros_run_test_cmd += [ + '--start', + # Don't persist any filesystem changes after the VM shutsdown. + '--copy-on-write', + ] + else: + cros_run_test_cmd += [ + '--device', args.device if args.device else LAB_DUT_HOSTNAME + ] + if args.verbose: + cros_run_test_cmd.append('--debug') + if args.flash: + cros_run_test_cmd.append('--flash') + if args.public_image: + cros_run_test_cmd += ['--public-image'] + + if args.logs_dir: + for log in SYSTEM_LOG_LOCATIONS: + cros_run_test_cmd += ['--results-src', log] + cros_run_test_cmd += [ + '--results-dest-dir', + os.path.join(args.logs_dir, 'system_logs') + ] + + test_env = setup_env() + if args.deploy_chrome or args.deploy_lacros: + # Mounting ash-chrome gives it enough disk space to not need stripping. + cros_run_test_cmd.extend([ + '--deploy-lacros', '--lacros-launcher-script', + LACROS_LAUNCHER_SCRIPT_PATH + ] if args.deploy_lacros else ['--deploy', '--mount', '--nostrip']) + + cros_run_test_cmd += [ + '--build-dir', + os.path.join(CHROMIUM_SRC_PATH, args.path_to_outdir) + ] + + cros_run_test_cmd += [ + '--host-cmd', + '--', + ] + cmd_args + + logging.info('Running the following command:') + logging.info(' '.join(cros_run_test_cmd)) + + return subprocess.call( + cros_run_test_cmd, stdout=sys.stdout, stderr=sys.stderr, env=test_env) + + +def setup_env(): + """Returns a copy of the current env with some needed vars added.""" + env = os.environ.copy() + # Some chromite scripts expect chromite/bin to be on PATH. + env['PATH'] = env['PATH'] + ':' + os.path.join(CHROMITE_PATH, 'bin') + # deploy_chrome needs a set of GN args used to build chrome to determine if + # certain libraries need to be pushed to the device. It looks for the args via + # an env var. To trigger the default deploying behavior, give it a dummy set + # of args. + # TODO(crbug.com/823996): Make the GN-dependent deps controllable via cmd + # line args. + if not env.get('GN_ARGS'): + env['GN_ARGS'] = 'enable_nacl = true' + if not env.get('USE'): + env['USE'] = 'highdpi' + return env + + +def add_common_args(*parsers): + for parser in parsers: + parser.add_argument('--verbose', '-v', action='store_true') + parser.add_argument( + '--board', type=str, required=True, help='Type of CrOS device.') + parser.add_argument( + '--cros-cache', + type=str, + default=DEFAULT_CROS_CACHE, + help='Path to cros cache.') + parser.add_argument( + '--path-to-outdir', + type=str, + required=True, + help='Path to output directory, all of whose contents will be ' + 'deployed to the device.') + parser.add_argument( + '--runtime-deps-path', + type=str, + help='Runtime data dependency file from GN.') + parser.add_argument( + '--vpython-dir', + type=str, + help='Location on host of a directory containing a vpython binary to ' + 'deploy to the device before the test starts. The location of ' + 'this dir will be added onto PATH in the device. WARNING: The ' + 'arch of the device might not match the arch of the host, so ' + 'avoid using "${platform}" when downloading vpython via CIPD.') + parser.add_argument( + '--logs-dir', + type=str, + dest='logs_dir', + help='Will copy everything under /var/log/ from the device after the ' + 'test into the specified dir.') + # Shard args are parsed here since we might also specify them via env vars. + parser.add_argument( + '--test-launcher-shard-index', + type=int, + default=os.environ.get('GTEST_SHARD_INDEX', 0), + help='Index of the external shard to run.') + parser.add_argument( + '--test-launcher-total-shards', + type=int, + default=os.environ.get('GTEST_TOTAL_SHARDS', 1), + help='Total number of external shards.') + parser.add_argument( + '--flash', + action='store_true', + help='Will flash the device to the current SDK version before running ' + 'the test.') + parser.add_argument( + '--public-image', + action='store_true', + help='Will flash a public "full" image to the device.') + + vm_or_device_group = parser.add_mutually_exclusive_group() + vm_or_device_group.add_argument( + '--use-vm', + action='store_true', + help='Will run the test in the VM instead of a device.') + vm_or_device_group.add_argument( + '--device', + type=str, + help='Hostname (or IP) of device to run the test on. This arg is not ' + 'required if --use-vm is set.') + + +def main(): + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest='test_type') + # Host-side test args. + host_cmd_parser = subparsers.add_parser( + 'host-cmd', + help='Runs a host-side test. Pass the host-side command to run after ' + '"--". If --use-vm is passed, hostname and port for the device ' + 'will be 127.0.0.1:9222.') + host_cmd_parser.set_defaults(func=host_cmd) + host_cmd_parser.add_argument( + '--deploy-chrome', + action='store_true', + help='Will deploy a locally built ash-chrome binary to the device before ' + 'running the host-cmd.') + host_cmd_parser.add_argument( + '--deploy-lacros', + action='store_true', + help='Deploy a lacros-chrome instead of ash-chrome.') + + gtest_parser = subparsers.add_parser( + 'gtest', help='Runs a device-side gtest.') + gtest_parser.set_defaults(func=device_test) + gtest_parser.add_argument( + '--test-exe', + type=str, + required=True, + help='Path to test executable to run inside the device.') + + # GTest args. Some are passed down to the test binary in the device. Others + # are parsed here since they might need tweaking or special handling. + gtest_parser.add_argument( + '--test-launcher-summary-output', + type=str, + help='When set, will pass the same option down to the test and retrieve ' + 'its result file at the specified location.') + gtest_parser.add_argument( + '--stop-ui', + action='store_true', + help='Will stop the UI service in the device before running the test.') + gtest_parser.add_argument( + '--trace-dir', + type=str, + help='When set, will pass down to the test to generate the trace and ' + 'retrieve the trace files to the specified location.') + gtest_parser.add_argument( + '--env-var', + nargs=2, + action='append', + default=[], + help='Env var to set on the device for the duration of the test. ' + 'Expected format is "--env-var SOME_VAR_NAME some_var_value". Specify ' + 'multiple times for more than one var.') + + # Tast test args. + # pylint: disable=line-too-long + tast_test_parser = subparsers.add_parser( + 'tast', + help='Runs a device-side set of Tast tests. For more details, see: ' + 'https://chromium.googlesource.com/chromiumos/platform/tast/+/main/docs/running_tests.md' + ) + tast_test_parser.set_defaults(func=device_test) + tast_test_parser.add_argument( + '--suite-name', + type=str, + required=True, + help='Name to apply to the set of Tast tests to run. This has no effect ' + 'on what is executed, but is used mainly for test results reporting ' + 'and tracking (eg: flakiness dashboard).') + tast_test_parser.add_argument( + '--test-launcher-summary-output', + type=str, + help='Generates a simple GTest-style JSON result file for the test run.') + tast_test_parser.add_argument( + '--attr-expr', + type=str, + help='A boolean expression whose matching tests will run ' + '(eg: ("dep:chrome")).') + tast_test_parser.add_argument( + '--strip-chrome', + action='store_true', + help='Strips symbols from ash-chrome before deploying to the device.') + tast_test_parser.add_argument( + '--deploy-lacros', + action='store_true', + help='Deploy a lacros-chrome instead of ash-chrome.') + tast_test_parser.add_argument( + '--tast-var', + action='append', + dest='tast_vars', + help='Runtime variables for Tast tests, and the format are expected to ' + 'be "key=value" pairs.') + tast_test_parser.add_argument( + '--test', + '-t', + action='append', + dest='tests', + help='A Tast test to run in the device (eg: "login.Chrome").') + tast_test_parser.add_argument( + '--gtest_filter', + type=str, + help="Similar to GTest's arg of the same name, this will filter out the " + "specified tests from the Tast run. However, due to the nature of Tast's " + 'cmd-line API, this will overwrite the value(s) of "--test" above.') + + add_common_args(gtest_parser, tast_test_parser, host_cmd_parser) + + args = sys.argv[1:] + unknown_args = [] + # If a '--' is present in the args, treat everything to the right of it as + # args to the test and everything to the left as args to this test runner. + # Otherwise treat all known args as args to this test runner and all unknown + # args as test args. + if '--' in args: + unknown_args = args[args.index('--') + 1:] + args = args[0:args.index('--')] + if unknown_args: + args = parser.parse_args(args=args) + else: + args, unknown_args = parser.parse_known_args() + + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARN) + + if not args.use_vm and not args.device: + logging.warning( + 'The test runner is now assuming running in the lab environment, if ' + 'this is unintentional, please re-invoke the test runner with the ' + '"--use-vm" arg if using a VM, otherwise use the "--device=<DUT>" arg ' + 'to specify a DUT.') + + # If we're not running on a VM, but haven't specified a hostname, assume + # we're on a lab bot and are trying to run a test on a lab DUT. See if the + # magic lab DUT hostname resolves to anything. (It will in the lab and will + # not on dev machines.) + try: + socket.getaddrinfo(LAB_DUT_HOSTNAME, None) + except socket.gaierror: + logging.error('The default lab DUT hostname of %s is unreachable.', + LAB_DUT_HOSTNAME) + return 1 + + if args.flash and args.public_image: + # The flashing tools depend on being unauthenticated with GS when flashing + # public images, so make sure the env var GS uses to locate its creds is + # unset in that case. + os.environ.pop('BOTO_CONFIG', None) + + return args.func(args, unknown_args) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/third_party/libwebrtc/build/chromeos/test_runner_test.py b/third_party/libwebrtc/build/chromeos/test_runner_test.py new file mode 100755 index 0000000000..0fa3a511e7 --- /dev/null +++ b/third_party/libwebrtc/build/chromeos/test_runner_test.py @@ -0,0 +1,352 @@ +#!/usr/bin/env vpython3 +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import json +import os +import shutil +import sys +import tempfile +import unittest +import six + +# The following non-std imports are fetched via vpython. See the list at +# //.vpython +import mock # pylint: disable=import-error +from parameterized import parameterized # pylint: disable=import-error + +import test_runner + +_TAST_TEST_RESULTS_JSON = { + "name": "login.Chrome", + "errors": None, + "start": "2020-01-01T15:41:30.799228462-08:00", + "end": "2020-01-01T15:41:53.318914698-08:00", + "skipReason": "" +} + + +class TestRunnerTest(unittest.TestCase): + + def setUp(self): + self._tmp_dir = tempfile.mkdtemp() + self.mock_rdb = mock.patch.object( + test_runner.result_sink, 'TryInitClient', return_value=None) + self.mock_rdb.start() + + def tearDown(self): + shutil.rmtree(self._tmp_dir, ignore_errors=True) + self.mock_rdb.stop() + + def safeAssertItemsEqual(self, list1, list2): + """A Py3 safe version of assertItemsEqual. + + See https://bugs.python.org/issue17866. + """ + if six.PY3: + self.assertSetEqual(set(list1), set(list2)) + else: + self.assertCountEqual(list1, list2) + + +class TastTests(TestRunnerTest): + + def get_common_tast_args(self, use_vm): + return [ + 'script_name', + 'tast', + '--suite-name=chrome_all_tast_tests', + '--board=eve', + '--flash', + '--path-to-outdir=out_eve/Release', + '--logs-dir=%s' % self._tmp_dir, + '--use-vm' if use_vm else '--device=localhost:2222', + ] + + def get_common_tast_expectations(self, use_vm, is_lacros=False): + expectation = [ + test_runner.CROS_RUN_TEST_PATH, + '--board', + 'eve', + '--cache-dir', + test_runner.DEFAULT_CROS_CACHE, + '--results-dest-dir', + '%s/system_logs' % self._tmp_dir, + '--flash', + '--build-dir', + 'out_eve/Release', + '--results-dir', + self._tmp_dir, + '--tast-total-shards=1', + '--tast-shard-index=0', + ] + expectation.extend(['--start', '--copy-on-write'] + if use_vm else ['--device', 'localhost:2222']) + for p in test_runner.SYSTEM_LOG_LOCATIONS: + expectation.extend(['--results-src', p]) + + if not is_lacros: + expectation += [ + '--mount', + '--deploy', + '--nostrip', + ] + return expectation + + def test_tast_gtest_filter(self): + """Tests running tast tests with a gtest-style filter.""" + with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f: + json.dump(_TAST_TEST_RESULTS_JSON, f) + + args = self.get_common_tast_args(False) + [ + '--attr-expr=( "group:mainline" && "dep:chrome" && !informational)', + '--gtest_filter=login.Chrome:ui.WindowControl', + ] + with mock.patch.object(sys, 'argv', args),\ + mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen: + mock_popen.return_value.returncode = 0 + + test_runner.main() + # The gtest filter should cause the Tast expr to be replaced with a list + # of the tests in the filter. + expected_cmd = self.get_common_tast_expectations(False) + [ + '--tast=("name:login.Chrome" || "name:ui.WindowControl")' + ] + + self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0]) + + @parameterized.expand([ + [True], + [False], + ]) + def test_tast_attr_expr(self, use_vm): + """Tests running a tast tests specified by an attribute expression.""" + with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f: + json.dump(_TAST_TEST_RESULTS_JSON, f) + + args = self.get_common_tast_args(use_vm) + [ + '--attr-expr=( "group:mainline" && "dep:chrome" && !informational)', + ] + with mock.patch.object(sys, 'argv', args),\ + mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen: + mock_popen.return_value.returncode = 0 + + test_runner.main() + expected_cmd = self.get_common_tast_expectations(use_vm) + [ + '--tast=( "group:mainline" && "dep:chrome" && !informational)', + ] + + self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0]) + + @parameterized.expand([ + [True], + [False], + ]) + def test_tast_lacros(self, use_vm): + """Tests running a tast tests for Lacros.""" + with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f: + json.dump(_TAST_TEST_RESULTS_JSON, f) + + args = self.get_common_tast_args(use_vm) + [ + '-t=lacros.Basic', + '--deploy-lacros', + ] + + with mock.patch.object(sys, 'argv', args),\ + mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen: + mock_popen.return_value.returncode = 0 + + test_runner.main() + expected_cmd = self.get_common_tast_expectations( + use_vm, is_lacros=True) + [ + '--tast', + 'lacros.Basic', + '--deploy-lacros', + '--lacros-launcher-script', + test_runner.LACROS_LAUNCHER_SCRIPT_PATH, + ] + + self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0]) + + @parameterized.expand([ + [True], + [False], + ]) + def test_tast_with_vars(self, use_vm): + """Tests running a tast tests with runtime variables.""" + with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f: + json.dump(_TAST_TEST_RESULTS_JSON, f) + + args = self.get_common_tast_args(use_vm) + [ + '-t=login.Chrome', + '--tast-var=key=value', + ] + with mock.patch.object(sys, 'argv', args),\ + mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen: + mock_popen.return_value.returncode = 0 + test_runner.main() + expected_cmd = self.get_common_tast_expectations(use_vm) + [ + '--tast', 'login.Chrome', '--tast-var', 'key=value' + ] + + self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0]) + + @parameterized.expand([ + [True], + [False], + ]) + def test_tast(self, use_vm): + """Tests running a tast tests.""" + with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f: + json.dump(_TAST_TEST_RESULTS_JSON, f) + + args = self.get_common_tast_args(use_vm) + [ + '-t=login.Chrome', + ] + with mock.patch.object(sys, 'argv', args),\ + mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen: + mock_popen.return_value.returncode = 0 + + test_runner.main() + expected_cmd = self.get_common_tast_expectations(use_vm) + [ + '--tast', 'login.Chrome' + ] + + self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0]) + + +class GTestTest(TestRunnerTest): + + @parameterized.expand([ + [True], + [False], + ]) + def test_gtest(self, use_vm): + """Tests running a gtest.""" + fd_mock = mock.mock_open() + + args = [ + 'script_name', + 'gtest', + '--test-exe=out_eve/Release/base_unittests', + '--board=eve', + '--path-to-outdir=out_eve/Release', + '--use-vm' if use_vm else '--device=localhost:2222', + ] + with mock.patch.object(sys, 'argv', args),\ + mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen,\ + mock.patch.object(os, 'fdopen', fd_mock),\ + mock.patch.object(os, 'remove') as mock_remove,\ + mock.patch.object(tempfile, 'mkstemp', + return_value=(3, 'out_eve/Release/device_script.sh')),\ + mock.patch.object(os, 'fchmod'): + mock_popen.return_value.returncode = 0 + + test_runner.main() + self.assertEqual(1, mock_popen.call_count) + expected_cmd = [ + test_runner.CROS_RUN_TEST_PATH, '--board', 'eve', '--cache-dir', + test_runner.DEFAULT_CROS_CACHE, '--as-chronos', '--remote-cmd', + '--cwd', 'out_eve/Release', '--files', + 'out_eve/Release/device_script.sh' + ] + expected_cmd.extend(['--start', '--copy-on-write'] + if use_vm else ['--device', 'localhost:2222']) + expected_cmd.extend(['--', './device_script.sh']) + self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0]) + + fd_mock().write.assert_called_once_with( + '#!/bin/sh\nexport HOME=/usr/local/tmp\n' + 'export TMPDIR=/usr/local/tmp\n' + 'LD_LIBRARY_PATH=./ ./out_eve/Release/base_unittests ' + '--test-launcher-shard-index=0 --test-launcher-total-shards=1\n') + mock_remove.assert_called_once_with('out_eve/Release/device_script.sh') + + def test_gtest_with_vpython(self): + """Tests building a gtest with --vpython-dir.""" + args = mock.MagicMock() + args.test_exe = 'base_unittests' + args.test_launcher_summary_output = None + args.trace_dir = None + args.runtime_deps_path = None + args.path_to_outdir = self._tmp_dir + args.vpython_dir = self._tmp_dir + args.logs_dir = self._tmp_dir + + # With vpython_dir initially empty, the test_runner should error out + # due to missing vpython binaries. + gtest = test_runner.GTestTest(args, None) + with self.assertRaises(test_runner.TestFormatError): + gtest.build_test_command() + + # Create the two expected tools, and the test should be ready to run. + with open(os.path.join(args.vpython_dir, 'vpython3'), 'w'): + pass # Just touch the file. + os.mkdir(os.path.join(args.vpython_dir, 'bin')) + with open(os.path.join(args.vpython_dir, 'bin', 'python3'), 'w'): + pass + gtest = test_runner.GTestTest(args, None) + gtest.build_test_command() + + +class HostCmdTests(TestRunnerTest): + + @parameterized.expand([ + [True], + [False], + ]) + def test_host_cmd(self, is_lacros): + args = [ + 'script_name', + 'host-cmd', + '--board=eve', + '--flash', + '--path-to-outdir=out/Release', + '--device=localhost:2222', + ] + if is_lacros: + args += ['--deploy-lacros'] + else: + args += ['--deploy-chrome'] + args += [ + '--', + 'fake_cmd', + ] + with mock.patch.object(sys, 'argv', args),\ + mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen: + mock_popen.return_value.returncode = 0 + + test_runner.main() + expected_cmd = [ + test_runner.CROS_RUN_TEST_PATH, + '--board', + 'eve', + '--cache-dir', + test_runner.DEFAULT_CROS_CACHE, + '--flash', + '--device', + 'localhost:2222', + '--build-dir', + os.path.join(test_runner.CHROMIUM_SRC_PATH, 'out/Release'), + '--host-cmd', + ] + if is_lacros: + expected_cmd += [ + '--deploy-lacros', + '--lacros-launcher-script', + test_runner.LACROS_LAUNCHER_SCRIPT_PATH, + ] + else: + expected_cmd += ['--mount', '--nostrip', '--deploy'] + + expected_cmd += [ + '--', + 'fake_cmd', + ] + + self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0]) + + +if __name__ == '__main__': + unittest.main() |