diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 09:22:09 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 09:22:09 +0000 |
| commit | 43a97878ce14b72f0981164f87f2e35e14151312 (patch) | |
| tree | 620249daf56c0258faa40cbdcf9cfba06de2a846 /third_party/libwebrtc/tools_webrtc | |
| parent | Initial commit. (diff) | |
| download | firefox-43a97878ce14b72f0981164f87f2e35e14151312.tar.xz firefox-43a97878ce14b72f0981164f87f2e35e14151312.zip | |
Adding upstream version 110.0.1.upstream/110.0.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'third_party/libwebrtc/tools_webrtc')
267 files changed, 10713 insertions, 0 deletions
diff --git a/third_party/libwebrtc/tools_webrtc/BUILD.gn b/third_party/libwebrtc/tools_webrtc/BUILD.gn new file mode 100644 index 0000000000..ee9a734107 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/BUILD.gn @@ -0,0 +1,18 @@ +# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +if (target_os == "android") { + action("binary_version_check") { + testonly = true + script = "binary_version_check.py" + deps = [ "../sdk/android:libjingle_peerconnection_so" ] + inputs = [ "$root_out_dir/libjingle_peerconnection_so.so" ] + outputs = [ "$root_out_dir/webrtc_binary_version_check" ] + args = [ "libjingle_peerconnection_so.so" ] + } +} diff --git a/third_party/libwebrtc/tools_webrtc/OWNERS b/third_party/libwebrtc/tools_webrtc/OWNERS new file mode 100644 index 0000000000..f73dc520b8 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/OWNERS @@ -0,0 +1,5 @@ +mbonadei@webrtc.org +jansson@webrtc.org +terelius@webrtc.org +landrey@webrtc.org +jleconte@webrtc.org diff --git a/third_party/libwebrtc/tools_webrtc/__init__.py b/third_party/libwebrtc/tools_webrtc/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/__init__.py diff --git a/third_party/libwebrtc/tools_webrtc/android/OWNERS b/third_party/libwebrtc/tools_webrtc/android/OWNERS new file mode 100644 index 0000000000..cf092a316a --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/android/OWNERS @@ -0,0 +1 @@ +xalep@webrtc.org diff --git a/third_party/libwebrtc/tools_webrtc/android/__init__.py b/third_party/libwebrtc/tools_webrtc/android/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/android/__init__.py diff --git a/third_party/libwebrtc/tools_webrtc/android/adb_shell.sh b/third_party/libwebrtc/tools_webrtc/android/adb_shell.sh new file mode 100755 index 0000000000..e2d4f9187e --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/android/adb_shell.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +# 'adb shell' always returns "0" regardless of executable return code. +# This handy script will return executable return code to shell which +# can be used by buildbots. + +adb_shell () { + local RET ADB_LOG + ADB_LOG=$(mktemp "${TMPDIR:-/tmp}/adb-XXXXXXXX") + adb "$1" "$2" shell "$3" "$4" ";" echo \$? | tee "$ADB_LOG" + sed -i -e 's![[:cntrl:]]!!g' "$ADB_LOG" # Remove \r. + RET=$(sed -e '$!d' "$ADB_LOG") # Last line contains status code. + rm -f "$ADB_LOG" + return $RET +} diff --git a/third_party/libwebrtc/tools_webrtc/android/build_aar.py b/third_party/libwebrtc/tools_webrtc/android/build_aar.py new file mode 100755 index 0000000000..de5fe2fb3f --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/android/build_aar.py @@ -0,0 +1,263 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""Script to generate libwebrtc.aar for distribution. + +The script has to be run from the root src folder. +./tools_webrtc/android/build_aar.py + +.aar-file is just a zip-archive containing the files of the library. The file +structure generated by this script looks like this: + - AndroidManifest.xml + - classes.jar + - libs/ + - armeabi-v7a/ + - libjingle_peerconnection_so.so + - x86/ + - libjingle_peerconnection_so.so +""" + +import argparse +import logging +import os +import shutil +import subprocess +import sys +import tempfile +import zipfile + +SCRIPT_DIR = os.path.dirname(os.path.realpath(sys.argv[0])) +SRC_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, os.pardir, os.pardir)) +DEFAULT_ARCHS = ['armeabi-v7a', 'arm64-v8a', 'x86', 'x86_64'] +NEEDED_SO_FILES = ['libjingle_peerconnection_so.so'] +JAR_FILE = 'lib.java/sdk/android/libwebrtc.jar' +MANIFEST_FILE = 'sdk/android/AndroidManifest.xml' +TARGETS = [ + 'sdk/android:libwebrtc', + 'sdk/android:libjingle_peerconnection_so', +] + +sys.path.append(os.path.join(SCRIPT_DIR, '..', 'libs')) +from generate_licenses import LicenseBuilder + +sys.path.append(os.path.join(SRC_DIR, 'build')) +import find_depot_tools + + +def _ParseArgs(): + parser = argparse.ArgumentParser(description='libwebrtc.aar generator.') + parser.add_argument( + '--build-dir', + type=os.path.abspath, + help='Build dir. By default will create and use temporary dir.') + parser.add_argument('--output', + default='libwebrtc.aar', + type=os.path.abspath, + help='Output file of the script.') + parser.add_argument('--arch', + default=DEFAULT_ARCHS, + nargs='*', + help='Architectures to build. Defaults to %(default)s.') + parser.add_argument('--use-goma', + action='store_true', + default=False, + help='Use goma.') + parser.add_argument('--use-unstripped-libs', + action='store_true', + default=False, + help='Use unstripped .so files within libwebrtc.aar') + parser.add_argument('--verbose', + action='store_true', + default=False, + help='Debug logging.') + parser.add_argument( + '--extra-gn-args', + default=[], + nargs='*', + help="""Additional GN arguments to be used during Ninja generation. + These are passed to gn inside `--args` switch and + applied after any other arguments and will + override any values defined by the script. + Example of building debug aar file: + build_aar.py --extra-gn-args='is_debug=true'""") + parser.add_argument( + '--extra-ninja-switches', + default=[], + nargs='*', + help="""Additional Ninja switches to be used during compilation. + These are applied after any other Ninja switches. + Example of enabling verbose Ninja output: + build_aar.py --extra-ninja-switches='-v'""") + parser.add_argument( + '--extra-gn-switches', + default=[], + nargs='*', + help="""Additional GN switches to be used during compilation. + These are applied after any other GN switches. + Example of enabling verbose GN output: + build_aar.py --extra-gn-switches='-v'""") + return parser.parse_args() + + +def _RunGN(args): + cmd = [ + sys.executable, + os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py') + ] + cmd.extend(args) + logging.debug('Running: %r', cmd) + subprocess.check_call(cmd) + + +def _RunNinja(output_directory, args): + cmd = [ + os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'ninja'), '-C', + output_directory + ] + cmd.extend(args) + logging.debug('Running: %r', cmd) + subprocess.check_call(cmd) + + +def _EncodeForGN(value): + """Encodes value as a GN literal.""" + if isinstance(value, str): + return '"' + value + '"' + if isinstance(value, bool): + return repr(value).lower() + return repr(value) + + +def _GetOutputDirectory(build_dir, arch): + """Returns the GN output directory for the target architecture.""" + return os.path.join(build_dir, arch) + + +def _GetTargetCpu(arch): + """Returns target_cpu for the GN build with the given architecture.""" + if arch in ['armeabi', 'armeabi-v7a']: + return 'arm' + if arch == 'arm64-v8a': + return 'arm64' + if arch == 'x86': + return 'x86' + if arch == 'x86_64': + return 'x64' + raise Exception('Unknown arch: ' + arch) + + +def _GetArmVersion(arch): + """Returns arm_version for the GN build with the given architecture.""" + if arch == 'armeabi': + return 6 + if arch == 'armeabi-v7a': + return 7 + if arch in ['arm64-v8a', 'x86', 'x86_64']: + return None + raise Exception('Unknown arch: ' + arch) + + +def Build(build_dir, arch, use_goma, extra_gn_args, extra_gn_switches, + extra_ninja_switches): + """Generates target architecture using GN and builds it using ninja.""" + logging.info('Building: %s', arch) + output_directory = _GetOutputDirectory(build_dir, arch) + gn_args = { + 'target_os': 'android', + 'is_debug': False, + 'is_component_build': False, + 'rtc_include_tests': False, + 'target_cpu': _GetTargetCpu(arch), + 'use_goma': use_goma + } + arm_version = _GetArmVersion(arch) + if arm_version: + gn_args['arm_version'] = arm_version + gn_args_str = '--args=' + ' '.join( + [k + '=' + _EncodeForGN(v) for k, v in gn_args.items()] + extra_gn_args) + + gn_args_list = ['gen', output_directory, gn_args_str] + gn_args_list.extend(extra_gn_switches) + _RunGN(gn_args_list) + + ninja_args = TARGETS[:] + if use_goma: + ninja_args.extend(['-j', '200']) + ninja_args.extend(extra_ninja_switches) + _RunNinja(output_directory, ninja_args) + + +def CollectCommon(aar_file, build_dir, arch): + """Collects architecture independent files into the .aar-archive.""" + logging.info('Collecting common files.') + output_directory = _GetOutputDirectory(build_dir, arch) + aar_file.write(MANIFEST_FILE, 'AndroidManifest.xml') + aar_file.write(os.path.join(output_directory, JAR_FILE), 'classes.jar') + + +def Collect(aar_file, build_dir, arch, unstripped): + """Collects architecture specific files into the .aar-archive.""" + logging.info('Collecting: %s', arch) + output_directory = _GetOutputDirectory(build_dir, arch) + + abi_dir = os.path.join('jni', arch) + for so_file in NEEDED_SO_FILES: + source_so_file = os.path.join("lib.unstripped", + so_file) if unstripped else so_file + aar_file.write(os.path.join(output_directory, source_so_file), + os.path.join(abi_dir, so_file)) + + +def GenerateLicenses(output_dir, build_dir, archs): + builder = LicenseBuilder( + [_GetOutputDirectory(build_dir, arch) for arch in archs], TARGETS) + builder.GenerateLicenseText(output_dir) + + +def BuildAar(archs, + output_file, + use_goma=False, + extra_gn_args=None, + ext_build_dir=None, + extra_gn_switches=None, + extra_ninja_switches=None, + unstripped=False): + extra_gn_args = extra_gn_args or [] + extra_gn_switches = extra_gn_switches or [] + extra_ninja_switches = extra_ninja_switches or [] + build_dir = ext_build_dir if ext_build_dir else tempfile.mkdtemp() + + for arch in archs: + Build(build_dir, arch, use_goma, extra_gn_args, extra_gn_switches, + extra_ninja_switches) + + with zipfile.ZipFile(output_file, 'w') as aar_file: + # Architecture doesn't matter here, arbitrarily using the first one. + CollectCommon(aar_file, build_dir, archs[0]) + for arch in archs: + Collect(aar_file, build_dir, arch, unstripped) + + license_dir = os.path.dirname(os.path.realpath(output_file)) + GenerateLicenses(license_dir, build_dir, archs) + + if not ext_build_dir: + shutil.rmtree(build_dir, True) + + +def main(): + args = _ParseArgs() + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) + + BuildAar(args.arch, args.output, args.use_goma, args.extra_gn_args, + args.build_dir, args.extra_gn_switches, args.extra_ninja_switches, + args.use_unstripped_libs) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/third_party/libwebrtc/tools_webrtc/android/profiling/perf_setup.sh b/third_party/libwebrtc/tools_webrtc/android/profiling/perf_setup.sh new file mode 100755 index 0000000000..9c6b0f98ea --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/android/profiling/perf_setup.sh @@ -0,0 +1,470 @@ +#!/bin/bash + +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +# +# Usage: +# +# It is assumed that a release build of AppRTCMobile exists and has been +# installed on an Android device which supports USB debugging. +# +# Source this script once from the WebRTC src/ directory and resolve any +# reported issues. Add relative path to build directory as parameter. +# Required tools will be downloaded if they don't already exist. +# +# Once all tests are passed, a list of available functions will be given. +# Use these functions to do the actual profiling and visualization of the +# results. +# +# Note that, using a rooted device is recommended since it allows us to +# resolve kernel symbols (kallsyms) as well. +# +# Example usage: +# +# > . tools_webrtc/android/profiling/perf_setup.sh out/Release +# > perf_record 120 +# > flame_graph +# > plot_flame_graph +# > perf_cleanup + +if [ -n "$ZSH_VERSION" ]; then + # Running inside zsh. + SCRIPT_PATH="${(%):-%N}" +else + # Running inside something else (most likely bash). + SCRIPT_PATH="${BASH_SOURCE[0]}" +fi +SCRIPT_DIR="$(cd $(dirname "$SCRIPT_PATH") && pwd -P)" +source "${SCRIPT_DIR}/utilities.sh" + +# Root directory for local symbol cache. +SYMBOL_DIR="${TMPDIR:-/tmp}/android_symbols" +# Used as a temporary folder on the Android device for data storage. +DEV_TMP_DIR="/data/local/tmp" +# Relative path to native shared library containing symbols. +NATIVE_LIB_PATH="/lib.unstripped/libjingle_peerconnection_so.so" +# Name of application package for the AppRTCMobile demo. +APP_NAME="org.appspot.apprtc" + +# Make sure we're being sourced. +if [[ -n "${BASH_VERSION}" && "${BASH_SOURCE:-$0}" == "$0" ]]; then + error "perf_setup must be sourced" + exit 1 +fi + +function usage() { + printf "usage: . perf_setup.sh <build_dir>\n" +} + +# Ensure that user includes name of build directory (e.g. out/Release) as +# input parameter. Store path in BUILD_DIR. +if [[ "$#" -eq 1 ]]; then + if is_not_dir "$1"; then + error "$1 is invalid" + return 1 + fi + BUILD_DIR="$1" +else + error "Missing required parameter". + usage + return 1 +fi + +# Full (relative) path to the libjingle_peerconnection_so.so file. +function native_shared_lib_path() { + echo "${BUILD_DIR}${NATIVE_LIB_PATH}" +} + +# Target CPU architecture for the native shared library. +# Example: AArch64. +function native_shared_lib_arch() { + readelf -h $(native_shared_lib_path) | grep Machine | awk '{print $2}' +} + +# Returns true if the device architecture and the build target are the same. +function arch_is_ok() { + if [[ "$(dev_arch)" == "aarch64" ]] \ + && [[ "$(native_shared_lib_arch)" == "AArch64" ]]; then + return 0 + elif [[ "$(dev_arch)" == "aarch32" ]] \ + && [[ "$(native_shared_lib_arch)" == "AArch32" ]]; then + return 0 + else + return 1 + fi +} + +# Copies the native shared library from the local host to the symbol cache +# which is used by simpleperf as base when searching for symbols. +function copy_native_shared_library_to_symbol_cache() { + local arm_lib="arm" + if [[ "$(native_shared_lib_arch)" == "AArch64" ]]; then + arm_lib="arm64" + fi + for num in 1 2; do + local dir="${SYMBOL_DIR}/data/app/${APP_NAME}-${num}/lib/${arm_lib}" + mkdir -p "${dir}" + cp -u $(native_shared_lib_path) "${dir}" + done +} + +# Copy kernel symbols from device to symbol cache in tmp. +function copy_kernel_symbols_from_device_to_symbol_cache() { + local symbol_cache="${SYMBOL_DIR}/kallsyms" + adb pull /proc/kallsyms "${symbol_cache}" +} 1> /dev/null + +# Download the correct version of 'simpleperf' to $DEV_TMP_DIR +# on the device and enable profiling. +function copy_simpleperf_to_device() { + local perf_binary + [[ $(dev_arch) == "aarch64" ]] \ + && perf_binary="/arm64/simpleperf" \ + || perf_binary="/arm/simpleperf" + # Copy the simpleperf binary from local host to temp folder on device. + adb push "${SCRIPT_DIR}/simpleperf/bin/android${perf_binary}" \ + "${DEV_TMP_DIR}" 1> /dev/null + # Copy simpleperf from temp folder to the application package. + adb shell run-as "${APP_NAME}" cp "${DEV_TMP_DIR}/simpleperf" . + adb shell run-as "${APP_NAME}" chmod a+x simpleperf + # Enable profiling on the device. + enable_profiling + # Allows usage of running report commands on the device. + if image_is_root; then + enable_report_symbols + fi +} + +# Copy the recorded 'perf.data' file from the device to the current directory. +# TODO(henrika): add support for specifying the destination. +function pull_perf_data_from_device() { + adb shell run-as "${APP_NAME}" cp perf.data /sdcard/perf.data + adb pull sdcard/perf.data . +} 1> /dev/null + + +# Wraps calls to simpleperf report. Used by e.g. perf_report_threads. +# A valid profile input file must exist in the current folder. +# TODO(henrika): possibly add support to add path to alternative input file. +function perf_report() { + local perf_data="perf.data" + is_file "${perf_data}" \ + && simpleperf report \ + -n \ + -i "${perf_data}" \ + "$@" \ + || error "$(pwd)/${perf_data} is invalid" +} + +# Removes the folder specified as input parameter. Mainly intended for removal +# of simpleperf and Flame Graph tools. +function remove_tool() { + local tool_dir="$1" + if is_dir "${tool_dir}"; then + echo "Removing ${tool_dir}..." + rm -rf "${tool_dir}" + path_remove "${tool_dir}" + fi +} + +# Utility method which deletes the downloaded simpleperf tool from the repo. +# It also removes the simpleperf root folder from PATH. +function rm_simpleperf() { + remove_tool "${SCRIPT_DIR}/simpleperf" +} + +# Utility method which deletes the downloaded Flame Graph tool from the repo. +# It also removes the Flame Graph root folder from PATH. +function rm_flame_graph() { + remove_tool "${SCRIPT_DIR}/flamegraph" +} + +# Lists the main available functions after sourcing this script. +function print_function_help() { + printf "\nAvailable functions in this shell:\n" + printf " perf_record [duration, default=60sec]\n" + printf " perf_report_threads\n" + printf " perf_report_bins\n" + printf " perf_report_symbols\n" + printf " perf_report_graph\n" + printf " perf_report_graph_callee\n" + printf " perf_update\n" + printf " perf_cleanup\n" + printf " flame_graph\n" + printf " plot_flame_graph\n" +} + +function cleanup() { + unset -f main +} + +# ----------------------------------------------------------------------------- +# Main methods to be used after sourcing the main script. +# ----------------------------------------------------------------------------- + +# Call this method after the application as been rebuilt and installed on the +# device to ensure that symbols are up-to-date. +function perf_update() { + copy_native_shared_library_to_symbol_cache + if image_is_root; then + copy_kernel_symbols_from_device_to_symbol_cache + fi +} + +# Record stack frame based call graphs while using the application. +# We use default events (cpu-cycles), and write records to 'perf.data' in the +# tmp folder on the device. Default duration is 60 seconds but it can be changed +# by adding one parameter. As soon as the recording is done, 'perf.data' is +# copied to the directory from which this method is called and a summary of +# the load distribution per thread is printed. +function perf_record() { + if app_is_running "${APP_NAME}"; then + # Ensure that the latest native shared library exists in the local cache. + copy_native_shared_library_to_symbol_cache + local duration=60 + if [ "$#" -eq 1 ]; then + duration="$1" + fi + local pid=$(find_app_pid "${APP_NAME}") + echo "Profiling PID $pid for $duration seconds (media must be is active)..." + adb shell run-as "${APP_NAME}" ./simpleperf record \ + --call-graph fp \ + -p "${pid}" \ + -f 1000 \ + --duration "${duration}" \ + --log error + # Copy profile results from device to current directory. + pull_perf_data_from_device + # Print out a summary report (load per thread). + perf_report_threads | tail -n +6 + else + # AppRTCMobile was not enabled. Start it up automatically and ask the user + # to start media and then call this method again. + warning "AppRTCMobile must be active" + app_start "${APP_NAME}" + echo "Start media and then call perf_record again..." + fi +} + +# Analyze the profile report and show samples per threads. +function perf_report_threads() { + perf_report --sort comm +} 2> /dev/null + +# Analyze the profile report and show samples per binary. +function perf_report_bins() { + perf_report --sort dso +} 2> /dev/null + +# Analyze the profile report and show samples per symbol. +function perf_report_symbols() { + perf_report --sort symbol --symfs "${SYMBOL_DIR}" +} + +# Print call graph showing how functions call others. +function perf_report_graph() { + perf_report -g caller --symfs "${SYMBOL_DIR}" +} + +# Print call graph showing how functions are called from others. +function perf_report_graph_callee() { + perf_report -g callee --symfs "${SYMBOL_DIR}" +} + +# Plots the default Flame Graph file if no parameter is provided. +# If a parameter is given, it will be used as file name instead of the default. +function plot_flame_graph() { + local file_name="flame_graph.svg" + if [[ "$#" -eq 1 ]]; then + file_name="$1" + fi + # Open up the SVG file in Chrome. Try unstable first and revert to stable + # if unstable fails. + google-chrome-unstable "${file_name}" \ + || google-chrome-stable "${file_name}" \ + || error "failed to find any Chrome instance" +} 2> /dev/null + +# Generate Flame Graph in interactive SVG format. +# First input parameter corresponds to output file name and second input +# parameter is the heading of the plot. +# Defaults will be utilized if parameters are not provided. +# See https://github.com/brendangregg/FlameGraph for details on Flame Graph. +function flame_graph() { + local perf_data="perf.data" + if is_not_file $perf_data; then + error "$(pwd)/${perf_data} is invalid" + return 1 + fi + local file_name="flame_graph.svg" + local title="WebRTC Flame Graph" + if [[ "$#" -eq 1 ]]; then + file_name="$1" + fi + if [[ "$#" -eq 2 ]]; then + file_name="$1" + title="$2" + fi + if image_is_not_root; then + report_sample.py \ + --symfs "${SYMBOL_DIR}" \ + perf.data >out.perf + else + report_sample.py \ + --symfs "${SYMBOL_DIR}" \ + --kallsyms "${SYMBOL_DIR}/kallsyms" \ + perf.data >out.perf + fi + stackcollapse-perf.pl out.perf >out.folded + flamegraph.pl --title="${title}" out.folded >"${file_name}" + rm out.perf + rm out.folded +} + +# Remove all downloaded third-party tools. +function perf_cleanup () { + rm_simpleperf + rm_flame_graph +} + +main() { + printf "%s\n" "Preparing profiling of AppRTCMobile on Android:" + # Verify that this script is called from the root folder of WebRTC, + # i.e., the src folder one step below where the .gclient file exists. + local -r project_root_dir=$(pwd) + local dir=${project_root_dir##*/} + if [[ "${dir}" != "src" ]]; then + error "script must be called from the WebRTC project root (src) folder" + return 1 + fi + ok "project root: ${project_root_dir}" + + # Verify that user has sourced envsetup.sh. + # TODO(henrika): might be possible to remove this check. + if [[ -z "$ENVSETUP_GYP_CHROME_SRC" ]]; then + error "must source envsetup script first" + return 1 + fi + ok "envsetup script has been sourced" + + # Given that envsetup is sourced, the adb tool should be accessible but + # do one extra check just in case. + local adb_full_path=$(which adb); + if [[ ! -x "${adb_full_path}" ]]; then + error "unable to find the Android Debug Bridge (adb) tool" + return 1 + fi + ok "adb tool is working" + + # Exactly one Android device must be connected. + if ! one_device_connected; then + error "one device must be connected" + return 1 + fi + ok "one device is connected via USB" + + # Restart adb with root permissions if needed. + if image_is_root && adb_has_no_root_permissions; then + adb root + ok "adb is running as root" + fi + + # Create an empty symbol cache in the tmp folder. + # TODO(henrika): it might not be required to start from a clean cache. + is_dir "${SYMBOL_DIR}" && rm -rf "${SYMBOL_DIR}" + mkdir "${SYMBOL_DIR}" \ + && ok "empty symbol cache created at ${SYMBOL_DIR}" \ + || error "failed to create symbol cache" + + # Ensure that path to the native library with symbols is valid. + local native_lib=$(native_shared_lib_path) + if is_not_file ${native_lib}; then + error "${native_lib} is not a valid file" + return 1 + fi + ok "native library: "${native_lib}"" + + # Verify that the architechture of the device matches the architecture + # of the native library. + if ! arch_is_ok; then + error "device is $(dev_arch) and lib is $(native_shared_lib_arch)" + return 1 + fi + ok "device is $(dev_arch) and lib is $(native_shared_lib_arch)" + + # Copy native shared library to symbol cache after creating an + # application specific tree structure under ${SYMBOL_DIR}/data. + copy_native_shared_library_to_symbol_cache + ok "native library copied to ${SYMBOL_DIR}/data/app/${APP_NAME}" + + # Verify that the application is installed on the device. + if ! app_is_installed "${APP_NAME}"; then + error "${APP_NAME} is not installed on the device" + return 1 + fi + ok "${APP_NAME} is installed on the device" + + # Download simpleperf to <src>/tools_webrtc/android/profiling/simpleperf/. + # Cloning will only take place if the target does not already exist. + # The PATH variable will also be updated. + # TODO(henrika): would it be better to use a target outside the WebRTC repo? + local simpleperf_dir="${SCRIPT_DIR}/simpleperf" + if is_not_dir "${simpleperf_dir}"; then + echo "Dowloading simpleperf..." + git clone https://android.googlesource.com/platform/prebuilts/simpleperf \ + "${simpleperf_dir}" + chmod u+x "${simpleperf_dir}/report_sample.py" + fi + path_add "${simpleperf_dir}" + ok "${simpleperf_dir}" is added to PATH + + # Update the PATH variable with the path to the Linux version of simpleperf. + local simpleperf_linux_dir="${SCRIPT_DIR}/simpleperf/bin/linux/x86_64/" + if is_not_dir "${simpleperf_linux_dir}"; then + error "${simpleperf_linux_dir} is invalid" + return 1 + fi + path_add "${simpleperf_linux_dir}" + ok "${simpleperf_linux_dir}" is added to PATH + + # Copy correct version (arm or arm64) of simpleperf to the device + # and enable profiling at the same time. + if ! copy_simpleperf_to_device; then + error "failed to install simpleperf on the device" + return 1 + fi + ok "simpleperf is installed on the device" + + # Refresh the symbol cache and read kernal symbols from device if not + # already done. + perf_update + ok "symbol cache is updated" + + # Download Flame Graph to <src>/tools_webrtc/android/profiling/flamegraph/. + # Cloning will only take place if the target does not already exist. + # The PATH variable will also be updated. + # TODO(henrika): would it be better to use a target outside the WebRTC repo? + local flamegraph_dir="${SCRIPT_DIR}/flamegraph" + if is_not_dir "${flamegraph_dir}"; then + echo "Dowloading Flame Graph visualization tool..." + git clone https://github.com/brendangregg/FlameGraph.git "${flamegraph_dir}" + fi + path_add "${flamegraph_dir}" + ok "${flamegraph_dir}" is added to PATH + + print_function_help + + cleanup + + return 0 +} + +# Only call main() if proper input parameter has been provided. +if is_set $BUILD_DIR; then + main "$@" +fi diff --git a/third_party/libwebrtc/tools_webrtc/android/profiling/utilities.sh b/third_party/libwebrtc/tools_webrtc/android/profiling/utilities.sh new file mode 100755 index 0000000000..46a97b8142 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/android/profiling/utilities.sh @@ -0,0 +1,154 @@ +#!/bin/bash + +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +# Utility functions to be used by perf_setup.sh. +# Contains helper methods and functions that wraps usage of adb. + +function error() { + echo "[ERROR] "$@"" >&2 +} + +function warning() { + echo "[WARNING] "$@"" >&1 +} + +function ok() { + echo "[OK] "$@"" >&1 +} + +function abs_path { + (cd $1; pwd) +} + +function is_set() { + local var="$1" + [[ -n "${var}" ]] +} + +function is_file() { + local file="$1" + [[ -f "${file}" ]] +} + +function is_not_file() { + local file="$1" + [[ ! -f "${file}" ]] +} + +function is_dir() { + local dir="$1" + [[ -d "${dir}" ]] +} + +function is_not_dir() { + local dir="$1" + [[ ! -d "${dir}" ]] +} + +# Adds (prepends) the PATH environment variable while avoid duplicates. +function path_add() { + case ":${PATH:=$1}:" in + *:$1:*) ;; + *) PATH="$1:$PATH" ;; + esac +} + +# Removes a path from the PATH environment variable using search-and-replace +# parameter expansion. +function path_remove { + local path="$1" + # Substitute first occurrence of ":path" in PATH with an empty string. + # Deletes instances in the middle or at the end. + PATH=${PATH/":$path"/} + # Substitute first occurrence of "path:" in PATH with an empty string. + # Delete instances at the beginning. + PATH=${PATH/"$path:"/} +} + +# Returns the process ID (PID) of the process that corresponds to the +# application name given as input parameter. +function find_app_pid() { + local app_name="$1" + adb shell ps | grep "${app_name}" | awk '{print $2}' +} + +function app_is_installed() { + local app_name="$1" + local installed_app_name=$(adb shell pm list packages \ + | grep "${app_name}" | awk -F':' '{print $2}') + is_set "${installed_app_name}" \ + && [[ "${installed_app_name}" = "${app_name}" ]] +} + +function app_is_running() { + local app_name="$1" + local app_pid=$(find_app_pid "${app_name}") + is_set "${app_pid}" +} + +function app_start() { + local app_name="$1" + adb shell am start \ + -n "${app_name}/.ConnectActivity" \ + -a android.intent.action.MAIN +} + +function app_stop() { + local app_name="$1" + adb shell am force-stop "${app_name}" +} + +function app_uninstall() { + local app_name="$1" + adb uninstall "${app_name}" +} + +function dev_arch() { + adb shell uname -m +} + +function dev_ls() { + local dir="$1" + adb shell ls "${dir}" +} + +# Returns true if exactly on device is connected. +function one_device_connected() { + [[ $(adb devices | wc -l) = 3 ]] +} + +# Returns true if device is rooted. +function image_is_root() { + [[ $(adb shell getprop ro.build.type) = "userdebug" ]] +} + +# Returns true if device is not rooted. +function image_is_not_root() { + [[ $(adb shell getprop ro.build.type) = "user" ]] +} + +# Returns true if adb is not already running as root. +# Should only be called on rooted devices. +function adb_has_no_root_permissions() { + [[ $(adb shell getprop service.adb.root) = 0 ]] +} + +# Android devices may disable profiling by default. We must enable it. +function enable_profiling() { + adb shell setprop security.perf_harden 0 +} + +# To make the report of symbols on device successful, we need to execute +# `echo 0 >/proc/sys/kernel/kptr_restrict`. +# Only needed if we run report commands on the same machine as we run +# record commands. +function enable_report_symbols() { + adb shell "echo 0 > /proc/sys/kernel/kptr_restrict" +} diff --git a/third_party/libwebrtc/tools_webrtc/android/templates/maven-repository.jinja b/third_party/libwebrtc/tools_webrtc/android/templates/maven-repository.jinja new file mode 100644 index 0000000000..10a37f40d3 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/android/templates/maven-repository.jinja @@ -0,0 +1,11 @@ +allprojects { + repositories { + maven { + url '{{ url }}' + credentials { + username '{{ username }}' + password '{{ password }}' + } + } + } +} diff --git a/third_party/libwebrtc/tools_webrtc/android/templates/pom.jinja b/third_party/libwebrtc/tools_webrtc/android/templates/pom.jinja new file mode 100644 index 0000000000..90ca51ca9c --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/android/templates/pom.jinja @@ -0,0 +1,18 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <groupId>org.webrtc</groupId> + <artifactId>google-webrtc</artifactId> + <version>{{ version }}</version> + <packaging>aar</packaging> + + <name>Google's WebRTC Android library</name> + <url>https://webrtc.org/</url> + + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <project.build.commitid>{{ commit }}</project.build.commitid> + </properties> + +</project> diff --git a/third_party/libwebrtc/tools_webrtc/android/test_aar.py b/third_party/libwebrtc/tools_webrtc/android/test_aar.py new file mode 100755 index 0000000000..7eb281aa9a --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/android/test_aar.py @@ -0,0 +1,143 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""Script for building and testing WebRTC AAR.""" + +import argparse +import logging +import os +import re +import shutil +import subprocess +import sys +import tempfile + +SCRIPT_DIR = os.path.dirname(os.path.realpath(sys.argv[0])) +CHECKOUT_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir, os.pardir)) + +sys.path.append(os.path.join(CHECKOUT_ROOT, 'tools_webrtc')) +from android.build_aar import BuildAar + +ARCHS = ['armeabi-v7a', 'arm64-v8a', 'x86', 'x86_64'] +ARTIFACT_ID = 'google-webrtc' +COMMIT_POSITION_REGEX = r'^Cr-Commit-Position: refs/heads/master@{#(\d+)}$' +GRADLEW_BIN = os.path.join(CHECKOUT_ROOT, + 'examples/androidtests/third_party/gradle/gradlew') +ADB_BIN = os.path.join(CHECKOUT_ROOT, + 'third_party/android_sdk/public/platform-tools/adb') +AAR_PROJECT_DIR = os.path.join(CHECKOUT_ROOT, 'examples/aarproject') + + +def _ParseArgs(): + parser = argparse.ArgumentParser(description='Releases WebRTC on Bintray.') + parser.add_argument('--use-goma', + action='store_true', + default=False, + help='Use goma.') + parser.add_argument('--skip-tests', + action='store_true', + default=False, + help='Skips running the tests.') + parser.add_argument( + '--build-dir', + default=None, + help='Temporary directory to store the build files. If not specified, ' + 'a new directory will be created.') + parser.add_argument('--verbose', + action='store_true', + default=False, + help='Debug logging.') + return parser.parse_args() + + +def _GetCommitHash(): + commit_hash = subprocess.check_output( + ['git', 'rev-parse', 'HEAD'], cwd=CHECKOUT_ROOT).decode('UTF-8').strip() + return commit_hash + + +def _GetCommitPos(): + commit_message = subprocess.check_output( + ['git', 'rev-list', '--format=%B', '--max-count=1', 'HEAD'], + cwd=CHECKOUT_ROOT).decode('UTF-8') + commit_pos_match = re.search(COMMIT_POSITION_REGEX, commit_message, + re.MULTILINE) + if not commit_pos_match: + raise Exception('Commit position not found in the commit message: %s' % + commit_message) + return commit_pos_match.group(1) + + +def _TestAAR(build_dir): + """Runs AppRTCMobile tests using the AAR. Returns true if the tests pass.""" + logging.info('Testing library.') + + # Uninstall any existing version of AppRTCMobile. + logging.info('Uninstalling previous AppRTCMobile versions. It is okay for ' + 'these commands to fail if AppRTCMobile is not installed.') + subprocess.call([ADB_BIN, 'uninstall', 'org.appspot.apprtc']) + subprocess.call([ADB_BIN, 'uninstall', 'org.appspot.apprtc.test']) + + # Run tests. + try: + # First clean the project. + subprocess.check_call([GRADLEW_BIN, 'clean'], cwd=AAR_PROJECT_DIR) + # Then run the tests. + subprocess.check_call([ + GRADLEW_BIN, 'connectedDebugAndroidTest', + '-PaarDir=' + os.path.abspath(build_dir) + ], + cwd=AAR_PROJECT_DIR) + except subprocess.CalledProcessError: + logging.exception('Test failure.') + return False # Clean or tests failed + + return True # Tests pass + + +def BuildAndTestAar(use_goma, skip_tests, build_dir): + version = '1.0.' + _GetCommitPos() + commit = _GetCommitHash() + logging.info('Building and Testing AAR version %s with hash %s', version, + commit) + + # If build directory is not specified, create a temporary directory. + use_tmp_dir = not build_dir + if use_tmp_dir: + build_dir = tempfile.mkdtemp() + + try: + base_name = ARTIFACT_ID + '-' + version + aar_file = os.path.join(build_dir, base_name + '.aar') + + logging.info('Building at %s', build_dir) + BuildAar(ARCHS, + aar_file, + use_goma=use_goma, + ext_build_dir=os.path.join(build_dir, 'aar-build')) + + tests_pass = skip_tests or _TestAAR(build_dir) + if not tests_pass: + raise Exception('Test failure.') + + logging.info('Test success.') + + finally: + if use_tmp_dir: + shutil.rmtree(build_dir, True) + + +def main(): + args = _ParseArgs() + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) + BuildAndTestAar(args.use_goma, args.skip_tests, args.build_dir) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/third_party/libwebrtc/tools_webrtc/apple/copy_framework_header.py b/third_party/libwebrtc/tools_webrtc/apple/copy_framework_header.py new file mode 100755 index 0000000000..3574a67d2a --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/apple/copy_framework_header.py @@ -0,0 +1,46 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import argparse +import re +import sys + + +def _ReplaceDoubleQuote(line): + re_rtc_import = re.compile(r'(\s*)#import\s+"(\S+/|)(\w+\+|)RTC(\w+)\.h"(.*)', + re.DOTALL) + match = re_rtc_import.match(line) + if not match: + return line + + return '%s#import <WebRTC/%sRTC%s.h>%s' % (match.group(1), match.group(3), + match.group(4), match.group(5)) + + +def Process(input_file, output_file): + with open(input_file, 'rb') as fb, open(output_file, 'wb') as fw: + for line in fb.read().decode('UTF-8').splitlines(): + fw.write(_ReplaceDoubleQuote(line).encode('UTF-8')) + fw.write(b"\n") + + +def main(): + parser = argparse.ArgumentParser( + description= + "Copy headers of framework and replace double-quoted includes to" + + " angle-bracketed respectively.") + parser.add_argument('--input', help='Input header files to copy.', type=str) + parser.add_argument('--output', help='Output file.', type=str) + parsed_args = parser.parse_args() + return Process(parsed_args.input, parsed_args.output) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/third_party/libwebrtc/tools_webrtc/apple/copy_framework_header_test.py b/third_party/libwebrtc/tools_webrtc/apple/copy_framework_header_test.py new file mode 100644 index 0000000000..7b8aceac6d --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/apple/copy_framework_header_test.py @@ -0,0 +1,35 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import unittest +from copy_framework_header import _ReplaceDoubleQuote + + +class TestCopyFramework(unittest.TestCase): + def testReplaceDoubleQuote(self): + self.assertEqual(_ReplaceDoubleQuote("""#import "RTCMacros.h\""""), + """#import <WebRTC/RTCMacros.h>""") + self.assertEqual(_ReplaceDoubleQuote("""#import "RTCMacros.h\"\n"""), + """#import <WebRTC/RTCMacros.h>\n""") + self.assertEqual( + _ReplaceDoubleQuote("""#import "UIDevice+RTCDevice.h\"\n"""), + """#import <WebRTC/UIDevice+RTCDevice.h>\n""") + self.assertEqual( + _ReplaceDoubleQuote("#import \"components/video_codec/" + + "RTCVideoDecoderFactoryH264.h\"\n"), + """#import <WebRTC/RTCVideoDecoderFactoryH264.h>\n""") + self.assertEqual( + _ReplaceDoubleQuote( + """@property(atomic, strong) RTC_OBJC_TYPE(RTCVideoFrame) *\n"""), + """@property(atomic, strong) RTC_OBJC_TYPE(RTCVideoFrame) *\n""") + + +if __name__ == '__main__': + unittest.main() diff --git a/third_party/libwebrtc/tools_webrtc/audio_quality/README b/third_party/libwebrtc/tools_webrtc/audio_quality/README new file mode 100644 index 0000000000..66bdd8a7e2 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/audio_quality/README @@ -0,0 +1,4 @@ +Use ../download_tools.py to download the files in this directory. Note that +they are downloaded from gs://chrome-webrtc-resources, which is a +google-internal bucket. If you're a non-Googler you'll have to download and +compile these tools manually in order to use them. diff --git a/third_party/libwebrtc/tools_webrtc/audio_quality/linux/PolqaOem64.sha1 b/third_party/libwebrtc/tools_webrtc/audio_quality/linux/PolqaOem64.sha1 new file mode 100644 index 0000000000..00dd87c901 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/audio_quality/linux/PolqaOem64.sha1 @@ -0,0 +1 @@ +2f705f4fc8f4175f75d0d946ad57787b99126e44
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/audio_quality/linux/pesq.sha1 b/third_party/libwebrtc/tools_webrtc/audio_quality/linux/pesq.sha1 new file mode 100644 index 0000000000..24f8a05f6b --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/audio_quality/linux/pesq.sha1 @@ -0,0 +1 @@ +bfbf5fa8fd9d6be5b4aa3f50caedbd786b0a034b
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/audio_quality/mac/pesq.sha1 b/third_party/libwebrtc/tools_webrtc/audio_quality/mac/pesq.sha1 new file mode 100644 index 0000000000..08b19a6f7d --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/audio_quality/mac/pesq.sha1 @@ -0,0 +1 @@ +9b9349dd7ef47709ca497cd66d9c453ab2b5c732
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/audio_quality/win/PolqaOem64.dll.sha1 b/third_party/libwebrtc/tools_webrtc/audio_quality/win/PolqaOem64.dll.sha1 new file mode 100644 index 0000000000..77c585308f --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/audio_quality/win/PolqaOem64.dll.sha1 @@ -0,0 +1 @@ +4b0a44ae698593cb3456aadd86207d168bb72abf
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/audio_quality/win/PolqaOem64.exe.sha1 b/third_party/libwebrtc/tools_webrtc/audio_quality/win/PolqaOem64.exe.sha1 new file mode 100644 index 0000000000..bae133aaa0 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/audio_quality/win/PolqaOem64.exe.sha1 @@ -0,0 +1 @@ +af3d2dce28bb52786bdad14270fb9f2decb4c220
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/audio_quality/win/pesq.exe.sha1 b/third_party/libwebrtc/tools_webrtc/audio_quality/win/pesq.exe.sha1 new file mode 100644 index 0000000000..23c98696d8 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/audio_quality/win/pesq.exe.sha1 @@ -0,0 +1 @@ +67726dd1d186b142a95914efc93233b8a9e583fe
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/audio_quality/win/vcomp120.dll.sha1 b/third_party/libwebrtc/tools_webrtc/audio_quality/win/vcomp120.dll.sha1 new file mode 100644 index 0000000000..b5b32877ce --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/audio_quality/win/vcomp120.dll.sha1 @@ -0,0 +1 @@ +1f8c667df810fed8fb42a6680a15465ac1e288eb
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/autoroller/roll_deps.py b/third_party/libwebrtc/tools_webrtc/autoroller/roll_deps.py new file mode 100755 index 0000000000..98c6404c3a --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/autoroller/roll_deps.py @@ -0,0 +1,778 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""Script to automatically roll dependencies in the WebRTC DEPS file.""" + + +import argparse +import base64 +import collections +import logging +import os +import re +import subprocess +import sys +import urllib.request + + +def FindSrcDirPath(): + """Returns the abs path to the src/ dir of the project.""" + src_dir = os.path.dirname(os.path.abspath(__file__)) + while os.path.basename(src_dir) != 'src': + src_dir = os.path.normpath(os.path.join(src_dir, os.pardir)) + return src_dir + + +# Skip these dependencies (list without solution name prefix). +DONT_AUTOROLL_THESE = [ + 'src/examples/androidtests/third_party/gradle', +] + +# These dependencies are missing in chromium/src/DEPS, either unused or already +# in-tree. For instance, src/base is a part of the Chromium source git repo, +# but we pull it through a subtree mirror, so therefore it isn't listed in +# Chromium's deps but it is in ours. +WEBRTC_ONLY_DEPS = [ + 'src/base', + 'src/build', + 'src/buildtools', + 'src/ios', + 'src/testing', + 'src/third_party', + 'src/third_party/gtest-parallel', + 'src/third_party/jdk/extras', + 'src/third_party/proguard', + 'src/tools', +] + +WEBRTC_URL = 'https://webrtc.googlesource.com/src' +CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src' +CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s' +CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s' +CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s' + +COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$') +CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'([-0-9a-z]+)\'$') +ROLL_BRANCH_NAME = 'roll_chromium_revision' + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +CHECKOUT_SRC_DIR = FindSrcDirPath() +CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir)) + +# Copied from tools/android/roll/android_deps/.../BuildConfigGenerator.groovy. +ANDROID_DEPS_START = r'=== ANDROID_DEPS Generated Code Start ===' +ANDROID_DEPS_END = r'=== ANDROID_DEPS Generated Code End ===' +# Location of automically gathered android deps. +ANDROID_DEPS_PATH = 'src/third_party/android_deps/' + +NOTIFY_EMAIL = 'webrtc-trooper@grotations.appspotmail.com' + +sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build')) +import find_depot_tools + +find_depot_tools.add_depot_tools_to_path() + +CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py' +CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(CHECKOUT_SRC_DIR, 'tools', + 'clang', 'scripts', 'update.py') + +DepsEntry = collections.namedtuple('DepsEntry', 'path url revision') +ChangedDep = collections.namedtuple('ChangedDep', + 'path url current_rev new_rev') +CipdDepsEntry = collections.namedtuple('CipdDepsEntry', 'path packages') +ChangedCipdPackage = collections.namedtuple( + 'ChangedCipdPackage', 'path package current_version new_version') + +ChromiumRevisionUpdate = collections.namedtuple('ChromiumRevisionUpdate', + ('current_chromium_rev ' + 'new_chromium_rev ')) + + +class RollError(Exception): + pass + + +def StrExpansion(): + return lambda str_value: str_value + + +def VarLookup(local_scope): + return lambda var_name: local_scope['vars'][var_name] + + +def ParseDepsDict(deps_content): + local_scope = {} + global_scope = { + 'Str': StrExpansion(), + 'Var': VarLookup(local_scope), + 'deps_os': {}, + } + exec(deps_content, global_scope, local_scope) + return local_scope + + +def ParseLocalDepsFile(filename): + with open(filename, 'rb') as f: + deps_content = f.read().decode('utf-8') + return ParseDepsDict(deps_content) + + +def ParseCommitPosition(commit_message): + for line in reversed(commit_message.splitlines()): + m = COMMIT_POSITION_RE.match(line.strip()) + if m: + return int(m.group(1)) + logging.error('Failed to parse commit position id from:\n%s\n', + commit_message) + sys.exit(-1) + + +def _RunCommand(command, + working_dir=None, + ignore_exit_code=False, + extra_env=None, + input_data=None): + """Runs a command and returns the output from that command. + + If the command fails (exit code != 0), the function will exit the process. + + Returns: + A tuple containing the stdout and stderr outputs as strings. + """ + working_dir = working_dir or CHECKOUT_SRC_DIR + logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir) + env = os.environ.copy() + if extra_env: + assert all(isinstance(value, str) for value in list(extra_env.values())) + logging.debug('extra env: %s', extra_env) + env.update(extra_env) + p = subprocess.Popen(command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + cwd=working_dir, + universal_newlines=True) + std_output, err_output = p.communicate(input_data) + p.stdout.close() + p.stderr.close() + if not ignore_exit_code and p.returncode != 0: + logging.error('Command failed: %s\n' + 'stdout:\n%s\n' + 'stderr:\n%s\n', ' '.join(command), std_output, err_output) + sys.exit(p.returncode) + return std_output, err_output + + +def _GetBranches(): + """Returns a tuple of active,branches. + + The 'active' is the name of the currently active branch and 'branches' is a + list of all branches. + """ + lines = _RunCommand(['git', 'branch'])[0].split('\n') + branches = [] + active = '' + for line in lines: + if '*' in line: + # The assumption is that the first char will always be the '*'. + active = line[1:].strip() + branches.append(active) + else: + branch = line.strip() + if branch: + branches.append(branch) + return active, branches + + +def _ReadGitilesContent(url): + # Download and decode BASE64 content until + # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed. + base64_content = ReadUrlContent(url + '?format=TEXT') + return base64.b64decode(base64_content[0]).decode('utf-8') + + +def ReadRemoteCrFile(path_below_src, revision): + """Reads a remote Chromium file of a specific revision. + + Args: + path_below_src: A path to the target file relative to src dir. + revision: Revision to read. + Returns: + A string with file content. + """ + return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % + (revision, path_below_src)) + + +def ReadRemoteCrCommit(revision): + """Reads a remote Chromium commit message. Returns a string.""" + return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision) + + +def ReadUrlContent(url): + """Connect to a remote host and read the contents. + + Args: + url: URL to connect to. + Returns: + A list of lines. + """ + conn = urllib.request.urlopen(url) + try: + return conn.readlines() + except IOError as e: + logging.exception('Error connecting to %s. Error: %s', url, e) + raise + finally: + conn.close() + + +def GetMatchingDepsEntries(depsentry_dict, dir_path): + """Gets all deps entries matching the provided path. + + This list may contain more than one DepsEntry object. + Example: dir_path='src/testing' would give results containing both + 'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's + DEPS. + Example 2: dir_path='src/build' should return 'src/build' but not + 'src/buildtools'. + + Returns: + A list of DepsEntry objects. + """ + result = [] + for path, depsentry in list(depsentry_dict.items()): + if path == dir_path: + result.append(depsentry) + else: + parts = path.split('/') + if all(part == parts[i] for i, part in enumerate(dir_path.split('/'))): + result.append(depsentry) + return result + + +def BuildDepsentryDict(deps_dict): + """Builds a dict of paths to DepsEntry objects from a raw deps dict.""" + result = {} + + def AddDepsEntries(deps_subdict): + for path, dep in list(deps_subdict.items()): + if path in result: + continue + if not isinstance(dep, dict): + dep = {'url': dep} + if dep.get('dep_type') == 'cipd': + result[path] = CipdDepsEntry(path, dep['packages']) + else: + if '@' not in dep['url']: + continue + url, revision = dep['url'].split('@') + result[path] = DepsEntry(path, url, revision) + + AddDepsEntries(deps_dict['deps']) + for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']: + AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {})) + return result + + +def _FindChangedCipdPackages(path, old_pkgs, new_pkgs): + old_pkgs_names = {p['package'] for p in old_pkgs} + new_pkgs_names = {p['package'] for p in new_pkgs} + pkgs_equal = (old_pkgs_names == new_pkgs_names) + added_pkgs = [p for p in new_pkgs_names if p not in old_pkgs_names] + removed_pkgs = [p for p in old_pkgs_names if p not in new_pkgs_names] + + assert pkgs_equal, ('Old: %s\n New: %s.\nYou need to do a manual roll ' + 'and remove/add entries in DEPS so the old and new ' + 'list match.\nMost likely, you should add \"%s\" and ' + 'remove \"%s\"' % + (old_pkgs, new_pkgs, added_pkgs, removed_pkgs)) + + for old_pkg in old_pkgs: + for new_pkg in new_pkgs: + old_version = old_pkg['version'] + new_version = new_pkg['version'] + if (old_pkg['package'] == new_pkg['package'] + and old_version != new_version): + logging.debug('Roll dependency %s to %s', path, new_version) + yield ChangedCipdPackage(path, old_pkg['package'], old_version, + new_version) + + +def _FindNewDeps(old, new): + """ Gather dependencies only in `new` and return corresponding paths. """ + old_entries = set(BuildDepsentryDict(old)) + new_entries = set(BuildDepsentryDict(new)) + return [ + path for path in new_entries - old_entries + if path not in DONT_AUTOROLL_THESE + ] + + +def FindAddedDeps(webrtc_deps, new_cr_deps): + """ + Calculate new deps entries of interest. + + Ideally, that would mean: only appearing in chromium DEPS + but transitively used in WebRTC. + + Since it's hard to compute, we restrict ourselves to a well defined subset: + deps sitting in `ANDROID_DEPS_PATH`. + Otherwise, assumes that's a Chromium-only dependency. + + Args: + webrtc_deps: dict of deps as defined in the WebRTC DEPS file. + new_cr_deps: dict of deps as defined in the chromium DEPS file. + + Caveat: Doesn't detect a new package in existing dep. + + Returns: + A tuple consisting of: + A list of paths added dependencies sitting in `ANDROID_DEPS_PATH`. + A list of paths for other added dependencies. + """ + all_added_deps = _FindNewDeps(webrtc_deps, new_cr_deps) + generated_android_deps = [ + path for path in all_added_deps if path.startswith(ANDROID_DEPS_PATH) + ] + other_deps = [ + path for path in all_added_deps if path not in generated_android_deps + ] + return generated_android_deps, other_deps + + +def FindRemovedDeps(webrtc_deps, new_cr_deps): + """ + Calculate obsolete deps entries. + + Ideally, that would mean: no more appearing in chromium DEPS + and not used in WebRTC. + + Since it's hard to compute: + 1/ We restrict ourselves to a well defined subset: + deps sitting in `ANDROID_DEPS_PATH`. + 2/ We rely on existing behavior of CalculateChangeDeps. + I.e. Assumes non-CIPD dependencies are WebRTC-only, don't remove them. + + Args: + webrtc_deps: dict of deps as defined in the WebRTC DEPS file. + new_cr_deps: dict of deps as defined in the chromium DEPS file. + + Caveat: Doesn't detect a deleted package in existing dep. + + Returns: + A tuple consisting of: + A list of paths of dependencies removed from `ANDROID_DEPS_PATH`. + A list of paths of unexpected disappearing dependencies. + """ + all_removed_deps = _FindNewDeps(new_cr_deps, webrtc_deps) + generated_android_deps = sorted( + [path for path in all_removed_deps if path.startswith(ANDROID_DEPS_PATH)]) + # Webrtc-only dependencies are handled in CalculateChangedDeps. + other_deps = sorted([ + path for path in all_removed_deps + if path not in generated_android_deps and path not in WEBRTC_ONLY_DEPS + ]) + return generated_android_deps, other_deps + + +def CalculateChangedDeps(webrtc_deps, new_cr_deps): + """ + Calculate changed deps entries based on entries defined in the WebRTC DEPS + file: + - If a shared dependency with the Chromium DEPS file: roll it to the same + revision as Chromium (i.e. entry in the new_cr_deps dict) + - If it's a Chromium sub-directory, roll it to the HEAD revision (notice + this means it may be ahead of the chromium_revision, but generally these + should be close). + - If it's another DEPS entry (not shared with Chromium), roll it to HEAD + unless it's configured to be skipped. + + Returns: + A list of ChangedDep objects representing the changed deps. + """ + result = [] + webrtc_entries = BuildDepsentryDict(webrtc_deps) + new_cr_entries = BuildDepsentryDict(new_cr_deps) + for path, webrtc_deps_entry in list(webrtc_entries.items()): + if path in DONT_AUTOROLL_THESE: + continue + cr_deps_entry = new_cr_entries.get(path) + if cr_deps_entry: + assert type(cr_deps_entry) is type(webrtc_deps_entry) + + if isinstance(cr_deps_entry, CipdDepsEntry): + result.extend( + _FindChangedCipdPackages(path, webrtc_deps_entry.packages, + cr_deps_entry.packages)) + continue + + # Use the revision from Chromium's DEPS file. + new_rev = cr_deps_entry.revision + assert webrtc_deps_entry.url == cr_deps_entry.url, ( + 'WebRTC DEPS entry %s has a different URL %s than Chromium %s.' % + (path, webrtc_deps_entry.url, cr_deps_entry.url)) + else: + if isinstance(webrtc_deps_entry, DepsEntry): + # Use the HEAD of the deps repo. + stdout, _ = _RunCommand( + ['git', 'ls-remote', webrtc_deps_entry.url, 'HEAD']) + new_rev = stdout.strip().split('\t')[0] + else: + # The dependency has been removed from chromium. + # This is handled by FindRemovedDeps. + continue + + # Check if an update is necessary. + if webrtc_deps_entry.revision != new_rev: + logging.debug('Roll dependency %s to %s', path, new_rev) + result.append( + ChangedDep(path, webrtc_deps_entry.url, webrtc_deps_entry.revision, + new_rev)) + return sorted(result) + + +def CalculateChangedClang(new_cr_rev): + + def GetClangRev(lines): + for line in lines: + match = CLANG_REVISION_RE.match(line) + if match: + return match.group(1) + raise RollError('Could not parse Clang revision!') + + with open(CLANG_UPDATE_SCRIPT_LOCAL_PATH, 'r') as f: + current_lines = f.readlines() + current_rev = GetClangRev(current_lines) + + new_clang_update_py = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH, + new_cr_rev).splitlines() + new_rev = GetClangRev(new_clang_update_py) + return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev) + + +def GenerateCommitMessage( + rev_update, + current_commit_pos, + new_commit_pos, + changed_deps_list, + added_deps_paths=None, + removed_deps_paths=None, + clang_change=None, +): + current_cr_rev = rev_update.current_chromium_rev[0:10] + new_cr_rev = rev_update.new_chromium_rev[0:10] + rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev) + git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos) + + commit_msg = [ + 'Roll chromium_revision %s (%s)\n' % (rev_interval, git_number_interval), + 'Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval), + 'Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % rev_interval) + ] + + def Section(adjective, deps): + noun = 'dependency' if len(deps) == 1 else 'dependencies' + commit_msg.append('%s %s' % (adjective, noun)) + + if changed_deps_list: + Section('Changed', changed_deps_list) + + for c in changed_deps_list: + if isinstance(c, ChangedCipdPackage): + commit_msg.append('* %s: %s..%s' % + (c.path, c.current_version, c.new_version)) + else: + commit_msg.append('* %s: %s/+log/%s..%s' % + (c.path, c.url, c.current_rev[0:10], c.new_rev[0:10])) + + if added_deps_paths: + Section('Added', added_deps_paths) + commit_msg.extend('* %s' % p for p in added_deps_paths) + + if removed_deps_paths: + Section('Removed', removed_deps_paths) + commit_msg.extend('* %s' % p for p in removed_deps_paths) + + if any([changed_deps_list, added_deps_paths, removed_deps_paths]): + change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS') + commit_msg.append('DEPS diff: %s\n' % change_url) + else: + commit_msg.append('No dependencies changed.') + + if clang_change and clang_change.current_rev != clang_change.new_rev: + commit_msg.append('Clang version changed %s:%s' % + (clang_change.current_rev, clang_change.new_rev)) + change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, + CLANG_UPDATE_SCRIPT_URL_PATH) + commit_msg.append('Details: %s\n' % change_url) + else: + commit_msg.append('No update to Clang.\n') + + commit_msg.append('BUG=None') + return '\n'.join(commit_msg) + + +def UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content): + """Update the DEPS file with the new revision.""" + + with open(deps_filename, 'rb') as deps_file: + deps_content = deps_file.read().decode('utf-8') + + # Update the chromium_revision variable. + deps_content = deps_content.replace(rev_update.current_chromium_rev, + rev_update.new_chromium_rev) + + # Add and remove dependencies. For now: only generated android deps. + # Since gclient cannot add or remove deps, we on the fact that + # these android deps are located in one place we can copy/paste. + deps_re = re.compile(ANDROID_DEPS_START + '.*' + ANDROID_DEPS_END, re.DOTALL) + new_deps = deps_re.search(new_cr_content) + old_deps = deps_re.search(deps_content) + if not new_deps or not old_deps: + faulty = 'Chromium' if not new_deps else 'WebRTC' + raise RollError('Was expecting to find "%s" and "%s"\n' + 'in %s DEPS' % + (ANDROID_DEPS_START, ANDROID_DEPS_END, faulty)) + deps_content = deps_re.sub(new_deps.group(0), deps_content) + + with open(deps_filename, 'wb') as deps_file: + deps_file.write(deps_content.encode('utf-8')) + + # Update each individual DEPS entry. + for dep in changed_deps: + local_dep_dir = os.path.join(CHECKOUT_ROOT_DIR, dep.path) + if not os.path.isdir(local_dep_dir): + raise RollError( + 'Cannot find local directory %s. Either run\n' + 'gclient sync --deps=all\n' + 'or make sure the .gclient file for your solution contains all ' + 'platforms in the target_os list, i.e.\n' + 'target_os = ["android", "unix", "mac", "ios", "win"];\n' + 'Then run "gclient sync" again.' % local_dep_dir) + if isinstance(dep, ChangedCipdPackage): + package = dep.package.format() # Eliminate double curly brackets + update = '%s:%s@%s' % (dep.path, package, dep.new_version) + else: + update = '%s@%s' % (dep.path, dep.new_rev) + _RunCommand(['gclient', 'setdep', '--revision', update], + working_dir=CHECKOUT_SRC_DIR) + + +def _IsTreeClean(): + stdout, _ = _RunCommand(['git', 'status', '--porcelain']) + if len(stdout) == 0: + return True + + logging.error('Dirty/unversioned files:\n%s', stdout) + return False + + +def _EnsureUpdatedMainBranch(dry_run): + current_branch = _RunCommand(['git', 'rev-parse', '--abbrev-ref', + 'HEAD'])[0].splitlines()[0] + if current_branch != 'main': + logging.error('Please checkout the main branch and re-run this script.') + if not dry_run: + sys.exit(-1) + + logging.info('Updating main branch...') + _RunCommand(['git', 'pull']) + + +def _CreateRollBranch(dry_run): + logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME) + if not dry_run: + _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME]) + + +def _RemovePreviousRollBranch(dry_run): + active_branch, branches = _GetBranches() + if active_branch == ROLL_BRANCH_NAME: + active_branch = 'main' + if ROLL_BRANCH_NAME in branches: + logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME) + if not dry_run: + _RunCommand(['git', 'checkout', active_branch]) + _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) + + +def _LocalCommit(commit_msg, dry_run): + logging.info('Committing changes locally.') + if not dry_run: + _RunCommand(['git', 'add', '--update', '.']) + _RunCommand(['git', 'commit', '-m', commit_msg]) + + +def ChooseCQMode(skip_cq, cq_over, current_commit_pos, new_commit_pos): + if skip_cq: + return 0 + if (new_commit_pos - current_commit_pos) < cq_over: + return 1 + return 2 + + +def _GetCcRecipients(changed_deps_list): + """Returns a list of emails to notify based on the changed deps list. + """ + cc_recipients = [] + for c in changed_deps_list: + if 'libvpx' in c.path or 'libaom' in c.path: + cc_recipients.append('marpan@webrtc.org') + cc_recipients.append('jianj@chromium.org') + return cc_recipients + + +def _UploadCL(commit_queue_mode, add_cc=None): + """Upload the committed changes as a changelist to Gerrit. + + commit_queue_mode: + - 2: Submit to commit queue. + - 1: Run trybots but do not submit to CQ. + - 0: Skip CQ, upload only. + + add_cc: A list of email addresses to add as CC recipients. + """ + cc_recipients = [NOTIFY_EMAIL] + if add_cc: + cc_recipients.extend(add_cc) + cmd = ['git', 'cl', 'upload', '--force', '--bypass-hooks'] + if commit_queue_mode >= 2: + logging.info('Sending the CL to the CQ...') + cmd.extend(['-o', 'label=Bot-Commit+1']) + cmd.extend(['-o', 'label=Commit-Queue+2']) + cmd.extend(['--send-mail', '--cc', ','.join(cc_recipients)]) + elif commit_queue_mode >= 1: + logging.info('Starting CQ dry run...') + cmd.extend(['-o', 'label=Commit-Queue+1']) + extra_env = { + 'EDITOR': 'true', + 'SKIP_GCE_AUTH_FOR_GIT': '1', + } + stdout, stderr = _RunCommand(cmd, extra_env=extra_env) + logging.debug('Output from "git cl upload":\nstdout:\n%s\n\nstderr:\n%s', + stdout, stderr) + + +def GetRollRevisionRanges(opts, webrtc_deps): + current_cr_rev = webrtc_deps['vars']['chromium_revision'] + new_cr_rev = opts.revision + if not new_cr_rev: + stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD']) + head_rev = stdout.strip().split('\t')[0] + logging.info('No revision specified. Using HEAD: %s', head_rev) + new_cr_rev = head_rev + + return ChromiumRevisionUpdate(current_cr_rev, new_cr_rev) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--clean', + action='store_true', + default=False, + help='Removes any previous local roll branch.') + p.add_argument('-r', + '--revision', + help=('Chromium Git revision to roll to. Defaults to the ' + 'Chromium HEAD revision if omitted.')) + p.add_argument('--dry-run', + action='store_true', + default=False, + help=('Calculate changes and modify DEPS, but don\'t create ' + 'any local branch, commit, upload CL or send any ' + 'tryjobs.')) + p.add_argument('-i', + '--ignore-unclean-workdir', + action='store_true', + default=False, + help=('Ignore if the current branch is not main or if there ' + 'are uncommitted changes (default: %(default)s).')) + grp = p.add_mutually_exclusive_group() + grp.add_argument('--skip-cq', + action='store_true', + default=False, + help='Skip sending the CL to the CQ (default: %(default)s)') + grp.add_argument('--cq-over', + type=int, + default=1, + help=('Commit queue dry run if the revision difference ' + 'is below this number (default: %(default)s)')) + p.add_argument('-v', + '--verbose', + action='store_true', + default=False, + help='Be extra verbose in printing of log messages.') + opts = p.parse_args() + + if opts.verbose: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + if not opts.ignore_unclean_workdir and not _IsTreeClean(): + logging.error('Please clean your local checkout first.') + return 1 + + if opts.clean: + _RemovePreviousRollBranch(opts.dry_run) + + if not opts.ignore_unclean_workdir: + _EnsureUpdatedMainBranch(opts.dry_run) + + deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS') + webrtc_deps = ParseLocalDepsFile(deps_filename) + + rev_update = GetRollRevisionRanges(opts, webrtc_deps) + + current_commit_pos = ParseCommitPosition( + ReadRemoteCrCommit(rev_update.current_chromium_rev)) + new_commit_pos = ParseCommitPosition( + ReadRemoteCrCommit(rev_update.new_chromium_rev)) + + new_cr_content = ReadRemoteCrFile('DEPS', rev_update.new_chromium_rev) + new_cr_deps = ParseDepsDict(new_cr_content) + changed_deps = CalculateChangedDeps(webrtc_deps, new_cr_deps) + # Discard other deps, assumed to be chromium-only dependencies. + new_generated_android_deps, _ = FindAddedDeps(webrtc_deps, new_cr_deps) + removed_generated_android_deps, other_deps = FindRemovedDeps( + webrtc_deps, new_cr_deps) + if other_deps: + raise RollError('WebRTC DEPS entries are missing from Chromium: %s.\n' + 'Remove them or add them to either ' + 'WEBRTC_ONLY_DEPS or DONT_AUTOROLL_THESE.' % other_deps) + clang_change = CalculateChangedClang(rev_update.new_chromium_rev) + commit_msg = GenerateCommitMessage( + rev_update, + current_commit_pos, + new_commit_pos, + changed_deps, + added_deps_paths=new_generated_android_deps, + removed_deps_paths=removed_generated_android_deps, + clang_change=clang_change) + logging.debug('Commit message:\n%s', commit_msg) + + _CreateRollBranch(opts.dry_run) + if not opts.dry_run: + UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content) + if _IsTreeClean(): + logging.info("No DEPS changes detected, skipping CL creation.") + else: + _LocalCommit(commit_msg, opts.dry_run) + commit_queue_mode = ChooseCQMode(opts.skip_cq, opts.cq_over, + current_commit_pos, new_commit_pos) + logging.info('Uploading CL...') + if not opts.dry_run: + _UploadCL(commit_queue_mode, _GetCcRecipients(changed_deps)) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/third_party/libwebrtc/tools_webrtc/autoroller/unittests/roll_deps_test.py b/third_party/libwebrtc/tools_webrtc/autoroller/unittests/roll_deps_test.py new file mode 100755 index 0000000000..b58f319eec --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/autoroller/unittests/roll_deps_test.py @@ -0,0 +1,366 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + + +import glob +import os +import shutil +import sys +import tempfile +import unittest +import mock + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +PARENT_DIR = os.path.join(SCRIPT_DIR, os.pardir) +sys.path.append(PARENT_DIR) + +import roll_deps +from roll_deps import CalculateChangedDeps, FindAddedDeps, \ + FindRemovedDeps, ChooseCQMode, GenerateCommitMessage, \ + GetMatchingDepsEntries, ParseDepsDict, ParseLocalDepsFile, UpdateDepsFile, \ + ChromiumRevisionUpdate + + +TEST_DATA_VARS = { + 'chromium_git': 'https://chromium.googlesource.com', + 'chromium_revision': '1b9c098a08e40114e44b6c1ec33ddf95c40b901d', +} + +DEPS_ENTRIES = { + 'src/build': 'https://build.com', + 'src/third_party/depot_tools': 'https://depottools.com', + 'src/testing/gtest': 'https://gtest.com', + 'src/testing/gmock': 'https://gmock.com', +} + +BUILD_OLD_REV = '52f7afeca991d96d68cf0507e20dbdd5b845691f' +BUILD_NEW_REV = 'HEAD' +DEPOTTOOLS_OLD_REV = 'b9ae2ca9a55d9b754c313f4c9e9f0f3b804a5e44' +DEPOTTOOLS_NEW_REV = '1206a353e40abb70d8454eb9af53db0ad10b713c' + +NO_CHROMIUM_REVISION_UPDATE = ChromiumRevisionUpdate('cafe', 'cafe') + + +class TestError(Exception): + pass + + +class FakeCmd: + def __init__(self): + self.expectations = [] + + def AddExpectation(self, *args, **kwargs): + returns = kwargs.pop('_returns', None) + ignores = kwargs.pop('_ignores', []) + self.expectations.append((args, kwargs, returns, ignores)) + + def __call__(self, *args, **kwargs): + if not self.expectations: + raise TestError('Got unexpected\n%s\n%s' % (args, kwargs)) + exp_args, exp_kwargs, exp_returns, ignores = self.expectations.pop(0) + for item in ignores: + kwargs.pop(item, None) + if args != exp_args or kwargs != exp_kwargs: + message = 'Expected:\n args: %s\n kwargs: %s\n' % (exp_args, exp_kwargs) + message += 'Got:\n args: %s\n kwargs: %s\n' % (args, kwargs) + raise TestError(message) + return exp_returns + + +class NullCmd: + """No-op mock when calls mustn't be checked. """ + + def __call__(self, *args, **kwargs): + # Empty stdout and stderr. + return None, None + + +class TestRollChromiumRevision(unittest.TestCase): + def setUp(self): + self._output_dir = tempfile.mkdtemp() + test_data_dir = os.path.join(SCRIPT_DIR, 'testdata', 'roll_deps') + for test_file in glob.glob(os.path.join(test_data_dir, '*')): + shutil.copy(test_file, self._output_dir) + join = lambda f: os.path.join(self._output_dir, f) + self._webrtc_depsfile = join('DEPS') + self._new_cr_depsfile = join('DEPS.chromium.new') + self._webrtc_depsfile_android = join('DEPS.with_android_deps') + self._new_cr_depsfile_android = join('DEPS.chromium.with_android_deps') + self.fake = FakeCmd() + + def tearDown(self): + shutil.rmtree(self._output_dir, ignore_errors=True) + self.assertEqual(self.fake.expectations, []) + + def testVarLookup(self): + local_scope = {'foo': 'wrong', 'vars': {'foo': 'bar'}} + lookup = roll_deps.VarLookup(local_scope) + self.assertEqual(lookup('foo'), 'bar') + + def testUpdateDepsFile(self): + new_rev = 'aaaaabbbbbcccccdddddeeeeefffff0000011111' + current_rev = TEST_DATA_VARS['chromium_revision'] + + with open(self._new_cr_depsfile_android, 'rb') as deps_file: + new_cr_contents = deps_file.read().decode('utf-8') + + UpdateDepsFile(self._webrtc_depsfile, + ChromiumRevisionUpdate(current_rev, new_rev), [], + new_cr_contents) + with open(self._webrtc_depsfile, 'rb') as deps_file: + deps_contents = deps_file.read().decode('utf-8') + self.assertTrue(new_rev in deps_contents, + 'Failed to find %s in\n%s' % (new_rev, deps_contents)) + + def _UpdateDepsSetup(self): + with open(self._webrtc_depsfile_android, 'rb') as deps_file: + webrtc_contents = deps_file.read().decode('utf-8') + with open(self._new_cr_depsfile_android, 'rb') as deps_file: + new_cr_contents = deps_file.read().decode('utf-8') + webrtc_deps = ParseDepsDict(webrtc_contents) + new_cr_deps = ParseDepsDict(new_cr_contents) + + changed_deps = CalculateChangedDeps(webrtc_deps, new_cr_deps) + with mock.patch('roll_deps._RunCommand', NullCmd()): + UpdateDepsFile(self._webrtc_depsfile_android, NO_CHROMIUM_REVISION_UPDATE, + changed_deps, new_cr_contents) + + with open(self._webrtc_depsfile_android, 'rb') as deps_file: + updated_contents = deps_file.read().decode('utf-8') + + return webrtc_contents, updated_contents + + def testUpdateAndroidGeneratedDeps(self): + _, updated_contents = self._UpdateDepsSetup() + + changed = 'third_party/android_deps/libs/android_arch_core_common' + changed_version = '1.0.0-cr0' + self.assertTrue(changed in updated_contents) + self.assertTrue(changed_version in updated_contents) + + def testAddAndroidGeneratedDeps(self): + webrtc_contents, updated_contents = self._UpdateDepsSetup() + + added = 'third_party/android_deps/libs/android_arch_lifecycle_common' + self.assertFalse(added in webrtc_contents) + self.assertTrue(added in updated_contents) + + def testRemoveAndroidGeneratedDeps(self): + webrtc_contents, updated_contents = self._UpdateDepsSetup() + + removed = 'third_party/android_deps/libs/android_arch_lifecycle_runtime' + self.assertTrue(removed in webrtc_contents) + self.assertFalse(removed in updated_contents) + + def testParseDepsDict(self): + with open(self._webrtc_depsfile, 'rb') as deps_file: + deps_contents = deps_file.read().decode('utf-8') + local_scope = ParseDepsDict(deps_contents) + vars_dict = local_scope['vars'] + + def AssertVar(variable_name): + self.assertEqual(vars_dict[variable_name], TEST_DATA_VARS[variable_name]) + + AssertVar('chromium_git') + AssertVar('chromium_revision') + self.assertEqual(len(local_scope['deps']), 3) + self.assertEqual(len(local_scope['deps_os']), 1) + + def testGetMatchingDepsEntriesReturnsPathInSimpleCase(self): + entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing/gtest') + self.assertEqual(len(entries), 1) + self.assertEqual(entries[0], DEPS_ENTRIES['src/testing/gtest']) + + def testGetMatchingDepsEntriesHandlesSimilarStartingPaths(self): + entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing') + self.assertEqual(len(entries), 2) + + def testGetMatchingDepsEntriesHandlesTwoPathsWithIdenticalFirstParts(self): + entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/build') + self.assertEqual(len(entries), 1) + + def testCalculateChangedDeps(self): + webrtc_deps = ParseLocalDepsFile(self._webrtc_depsfile) + new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile) + with mock.patch('roll_deps._RunCommand', self.fake): + _SetupGitLsRemoteCall( + self.fake, 'https://chromium.googlesource.com/chromium/src/build', + BUILD_NEW_REV) + changed_deps = CalculateChangedDeps(webrtc_deps, new_cr_deps) + + self.assertEqual(len(changed_deps), 3) + self.assertEqual(changed_deps[0].path, 'src/build') + self.assertEqual(changed_deps[0].current_rev, BUILD_OLD_REV) + self.assertEqual(changed_deps[0].new_rev, BUILD_NEW_REV) + + self.assertEqual(changed_deps[1].path, 'src/buildtools/linux64') + self.assertEqual(changed_deps[1].package, 'gn/gn/linux-amd64') + self.assertEqual(changed_deps[1].current_version, + 'git_revision:69ec4fca1fa69ddadae13f9e6b7507efa0675263') + self.assertEqual(changed_deps[1].new_version, 'git_revision:new-revision') + + self.assertEqual(changed_deps[2].path, 'src/third_party/depot_tools') + self.assertEqual(changed_deps[2].current_rev, DEPOTTOOLS_OLD_REV) + self.assertEqual(changed_deps[2].new_rev, DEPOTTOOLS_NEW_REV) + + def testWithDistinctDeps(self): + """Check CalculateChangedDeps works when deps are added/removed.""" + webrtc_deps = ParseLocalDepsFile(self._webrtc_depsfile_android) + new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile_android) + changed_deps = CalculateChangedDeps(webrtc_deps, new_cr_deps) + self.assertEqual(len(changed_deps), 1) + self.assertEqual( + changed_deps[0].path, + 'src/third_party/android_deps/libs/android_arch_core_common') + self.assertEqual( + changed_deps[0].package, + 'chromium/third_party/android_deps/libs/android_arch_core_common') + self.assertEqual(changed_deps[0].current_version, 'version:0.9.0') + self.assertEqual(changed_deps[0].new_version, 'version:1.0.0-cr0') + + def testFindAddedDeps(self): + webrtc_deps = ParseLocalDepsFile(self._webrtc_depsfile_android) + new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile_android) + added_android_paths, other_paths = FindAddedDeps(webrtc_deps, new_cr_deps) + self.assertEqual( + added_android_paths, + ['src/third_party/android_deps/libs/android_arch_lifecycle_common']) + self.assertEqual(other_paths, []) + + def testFindRemovedDeps(self): + webrtc_deps = ParseLocalDepsFile(self._webrtc_depsfile_android) + new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile_android) + removed_android_paths, other_paths = FindRemovedDeps( + webrtc_deps, new_cr_deps) + self.assertEqual( + removed_android_paths, + ['src/third_party/android_deps/libs/android_arch_lifecycle_runtime']) + self.assertEqual(other_paths, []) + + def testMissingDepsIsDetected(self): + """Check error is reported when deps cannot be automatically removed.""" + # The situation at test is the following: + # * A WebRTC DEPS entry is missing from Chromium. + # * The dependency isn't an android_deps (those are supported). + webrtc_deps = ParseLocalDepsFile(self._webrtc_depsfile) + new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile_android) + _, other_paths = FindRemovedDeps(webrtc_deps, new_cr_deps) + self.assertEqual(other_paths, + ['src/buildtools/linux64', 'src/third_party/depot_tools']) + + def testExpectedDepsIsNotReportedMissing(self): + """Some deps musn't be seen as missing, even if absent from Chromium.""" + webrtc_deps = ParseLocalDepsFile(self._webrtc_depsfile) + new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile_android) + removed_android_paths, other_paths = FindRemovedDeps( + webrtc_deps, new_cr_deps) + self.assertTrue('src/build' not in removed_android_paths) + self.assertTrue('src/build' not in other_paths) + + def _CommitMessageSetup(self): + webrtc_deps = ParseLocalDepsFile(self._webrtc_depsfile_android) + new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile_android) + + changed_deps = CalculateChangedDeps(webrtc_deps, new_cr_deps) + added_paths, _ = FindAddedDeps(webrtc_deps, new_cr_deps) + removed_paths, _ = FindRemovedDeps(webrtc_deps, new_cr_deps) + + current_commit_pos = 'cafe' + new_commit_pos = 'f00d' + + commit_msg = GenerateCommitMessage(NO_CHROMIUM_REVISION_UPDATE, + current_commit_pos, new_commit_pos, + changed_deps, added_paths, removed_paths) + + return [l.strip() for l in commit_msg.split('\n')] + + def testChangedDepsInCommitMessage(self): + commit_lines = self._CommitMessageSetup() + + changed = '* src/third_party/android_deps/libs/' \ + 'android_arch_core_common: version:0.9.0..version:1.0.0-cr0' + self.assertTrue(changed in commit_lines) + # Check it is in adequate section. + changed_line = commit_lines.index(changed) + self.assertTrue('Changed' in commit_lines[changed_line - 1]) + + def testAddedDepsInCommitMessage(self): + commit_lines = self._CommitMessageSetup() + + added = '* src/third_party/android_deps/libs/' \ + 'android_arch_lifecycle_common' + self.assertTrue(added in commit_lines) + # Check it is in adequate section. + added_line = commit_lines.index(added) + self.assertTrue('Added' in commit_lines[added_line - 1]) + + def testRemovedDepsInCommitMessage(self): + commit_lines = self._CommitMessageSetup() + + removed = '* src/third_party/android_deps/libs/' \ + 'android_arch_lifecycle_runtime' + self.assertTrue(removed in commit_lines) + # Check it is in adequate section. + removed_line = commit_lines.index(removed) + self.assertTrue('Removed' in commit_lines[removed_line - 1]) + + +class TestChooseCQMode(unittest.TestCase): + def testSkip(self): + self.assertEqual(ChooseCQMode(True, 99, 500000, 500100), 0) + + def testDryRun(self): + self.assertEqual(ChooseCQMode(False, 101, 500000, 500100), 1) + + def testSubmit(self): + self.assertEqual(ChooseCQMode(False, 100, 500000, 500100), 2) + + +class TestReadUrlContent(unittest.TestCase): + def setUp(self): + self.url = 'http://localhost+?format=TEXT' + + def testReadUrlContent(self): + url_mock = mock.Mock() + roll_deps.urllib.request.urlopen = url_mock + + roll_deps.ReadUrlContent(self.url) + + calls = [ + mock.call('http://localhost+?format=TEXT'), + mock.call().readlines(), + mock.call().close() + ] + self.assertEqual(url_mock.mock_calls, calls) + + def testReadUrlContentError(self): + roll_deps.logging = mock.Mock() + + readlines_mock = mock.Mock() + readlines_mock.readlines = mock.Mock( + side_effect=IOError('Connection error')) + readlines_mock.close = mock.Mock() + + url_mock = mock.Mock(return_value=readlines_mock) + roll_deps.urllib.request.urlopen = url_mock + + try: + roll_deps.ReadUrlContent(self.url) + except OSError: + self.assertTrue(roll_deps.logging.exception.called) + + +def _SetupGitLsRemoteCall(cmd_fake, url, revision): + cmd = ['git', 'ls-remote', url, revision] + cmd_fake.AddExpectation(cmd, _returns=(revision, None)) + + +if __name__ == '__main__': + unittest.main() diff --git a/third_party/libwebrtc/tools_webrtc/autoroller/unittests/testdata/roll_deps/DEPS b/third_party/libwebrtc/tools_webrtc/autoroller/unittests/testdata/roll_deps/DEPS new file mode 100644 index 0000000000..c39af68a9e --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/autoroller/unittests/testdata/roll_deps/DEPS @@ -0,0 +1,41 @@ +# DEPS file for unit tests. + +vars = { + 'chromium_git': 'https://chromium.googlesource.com', + 'chromium_revision': '1b9c098a08e40114e44b6c1ec33ddf95c40b901d', +} + +deps = { + # Entry that is a directory in Chromium, so we're using a Git subtree mirror for it. + 'src/build': + Var('chromium_git') + '/chromium/src/build' + '@' + '52f7afeca991d96d68cf0507e20dbdd5b845691f', + + # Entry that's also a DEPS entry in the Chromium DEPS file. + 'src/third_party/depot_tools': + Var('chromium_git') + '/chromium/tools/depot_tools.git' + '@' + 'b9ae2ca9a55d9b754c313f4c9e9f0f3b804a5e44', + + # Entry that's also a CIPD entry in the Chromium DEPS file. + 'src/buildtools/linux64': { + 'packages': [ + { + 'package': 'gn/gn/linux-amd64', + 'version': 'git_revision:69ec4fca1fa69ddadae13f9e6b7507efa0675263', + } + ], + 'dep_type': 'cipd', + 'condition': 'checkout_linux', + }, + + # Script expects to find these markers. + # === ANDROID_DEPS Generated Code Start === + # === ANDROID_DEPS Generated Code End === +} + +deps_os = { + # Entry only present in WebRTC, not Chromium. + 'android': { + 'src/examples/androidtests/third_party/gradle': + Var('chromium_git') + '/external/github.com/gradle/gradle.git' + '@' + + '89af43c4d0506f69980f00dde78c97b2f81437f8', + }, +} diff --git a/third_party/libwebrtc/tools_webrtc/autoroller/unittests/testdata/roll_deps/DEPS.chromium.new b/third_party/libwebrtc/tools_webrtc/autoroller/unittests/testdata/roll_deps/DEPS.chromium.new new file mode 100644 index 0000000000..7d40e425d5 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/autoroller/unittests/testdata/roll_deps/DEPS.chromium.new @@ -0,0 +1,28 @@ +# DEPS file for unit tests. + +vars = { + 'chromium_git': 'https://chromium.googlesource.com', + + # This is updated compared to the DEPS file. + 'depot_tools_revision': '1206a353e40abb70d8454eb9af53db0ad10b713c', +} + +deps = { + 'src/third_party/depot_tools': + Var('chromium_git') + '/chromium/tools/depot_tools.git' + '@' + Var('depot_tools_revision'), + + 'src/buildtools/linux64': { + 'packages': [ + { + 'package': 'gn/gn/linux-amd64', + 'version': 'git_revision:new-revision', + } + ], + 'dep_type': 'cipd', + 'condition': 'checkout_linux', + }, + + # Script expects to find these markers. + # === ANDROID_DEPS Generated Code Start === + # === ANDROID_DEPS Generated Code End === +} diff --git a/third_party/libwebrtc/tools_webrtc/autoroller/unittests/testdata/roll_deps/DEPS.chromium.with_android_deps b/third_party/libwebrtc/tools_webrtc/autoroller/unittests/testdata/roll_deps/DEPS.chromium.with_android_deps new file mode 100644 index 0000000000..5cad4fd0d1 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/autoroller/unittests/testdata/roll_deps/DEPS.chromium.with_android_deps @@ -0,0 +1,30 @@ +# DEPS file for unit tests. + +deps = { + # === ANDROID_DEPS Generated Code Start === + # Usually generated by //tools/android/roll/android_deps/fetch_all.py + 'src/third_party/android_deps/libs/android_arch_core_common': { + 'packages': [ + { + 'package': 'chromium/third_party/android_deps/libs/android_arch_core_common', + 'version': 'version:1.0.0-cr0', + }, + ], + 'condition': 'checkout_android', + 'dep_type': 'cipd', + }, + + 'src/third_party/android_deps/libs/android_arch_lifecycle_common': { + 'packages': [ + { + 'package': 'chromium/third_party/android_deps/libs/android_arch_lifecycle_common', + 'version': 'version:1.0.0-cr0', + }, + ], + 'condition': 'checkout_android', + 'dep_type': 'cipd', + }, + # === ANDROID_DEPS Generated Code End === + +} + diff --git a/third_party/libwebrtc/tools_webrtc/autoroller/unittests/testdata/roll_deps/DEPS.with_android_deps b/third_party/libwebrtc/tools_webrtc/autoroller/unittests/testdata/roll_deps/DEPS.with_android_deps new file mode 100644 index 0000000000..985ef290a0 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/autoroller/unittests/testdata/roll_deps/DEPS.with_android_deps @@ -0,0 +1,33 @@ +# DEPS file for unit tests. + +deps = { + # === ANDROID_DEPS Generated Code Start === + + # Version must be updated. + 'src/third_party/android_deps/libs/android_arch_core_common': { + 'packages': [ + { + 'package': 'chromium/third_party/android_deps/libs/android_arch_core_common', + 'version': 'version:0.9.0', + }, + ], + 'condition': 'checkout_android', + 'dep_type': 'cipd', + }, + + # Missing here: android_arch_lifecycle_common. Must be added automatically. + + # Missing in ref DEPS, must be removed automatically. + 'src/third_party/android_deps/libs/android_arch_lifecycle_runtime': { + 'packages': [ + { + 'package': 'chromium/third_party/android_deps/libs/android_arch_lifecycle_runtime', + 'version': 'version:1.0.0-cr0', + }, + ], + 'condition': 'checkout_android', + 'dep_type': 'cipd', + }, + # === ANDROID_DEPS Generated Code End === +} + diff --git a/third_party/libwebrtc/tools_webrtc/binary_version_check.py b/third_party/libwebrtc/tools_webrtc/binary_version_check.py new file mode 100644 index 0000000000..563fe36186 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/binary_version_check.py @@ -0,0 +1,36 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import re +import subprocess +import sys + +WEBRTC_VERSION_RE = re.compile( + r'WebRTC source stamp [0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}' +) + + +if __name__ == '__main__': + args = sys.argv + if len(args) != 2: + print('Usage: binary_version_test.py <FILE_NAME>') + sys.exit(1) + filename = sys.argv[1] + output = subprocess.check_output(['strings', filename]) + strings_in_binary = output.decode('utf-8').splitlines() + for symbol in strings_in_binary: + if WEBRTC_VERSION_RE.match(symbol): + with open('webrtc_binary_version_check', 'w') as f: + f.write(symbol) + sys.exit(0) + print('WebRTC source timestamp not found in "%s"' % filename) + print('Check why "kSourceTimestamp" from call/version.cc is not linked ' + '(or why it has been optimized away by the compiler/linker)') + sys.exit(1) diff --git a/third_party/libwebrtc/tools_webrtc/clang_tidy.py b/third_party/libwebrtc/tools_webrtc/clang_tidy.py new file mode 100755 index 0000000000..b3f738eab3 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/clang_tidy.py @@ -0,0 +1,97 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""Invoke clang-tidy tool. + +Usage: clang_tidy.py file.cc [clang-tidy-args...] + +Just a proof of concept! +We use an embedded clang-tidy whose version doesn't match clang++. +""" + +import argparse +import os +import shutil +import subprocess +import sys +import tempfile +from presubmit_checks_lib.build_helpers import (GetClangTidyPath, + GetCompilationCommand) + +# We enable all checkers by default for investigation purpose. +# This includes clang-analyzer-* checks. +# Individual checkers can be disabled via command line options. +# TODO(bugs.webrtc.com/10258): Select checkers relevant to webrtc guidelines. +CHECKER_OPTION = '-checks=*' + + +def Process(filepath, args): + # Build directory is needed to gather compilation flags. + # Create a temporary one (instead of reusing an existing one) + # to keep the CLI simple and unencumbered. + out_dir = tempfile.mkdtemp('clang_tidy') + + try: + gn_args = [] # Use default build. + command = GetCompilationCommand(filepath, gn_args, out_dir) + + # Remove warning flags. They aren't needed and they cause trouble + # when clang-tidy doesn't match most recent clang. + # Same battle for -f (e.g. -fcomplete-member-pointers). + command = [ + arg for arg in command + if not (arg.startswith('-W') or arg.startswith('-f')) + ] + + # Path from build dir. + rel_path = os.path.relpath(os.path.abspath(filepath), out_dir) + + # Replace clang++ by clang-tidy + command[0:1] = [GetClangTidyPath(), CHECKER_OPTION, rel_path + ] + args + ['--'] # Separator for clang flags. + print("Running: %s" % ' '.join(command)) + # Run from build dir so that relative paths are correct. + p = subprocess.Popen(command, + cwd=out_dir, + stdout=sys.stdout, + stderr=sys.stderr) + p.communicate() + return p.returncode + finally: + shutil.rmtree(out_dir, ignore_errors=True) + + +def ValidateCC(filepath): + """We can only analyze .cc files. Provide explicit message about that.""" + if filepath.endswith('.cc'): + return filepath + msg = ('%s not supported.\n' + 'For now, we can only analyze translation units (.cc files).' % + filepath) + raise argparse.ArgumentTypeError(msg) + + +def Main(): + description = ( + "Run clang-tidy on single cc file.\n" + "Use flags, defines and include paths as in default debug build.\n" + "WARNING, this is a POC version with rough edges.") + parser = argparse.ArgumentParser(description=description) + parser.add_argument('filepath', + help='Specifies the path of the .cc file to analyze.', + type=ValidateCC) + parser.add_argument('args', + nargs=argparse.REMAINDER, + help='Arguments passed to clang-tidy') + parsed_args = parser.parse_args() + return Process(parsed_args.filepath, parsed_args.args) + + +if __name__ == '__main__': + sys.exit(Main()) diff --git a/third_party/libwebrtc/tools_webrtc/coverage/generate_coverage_command.py b/third_party/libwebrtc/tools_webrtc/coverage/generate_coverage_command.py new file mode 100644 index 0000000000..650513ef1d --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/coverage/generate_coverage_command.py @@ -0,0 +1,47 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""Generates a command-line for coverage.py. Useful for manual coverage runs. + +Before running the generated command line, do this: + +gn gen out/coverage --args='use_clang_coverage=true is_component_build=false' +""" + +import sys + +TESTS = [ + 'video_capture_tests', 'webrtc_nonparallel_tests', 'video_engine_tests', + 'tools_unittests', 'test_support_unittests', + 'system_wrappers_unittests', 'rtc_unittests', 'rtc_stats_unittests', + 'rtc_pc_unittests', 'rtc_media_unittests', 'peerconnection_unittests', + 'modules_unittests', 'modules_tests', 'low_bandwidth_audio_test', + 'common_video_unittests', 'common_audio_unittests', + 'audio_decoder_unittests' +] + + +def main(): + cmd = ([sys.executable, 'tools/code_coverage/coverage.py'] + TESTS + + ['-b out/coverage', '-o out/report'] + + ['-i=\'.*/out/.*|.*/third_party/.*|.*test.*\''] + + ['-c \'out/coverage/%s\'' % t for t in TESTS]) + + def WithXvfb(binary): + return '-c \'%s testing/xvfb.py %s\'' % (sys.executable, binary) + + modules_unittests = 'out/coverage/modules_unittests' + cmd[cmd.index('-c \'%s\'' % modules_unittests)] = WithXvfb(modules_unittests) + + print(' '.join(cmd)) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/third_party/libwebrtc/tools_webrtc/coverage/generate_ios_coverage_command.py b/third_party/libwebrtc/tools_webrtc/coverage/generate_ios_coverage_command.py new file mode 100644 index 0000000000..249d8ce2a5 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/coverage/generate_ios_coverage_command.py @@ -0,0 +1,165 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""Generates command-line instructions to produce one-time iOS coverage using +coverage.py. + +This script is usable for both real devices and simulator. +But for real devices actual execution should be done manually from Xcode +and coverage.profraw files should be also copied manually from the device. + +Additional prerequisites: + +1. Xcode 10+ with iPhone Simulator SDK. Can be installed by command: + $ mac_toolchain install -kind ios -xcode-version 10l232m \ + -output-dir build/mac_files/Xcode.app + +2. For computing coverage on real device you probably also need to apply +following patch to code_coverage/coverage.py script: + +========== BEGINNING OF PATCH ========== +--- a/code_coverage/coverage.py ++++ b/code_coverage/coverage.py +@@ -693,8 +693,7 @@ def _AddArchArgumentForIOSIfNeeded(cmd_list, num_archs): + to use, and one architecture needs to be specified for each binary. + "" " +if _IsIOS(): +- cmd_list.extend(['-arch=x86_64'] * num_archs) ++ cmd_list.extend(['-arch=arm64'] * num_archs) + + +def _GetBinaryPath(command): +@@ -836,8 +835,8 @@ def _GetBinaryPathsFromTargets(targets, build_dir): + binary_path = os.path.join(build_dir, target) + if coverage_utils.GetHostPlatform() == 'win': + binary_path += '.exe' ++ elif coverage_utils.GetHostPlatform() == 'mac': ++ binary_path += '.app/%s' % target + +if os.path.exists(binary_path): + binary_paths.append(binary_path) +========== ENDING OF PATCH ========== + +""" +import sys + +DIRECTORY = 'out/coverage' + +TESTS = [ + 'audio_decoder_unittests', + 'common_audio_unittests', + 'common_video_unittests', + 'modules_tests', + 'modules_unittests', + 'rtc_media_unittests', + 'rtc_pc_unittests', + 'rtc_stats_unittests', + 'rtc_unittests', + 'system_wrappers_unittests', + 'test_support_unittests', + 'tools_unittests', + 'video_capture_tests', + 'video_engine_tests', + 'webrtc_nonparallel_tests', +] + +XC_TESTS = [ + 'apprtcmobile_tests', + 'sdk_framework_unittests', + 'sdk_unittests', +] + + +def FormatIossimTest(test_name, is_xctest=False): + args = ['%s/%s.app' % (DIRECTORY, test_name)] + if is_xctest: + args += ['%s/%s_module.xctest' % (DIRECTORY, test_name)] + + return '-c \'%s/iossim %s\'' % (DIRECTORY, ' '.join(args)) + + +def GetGNArgs(is_simulator): + target_cpu = 'x64' if is_simulator else 'arm64' + return ([] + ['target_os="ios"'] + ['target_cpu="%s"' % target_cpu] + + ['use_clang_coverage=true'] + ['is_component_build=false'] + + ['dcheck_always_on=true']) + + +def GenerateIOSSimulatorCommand(): + gn_args_string = ' '.join(GetGNArgs(is_simulator=True)) + gn_cmd = ['gn', 'gen', DIRECTORY, '--args=\'%s\'' % gn_args_string] + + coverage_cmd = ([sys.executable, 'tools/code_coverage/coverage.py'] + + ["%s.app" % t for t in XC_TESTS + TESTS] + + ['-b %s' % DIRECTORY, '-o out/report'] + + ['-i=\'.*/out/.*|.*/third_party/.*|.*test.*\''] + + [FormatIossimTest(t, is_xctest=True) for t in XC_TESTS] + + [FormatIossimTest(t, is_xctest=False) for t in TESTS]) + + print('To get code coverage using iOS sim just run following commands:') + print('') + print(' '.join(gn_cmd)) + print('') + print(' '.join(coverage_cmd)) + return 0 + + +def GenerateIOSDeviceCommand(): + gn_args_string = ' '.join(GetGNArgs(is_simulator=False)) + + coverage_report_cmd = ([sys.executable, 'tools/code_coverage/coverage.py'] + + ['%s.app' % t for t in TESTS] + ['-b %s' % DIRECTORY] + + ['-o out/report'] + + ['-p %s/merged.profdata' % DIRECTORY] + + ['-i=\'.*/out/.*|.*/third_party/.*|.*test.*\'']) + + print('Computing code coverage for real iOS device is a little bit tedious.') + print('') + print('You will need:') + print('') + print('1. Generate xcode project and open it with Xcode 10+:') + print(' gn gen %s --ide=xcode --args=\'%s\'' % (DIRECTORY, gn_args_string)) + print(' open %s/all.xcworkspace' % DIRECTORY) + print('') + print('2. Execute these Run targets manually with Xcode Run button and ') + print('manually save generated coverage.profraw file to %s:' % DIRECTORY) + print('\n'.join('- %s' % t for t in TESTS)) + print('') + print('3. Execute these Test targets manually with Xcode Test button and ') + print('manually save generated coverage.profraw file to %s:' % DIRECTORY) + print('\n'.join('- %s' % t for t in XC_TESTS)) + print('') + print('4. Merge *.profraw files to *.profdata using llvm-profdata tool:') + print((' build/mac_files/Xcode.app/Contents/Developer/Toolchains/' + + 'XcodeDefault.xctoolchain/usr/bin/llvm-profdata merge ' + + '-o %s/merged.profdata ' % DIRECTORY + + '-sparse=true %s/*.profraw' % DIRECTORY)) + print('') + print('5. Generate coverage report:') + print(' ' + ' '.join(coverage_report_cmd)) + return 0 + + +def main(): + if len(sys.argv) < 2: + print('Please specify type of coverage:') + print(' %s simulator' % sys.argv[0]) + print(' %s device' % sys.argv[0]) + elif sys.argv[1] == 'simulator': + GenerateIOSSimulatorCommand() + elif sys.argv[1] == 'device': + GenerateIOSDeviceCommand() + else: + print('Unsupported type of coverage') + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/third_party/libwebrtc/tools_webrtc/cpu/OWNERS b/third_party/libwebrtc/tools_webrtc/cpu/OWNERS new file mode 100644 index 0000000000..0fba125734 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/cpu/OWNERS @@ -0,0 +1 @@ +tommi@webrtc.org diff --git a/third_party/libwebrtc/tools_webrtc/cpu/README b/third_party/libwebrtc/tools_webrtc/cpu/README new file mode 100644 index 0000000000..a0cee30a82 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/cpu/README @@ -0,0 +1,12 @@ +This directory contains a little utility for doing CPU measurements. +It requires a Python package, psutil, to be installed. +See: https://pypi.python.org/pypi/psutil + +On mac you can install this package like so: +sudo ARCHFLAGS='-Wno-error=unused-command-line-argument-hard-error-in-future' easy_install psutil + +On Windows: +- TBD (see link above) + +On Linux: +- TBD (see link above) diff --git a/third_party/libwebrtc/tools_webrtc/cpu/cpu_mon.py b/third_party/libwebrtc/tools_webrtc/cpu/cpu_mon.py new file mode 100644 index 0000000000..9c25fbd088 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/cpu/cpu_mon.py @@ -0,0 +1,83 @@ +#!/usr/bin/env vpython3 +# +# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import sys + +import psutil +import numpy +from matplotlib import pyplot + + +class CpuSnapshot: + def __init__(self, label): + self.label = label + self.samples = [] + + def Capture(self, sample_count): + print(('Capturing %d CPU samples for %s...' % + ((sample_count - len(self.samples)), self.label))) + while len(self.samples) < sample_count: + self.samples.append(psutil.cpu_percent(1.0, False)) + + def Text(self): + return ( + '%s: avg=%s, median=%s, min=%s, max=%s' % + (self.label, numpy.average(self.samples), numpy.median( + self.samples), numpy.min(self.samples), numpy.max(self.samples))) + + def Max(self): + return numpy.max(self.samples) + + +def GrabCpuSamples(sample_count): + print('Label for snapshot (enter to quit): ') + label = eval(input().strip()) + if len(label) == 0: + return None + + snapshot = CpuSnapshot(label) + snapshot.Capture(sample_count) + + return snapshot + + +def main(): + print('How many seconds to capture per snapshot (enter for 60)?') + sample_count = eval(input().strip()) + if len(sample_count) > 0 and int(sample_count) > 0: + sample_count = int(sample_count) + else: + print('Defaulting to 60 samples.') + sample_count = 60 + + snapshots = [] + while True: + snapshot = GrabCpuSamples(sample_count) + if snapshot is None: + break + snapshots.append(snapshot) + + if len(snapshots) == 0: + print('no samples captured') + return -1 + + pyplot.title('CPU usage') + + for s in snapshots: + pyplot.plot(s.samples, label=s.Text(), linewidth=2) + + pyplot.legend() + + pyplot.show() + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/third_party/libwebrtc/tools_webrtc/download_tools.py b/third_party/libwebrtc/tools_webrtc/download_tools.py new file mode 100755 index 0000000000..16d7e5f3f0 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/download_tools.py @@ -0,0 +1,61 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""Downloads precompiled tools. + +These are checked into the repository as SHA-1 hashes (see *.sha1 files in +subdirectories). Note that chrome-webrtc-resources is a Google-internal bucket, +so please download and compile these tools manually if this script fails. +""" + +import os +import sys + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +SRC_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir)) +sys.path.append(os.path.join(SRC_DIR, 'build')) + +import find_depot_tools +find_depot_tools.add_depot_tools_to_path() +import gclient_utils +import subprocess2 + + +def main(directories): + if not directories: + directories = [SCRIPT_DIR] + + for path in directories: + cmd = [ + sys.executable, + os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, + 'download_from_google_storage.py'), + '--directory', + '--num_threads=10', + '--bucket', + 'chrome-webrtc-resources', + '--auto_platform', + '--recursive', + path, + ] + print('Downloading precompiled tools...') + + # Perform download similar to how gclient hooks execute. + try: + gclient_utils.CheckCallAndFilter(cmd, + cwd=SRC_DIR, + always_show_header=True) + except (gclient_utils.Error, subprocess2.CalledProcessError) as e: + print('Error: %s' % str(e)) + return 2 + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/third_party/libwebrtc/tools_webrtc/ensure_webcam_is_running.py b/third_party/libwebrtc/tools_webrtc/ensure_webcam_is_running.py new file mode 100755 index 0000000000..4428d79bd8 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/ensure_webcam_is_running.py @@ -0,0 +1,103 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""Checks if a virtual webcam is running and starts it if not. + +Returns a non-zero return code if the webcam could not be started. + +Prerequisites: +* The Python interpreter must have the psutil package installed. +* Windows: a scheduled task named 'ManyCam' must exist and be configured to + launch ManyCam preconfigured to auto-play the test clip. +* Mac: ManyCam must be installed in the default location and be preconfigured + to auto-play the test clip. +* Linux: Not implemented + +NOTICE: When running this script as a buildbot step, make sure to set +usePTY=False for the build step when adding it, or the subprocess will die as +soon the step has executed. + +If any command line arguments are passed to the script, it is executed as a +command in a subprocess. +""" + +import subprocess +import sys +# psutil is not installed on non-Linux machines by default. +import psutil # pylint: disable=F0401 + +WEBCAM_WIN = ('schtasks', '/run', '/tn', 'ManyCam') +WEBCAM_MAC = ('open', '/Applications/ManyCam/ManyCam.app') + + +def IsWebCamRunning(): + if sys.platform == 'win32': + process_name = 'ManyCam.exe' + elif sys.platform.startswith('darwin'): + process_name = 'ManyCam' + elif sys.platform.startswith('linux'): + # TODO(bugs.webrtc.org/9636): Currently a no-op on Linux: sw webcams no + # longer in use. + print('Virtual webcam: no-op on Linux') + return True + else: + raise Exception('Unsupported platform: %s' % sys.platform) + for p in psutil.process_iter(): + try: + if process_name == p.name: + print('Found a running virtual webcam (%s with PID %s)' % + (p.name, p.pid)) + return True + except psutil.AccessDenied: + pass # This is normal if we query sys processes, etc. + return False + + +def StartWebCam(): + try: + if sys.platform == 'win32': + subprocess.check_call(WEBCAM_WIN) + print('Successfully launched virtual webcam.') + elif sys.platform.startswith('darwin'): + subprocess.check_call(WEBCAM_MAC) + print('Successfully launched virtual webcam.') + elif sys.platform.startswith('linux'): + # TODO(bugs.webrtc.org/9636): Currently a no-op on Linux: sw webcams no + # longer in use. + print('Not implemented on Linux') + + except Exception as e: + print('Failed to launch virtual webcam: %s' % e) + return False + + return True + + +def _ForcePythonInterpreter(cmd): + """Returns the fixed command line to call the right python executable.""" + out = cmd[:] + if out[0] == 'vpython3': + out[0] = sys.executable + elif out[0].endswith('.py'): + out.insert(0, sys.executable) + return out + + +def Main(argv): + if not IsWebCamRunning(): + if not StartWebCam(): + return 1 + + if argv: + return subprocess.call(_ForcePythonInterpreter(argv)) + return 0 + + +if __name__ == '__main__': + sys.exit(Main(sys.argv[1:])) diff --git a/third_party/libwebrtc/tools_webrtc/executable_host_build.py b/third_party/libwebrtc/tools_webrtc/executable_host_build.py new file mode 100644 index 0000000000..e8c9aeb583 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/executable_host_build.py @@ -0,0 +1,100 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +""" +This script builds a GN executable targeting the host machine. + +It is useful, for example, for mobile devices performance testing where +it makes sense to build WebRTC for a mobile platform (e.g. Android) but +part of the test is performed on the host machine (e.g. running an +executable to analyze a video downloaded from a device). + +The script has only one (mandatory) option: --executable_name, which is +the output name of the GN executable. For example, if you have the +following executable in your out folder: + + out/Debug/random_exec + +You will be able to compile the same executable targeting your host machine +by running: + + $ vpython3 tools_webrtc/executable_host_build.py --executable_name random_exec + +The generated executable will have the same name as the input executable with +suffix '_host'. + +This script should not be used standalone but from GN, through an action: + + action("random_exec_host") { + script = "//tools_webrtc/executable_host_build.py" + outputs = [ + "${root_out_dir}/random_exec_host", + ] + args = [ + "--executable_name", + "random_exec", + ] + } + +The executable for the host machine will be generated in the GN out directory +(e.g. out/Debug in the previous example). +""" + +from contextlib import contextmanager + +import argparse +import os +import shutil +import subprocess +import sys +import tempfile + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +SRC_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, os.pardir)) +sys.path.append(os.path.join(SRC_DIR, 'build')) +import find_depot_tools + + +def _ParseArgs(): + desc = 'Generates a GN executable targeting the host machine.' + parser = argparse.ArgumentParser(description=desc) + parser.add_argument('--executable_name', + required=True, + help='Name of the executable to build') + args = parser.parse_args() + return args + + +@contextmanager +def HostBuildDir(): + temp_dir = tempfile.mkdtemp() + try: + yield temp_dir + finally: + shutil.rmtree(temp_dir) + + +def _RunCommand(argv, cwd=SRC_DIR, **kwargs): + with open(os.devnull, 'w') as devnull: + subprocess.check_call(argv, cwd=cwd, stdout=devnull, **kwargs) + + +def DepotToolPath(*args): + return os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, *args) + + +if __name__ == '__main__': + ARGS = _ParseArgs() + EXECUTABLE_TO_BUILD = ARGS.executable_name + EXECUTABLE_FINAL_NAME = ARGS.executable_name + '_host' + with HostBuildDir() as build_dir: + _RunCommand([sys.executable, DepotToolPath('gn.py'), 'gen', build_dir]) + _RunCommand([DepotToolPath('ninja'), '-C', build_dir, EXECUTABLE_TO_BUILD]) + shutil.copy(os.path.join(build_dir, EXECUTABLE_TO_BUILD), + EXECUTABLE_FINAL_NAME) diff --git a/third_party/libwebrtc/tools_webrtc/flags_compatibility.py b/third_party/libwebrtc/tools_webrtc/flags_compatibility.py new file mode 100755 index 0000000000..72c66a594f --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/flags_compatibility.py @@ -0,0 +1,51 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import argparse +import logging +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--isolated-script-test-output') + parser.add_argument('--isolated-script-test-perf-output') + args, unrecognized_args = parser.parse_known_args() + + test_command = _ForcePythonInterpreter(unrecognized_args) + if args.isolated_script_test_output: + test_command += [ + '--isolated_script_test_output', args.isolated_script_test_output + ] + if args.isolated_script_test_perf_output: + test_command += [ + '--isolated_script_test_perf_output=' + + args.isolated_script_test_perf_output + ] + logging.info('Running %r', test_command) + + return subprocess.call(test_command) + + +def _ForcePythonInterpreter(cmd): + """Returns the fixed command line to call the right python executable.""" + out = cmd[:] + if len(out) > 0: + if out[0] == 'python': + out[0] = sys.executable + elif out[0].endswith('.py'): + out.insert(0, sys.executable) + return out + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + sys.exit(main()) diff --git a/third_party/libwebrtc/tools_webrtc/get_landmines.py b/third_party/libwebrtc/tools_webrtc/get_landmines.py new file mode 100755 index 0000000000..45fd6af2f1 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/get_landmines.py @@ -0,0 +1,70 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +""" +This file emits the list of reasons why a particular build needs to be clobbered +(or a list of 'landmines'). +""" + +import os +import sys + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +CHECKOUT_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir)) +sys.path.insert(0, os.path.join(CHECKOUT_ROOT, 'build')) +import landmine_utils + +host_os = landmine_utils.host_os # pylint: disable=invalid-name + + +def print_landmines(): # pylint: disable=invalid-name + """ + ALL LANDMINES ARE EMITTED FROM HERE. + """ + # DO NOT add landmines as part of a regular CL. Landmines are a last-effort + # bandaid fix if a CL that got landed has a build dependency bug and all + # bots need to be cleaned up. If you're writing a new CL that causes build + # dependency problems, fix the dependency problems instead of adding a + # landmine. + # See the Chromium version in src/build/get_landmines.py for usage examples. + print('Clobber to remove out/{Debug,Release}/args.gn (webrtc:5070)') + if host_os() == 'win': + print('Clobber to resolve some issues with corrupt .pdb files on bots.') + print('Clobber due to corrupt .pdb files (after #14623)') + print('Clobber due to Win 64-bit Debug linking error (crbug.com/668961)') + print('Clobber due to Win Clang Debug linking errors in ' + 'https://codereview.webrtc.org/2786603002') + print('Clobber due to Win Debug linking errors in ' + 'https://codereview.webrtc.org/2832063003/') + print('Clobber win x86 bots (issues with isolated files).') + print('Clobber because of libc++ issue') + print('Clobber because of libc++ issue - take 2') + if host_os() == 'mac': + print('Clobber due to iOS compile errors (crbug.com/694721)') + print('Clobber to unblock https://codereview.webrtc.org/2709573003') + print('Clobber to fix https://codereview.webrtc.org/2709573003 after ' + 'landing') + print('Clobber to fix https://codereview.webrtc.org/2767383005 before' + 'landing (changing rtc_executable -> rtc_test on iOS)') + print('Clobber to fix https://codereview.webrtc.org/2767383005 before' + 'landing (changing rtc_executable -> rtc_test on iOS)') + print('Another landmine for low_bandwidth_audio_test (webrtc:7430)') + print('Clobber to change neteq_rtpplay type to executable') + print('Clobber to remove .xctest files.') + print('Clobber to remove .xctest files (take 2).') + print('Switching rtc_executable to rtc_test') + + +def main(): + print_landmines() + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/third_party/libwebrtc/tools_webrtc/gn_check_autofix.py b/third_party/libwebrtc/tools_webrtc/gn_check_autofix.py new file mode 100644 index 0000000000..c68e370037 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/gn_check_autofix.py @@ -0,0 +1,201 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +""" +This tool tries to fix (some) errors reported by `gn gen --check` or +`gn check`. +It will run `mb gen` in a temporary directory and it is really useful to +check for different configurations. + +Usage: + $ vpython3 tools_webrtc/gn_check_autofix.py -m some_mater -b some_bot + or + $ vpython3 tools_webrtc/gn_check_autofix.py -c some_mb_config +""" + +import os +import re +import shutil +import subprocess +import sys +import tempfile + +from collections import defaultdict + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + +CHROMIUM_DIRS = [ + 'base', 'build', 'buildtools', 'testing', 'third_party', 'tools' +] + +TARGET_RE = re.compile( + r'(?P<indentation_level>\s*)\w*\("(?P<target_name>\w*)"\) {$') + + +class TemporaryDirectory: + def __init__(self): + self._closed = False + self._name = None + self._name = tempfile.mkdtemp() + + def __enter__(self): + return self._name + + def __exit__(self, exc, value, _tb): + if self._name and not self._closed: + shutil.rmtree(self._name) + self._closed = True + + +def Run(cmd): + print('Running:', ' '.join(cmd)) + sub = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) + return sub.communicate() + + +def FixErrors(filename, missing_deps, deleted_sources): + with open(filename) as f: + lines = f.readlines() + + fixed_file = '' + indentation_level = None + for line in lines: + match = TARGET_RE.match(line) + if match: + target = match.group('target_name') + if target in missing_deps: + indentation_level = match.group('indentation_level') + elif indentation_level is not None: + match = re.match(indentation_level + '}$', line) + if match: + line = ('deps = [\n' + ''.join(' "' + dep + '",\n' + for dep in missing_deps[target]) + + ']\n') + line + indentation_level = None + elif line.strip().startswith('deps = ['): + joined_deps = ''.join(' "' + dep + '",\n' + for dep in missing_deps[target]) + line = line.replace('deps = [', 'deps = [' + joined_deps) + indentation_level = None + + if line.strip() not in deleted_sources: + fixed_file += line + + with open(filename, 'w') as f: + f.write(fixed_file) + + Run(['gn', 'format', filename]) + + +def FirstNonEmpty(iterable): + """Return first item which evaluates to True, or fallback to None.""" + return next((x for x in iterable if x), None) + + +def Rebase(base_path, dependency_path, dependency): + """Adapt paths so they work both in stand-alone WebRTC and Chromium tree. + + To cope with varying top-level directory (WebRTC VS Chromium), we use: + * relative paths for WebRTC modules. + * absolute paths for shared ones. + E.g. '//common_audio/...' -> '../../common_audio/' + '//third_party/...' remains as is. + + Args: + base_path: current module path (E.g. '//video') + dependency_path: path from root (E.g. '//rtc_base/time') + dependency: target itself (E.g. 'timestamp_extrapolator') + + Returns: + Full target path (E.g. '../rtc_base/time:timestamp_extrapolator'). + """ + + root = FirstNonEmpty(dependency_path.split('/')) + if root in CHROMIUM_DIRS: + # Chromium paths must remain absolute. E.g. //third_party//abseil-cpp... + rebased = dependency_path + else: + base_path = base_path.split(os.path.sep) + dependency_path = dependency_path.split(os.path.sep) + + first_difference = None + shortest_length = min(len(dependency_path), len(base_path)) + for i in range(shortest_length): + if dependency_path[i] != base_path[i]: + first_difference = i + break + + first_difference = first_difference or shortest_length + base_path = base_path[first_difference:] + dependency_path = dependency_path[first_difference:] + rebased = os.path.sep.join((['..'] * len(base_path)) + dependency_path) + return rebased + ':' + dependency + + +def main(): + deleted_sources = set() + errors_by_file = defaultdict(lambda: defaultdict(set)) + + with TemporaryDirectory() as tmp_dir: + mb_script_path = os.path.join(SCRIPT_DIR, 'mb', 'mb.py') + mb_config_file_path = os.path.join(SCRIPT_DIR, 'mb', 'mb_config.pyl') + mb_gen_command = ([ + mb_script_path, + 'gen', + tmp_dir, + '--config-file', + mb_config_file_path, + ] + sys.argv[1:]) + + mb_output = Run(mb_gen_command) + errors = mb_output[0].split('ERROR')[1:] + + if mb_output[1]: + print(mb_output[1]) + return 1 + + for error in errors: + error = error.split('\n') + target_msg = 'The target:' + if target_msg not in error: + target_msg = 'It is not in any dependency of' + if target_msg not in error: + print('\n'.join(error)) + continue + index = error.index(target_msg) + 1 + path, target = error[index].strip().split(':') + if error[index + 1] in ('is including a file from the target:', + 'The include file is in the target(s):'): + dep = error[index + 2].strip() + dep_path, dep = dep.split(':') + dep = Rebase(path, dep_path, dep) + # Replacing /target:target with /target + dep = re.sub(r'/(\w+):(\1)$', r'/\1', dep) + # Replacing target:target with target + dep = re.sub(r'^(\w+):(\1)$', r'\1', dep) + path = os.path.join(path[2:], 'BUILD.gn') + errors_by_file[path][target].add(dep) + elif error[index + 1] == 'has a source file:': + deleted_file = '"' + os.path.basename(error[index + 2].strip()) + '",' + deleted_sources.add(deleted_file) + else: + print('\n'.join(error)) + continue + + for path, missing_deps in list(errors_by_file.items()): + FixErrors(path, missing_deps, deleted_sources) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/third_party/libwebrtc/tools_webrtc/gtest-parallel-wrapper.py b/third_party/libwebrtc/tools_webrtc/gtest-parallel-wrapper.py new file mode 100755 index 0000000000..2972e6c9bf --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/gtest-parallel-wrapper.py @@ -0,0 +1,254 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +# pylint: disable=invalid-name +""" +This script acts as an interface between the Chromium infrastructure and +gtest-parallel, renaming options and translating environment variables into +flags. Developers should execute gtest-parallel directly. + +In particular, this translates the GTEST_SHARD_INDEX and GTEST_TOTAL_SHARDS +environment variables to the --shard_index and --shard_count flags, renames +the --isolated-script-test-output flag to --dump_json_test_results, +and interprets e.g. --workers=2x as 2 workers per core. + +Flags before '--' will be attempted to be understood as arguments to +gtest-parallel. If gtest-parallel doesn't recognize the flag or the flag is +after '--', the flag will be passed on to the test executable. + +--isolated-script-test-perf-output is renamed to +--isolated_script_test_perf_output. The Android test runner needs the flag to +be in the former form, but our tests require the latter, so this is the only +place we can do it. + +If the --store-test-artifacts flag is set, an --output_dir must be also +specified. + +The test artifacts will then be stored in a 'test_artifacts' subdirectory of the +output dir, and will be compressed into a zip file once the test finishes +executing. + +This is useful when running the tests in swarming, since the output directory +is not known beforehand. + +For example: + + gtest-parallel-wrapper.py some_test \ + --some_flag=some_value \ + --another_flag \ + --output_dir=SOME_OUTPUT_DIR \ + --store-test-artifacts + --isolated-script-test-output=SOME_DIR \ + --isolated-script-test-perf-output=SOME_OTHER_DIR \ + -- \ + --foo=bar \ + --baz + +Will be converted into: + + vpython3 gtest-parallel \ + --shard_index 0 \ + --shard_count 1 \ + --output_dir=SOME_OUTPUT_DIR \ + --dump_json_test_results=SOME_DIR \ + some_test \ + -- \ + --test_artifacts_dir=SOME_OUTPUT_DIR/test_artifacts \ + --some_flag=some_value \ + --another_flag \ + --isolated_script_test_perf_output=SOME_OTHER_DIR \ + --foo=bar \ + --baz + +""" + +import argparse +import collections +import multiprocessing +import os +import shutil +import subprocess +import sys + +Args = collections.namedtuple( + 'Args', + ['gtest_parallel_args', 'test_env', 'output_dir', 'test_artifacts_dir']) + + +def _CatFiles(file_list, output_file_destination): + with open(output_file_destination, 'w') as output_file: + for filename in file_list: + with open(filename) as input_file: + output_file.write(input_file.read()) + os.remove(filename) + + +def _ParseWorkersOption(workers): + """Interpret Nx syntax as N * cpu_count. Int value is left as is.""" + base = float(workers.rstrip('x')) + if workers.endswith('x'): + result = int(base * multiprocessing.cpu_count()) + else: + result = int(base) + return max(result, 1) # Sanitize when using e.g. '0.5x'. + + +class ReconstructibleArgumentGroup: + """An argument group that can be converted back into a command line. + + This acts like ArgumentParser.add_argument_group, but names of arguments added + to it are also kept in a list, so that parsed options from + ArgumentParser.parse_args can be reconstructed back into a command line (list + of args) based on the list of wanted keys.""" + + def __init__(self, parser, *args, **kwargs): + self._group = parser.add_argument_group(*args, **kwargs) + self._keys = [] + + def AddArgument(self, *args, **kwargs): + arg = self._group.add_argument(*args, **kwargs) + self._keys.append(arg.dest) + + def RemakeCommandLine(self, options): + result = [] + for key in self._keys: + value = getattr(options, key) + if value is True: + result.append('--%s' % key) + elif value is not None: + result.append('--%s=%s' % (key, value)) + return result + + +def ParseArgs(argv=None): + parser = argparse.ArgumentParser(argv) + + gtest_group = ReconstructibleArgumentGroup(parser, + 'Arguments to gtest-parallel') + # These options will be passed unchanged to gtest-parallel. + gtest_group.AddArgument('-d', '--output_dir') + gtest_group.AddArgument('-r', '--repeat') + # --isolated-script-test-output is used to upload results to the flakiness + # dashboard. This translation is made because gtest-parallel expects the flag + # to be called --dump_json_test_results instead. + gtest_group.AddArgument('--isolated-script-test-output', + dest='dump_json_test_results') + gtest_group.AddArgument('--retry_failed') + gtest_group.AddArgument('--gtest_color') + gtest_group.AddArgument('--gtest_filter') + gtest_group.AddArgument('--gtest_also_run_disabled_tests', + action='store_true', + default=None) + gtest_group.AddArgument('--timeout') + + # Syntax 'Nx' will be interpreted as N * number of cpu cores. + gtest_group.AddArgument('-w', '--workers', type=_ParseWorkersOption) + + # Needed when the test wants to store test artifacts, because it doesn't + # know what will be the swarming output dir. + parser.add_argument('--store-test-artifacts', action='store_true') + + # No-sandbox is a Chromium-specific flag, ignore it. + # TODO(bugs.webrtc.org/8115): Remove workaround when fixed. + parser.add_argument('--no-sandbox', + action='store_true', + help=argparse.SUPPRESS) + + parser.add_argument('executable') + parser.add_argument('executable_args', nargs='*') + + options, unrecognized_args = parser.parse_known_args(argv) + + webrtc_flags_to_change = { + '--isolated-script-test-perf-output': + '--isolated_script_test_perf_output', + '--isolated-script-test-output': '--isolated_script_test_output', + } + args_to_pass = [] + for arg in unrecognized_args: + if any(arg.startswith(k) for k in list(webrtc_flags_to_change.keys())): + arg_split = arg.split('=') + args_to_pass.append(webrtc_flags_to_change[arg_split[0]] + '=' + + arg_split[1]) + else: + args_to_pass.append(arg) + + executable_args = options.executable_args + args_to_pass + + if options.store_test_artifacts: + assert options.output_dir, ( + '--output_dir must be specified for storing test artifacts.') + test_artifacts_dir = os.path.join(options.output_dir, 'test_artifacts') + + executable_args.insert(0, '--test_artifacts_dir=%s' % test_artifacts_dir) + else: + test_artifacts_dir = None + + gtest_parallel_args = gtest_group.RemakeCommandLine(options) + + # GTEST_SHARD_INDEX and GTEST_TOTAL_SHARDS must be removed from the + # environment. Otherwise it will be picked up by the binary, causing a bug + # where only tests in the first shard are executed. + test_env = os.environ.copy() + gtest_shard_index = test_env.pop('GTEST_SHARD_INDEX', '0') + gtest_total_shards = test_env.pop('GTEST_TOTAL_SHARDS', '1') + + gtest_parallel_args.insert(0, '--shard_index=%s' % gtest_shard_index) + gtest_parallel_args.insert(1, '--shard_count=%s' % gtest_total_shards) + + gtest_parallel_args.append(options.executable) + if executable_args: + gtest_parallel_args += ['--'] + executable_args + + return Args(gtest_parallel_args, test_env, options.output_dir, + test_artifacts_dir) + + +def main(): + webrtc_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + gtest_parallel_path = os.path.join(webrtc_root, 'third_party', + 'gtest-parallel', 'gtest-parallel') + + gtest_parallel_args, test_env, output_dir, test_artifacts_dir = ParseArgs() + + command = [ + sys.executable, + gtest_parallel_path, + ] + gtest_parallel_args + + if output_dir and not os.path.isdir(output_dir): + os.makedirs(output_dir) + if test_artifacts_dir and not os.path.isdir(test_artifacts_dir): + os.makedirs(test_artifacts_dir) + + print('gtest-parallel-wrapper: Executing command %s' % ' '.join(command)) + sys.stdout.flush() + + exit_code = subprocess.call(command, env=test_env, cwd=os.getcwd()) + + if output_dir: + for test_status in 'passed', 'failed', 'interrupted': + logs_dir = os.path.join(output_dir, 'gtest-parallel-logs', test_status) + if not os.path.isdir(logs_dir): + continue + logs = [os.path.join(logs_dir, log) for log in os.listdir(logs_dir)] + log_file = os.path.join(output_dir, '%s-tests.log' % test_status) + _CatFiles(logs, log_file) + os.rmdir(logs_dir) + + if test_artifacts_dir: + shutil.make_archive(test_artifacts_dir, 'zip', test_artifacts_dir) + shutil.rmtree(test_artifacts_dir) + + return exit_code + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/third_party/libwebrtc/tools_webrtc/gtest_parallel_wrapper_test.py b/third_party/libwebrtc/tools_webrtc/gtest_parallel_wrapper_test.py new file mode 100755 index 0000000000..609052d248 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/gtest_parallel_wrapper_test.py @@ -0,0 +1,174 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +from contextlib import contextmanager + +import multiprocessing +import os +import tempfile +import unittest + +# pylint: disable=invalid-name +gtest_parallel_wrapper = __import__('gtest-parallel-wrapper') + + +@contextmanager +def TemporaryDirectory(): + tmp_dir = tempfile.mkdtemp() + yield tmp_dir + os.rmdir(tmp_dir) + + +class GtestParallelWrapperHelpersTest(unittest.TestCase): + def testGetWorkersAsIs(self): + # pylint: disable=protected-access + self.assertEqual(gtest_parallel_wrapper._ParseWorkersOption('12'), 12) + + def testGetTwiceWorkers(self): + expected = 2 * multiprocessing.cpu_count() + # pylint: disable=protected-access + self.assertEqual(gtest_parallel_wrapper._ParseWorkersOption('2x'), expected) + + def testGetHalfWorkers(self): + expected = max(multiprocessing.cpu_count() // 2, 1) + # pylint: disable=protected-access + self.assertEqual(gtest_parallel_wrapper._ParseWorkersOption('0.5x'), + expected) + + +class GtestParallelWrapperTest(unittest.TestCase): + @classmethod + def _Expected(cls, gtest_parallel_args): + return ['--shard_index=0', '--shard_count=1'] + gtest_parallel_args + + def testOverwrite(self): + result = gtest_parallel_wrapper.ParseArgs( + ['--timeout=123', 'exec', '--timeout', '124']) + expected = self._Expected(['--timeout=124', 'exec']) + self.assertEqual(result.gtest_parallel_args, expected) + + def testMixing(self): + result = gtest_parallel_wrapper.ParseArgs( + ['--timeout=123', '--param1', 'exec', '--param2', '--timeout', '124']) + expected = self._Expected( + ['--timeout=124', 'exec', '--', '--param1', '--param2']) + self.assertEqual(result.gtest_parallel_args, expected) + + def testMixingPositional(self): + result = gtest_parallel_wrapper.ParseArgs([ + '--timeout=123', 'exec', '--foo1', 'bar1', '--timeout', '124', '--foo2', + 'bar2' + ]) + expected = self._Expected( + ['--timeout=124', 'exec', '--', '--foo1', 'bar1', '--foo2', 'bar2']) + self.assertEqual(result.gtest_parallel_args, expected) + + def testDoubleDash1(self): + result = gtest_parallel_wrapper.ParseArgs( + ['--timeout', '123', 'exec', '--', '--timeout', '124']) + expected = self._Expected( + ['--timeout=123', 'exec', '--', '--timeout', '124']) + self.assertEqual(result.gtest_parallel_args, expected) + + def testDoubleDash2(self): + result = gtest_parallel_wrapper.ParseArgs( + ['--timeout=123', '--', 'exec', '--timeout=124']) + expected = self._Expected(['--timeout=123', 'exec', '--', '--timeout=124']) + self.assertEqual(result.gtest_parallel_args, expected) + + def testArtifacts(self): + with TemporaryDirectory() as tmp_dir: + output_dir = os.path.join(tmp_dir, 'foo') + result = gtest_parallel_wrapper.ParseArgs( + ['exec', '--store-test-artifacts', '--output_dir', output_dir]) + exp_artifacts_dir = os.path.join(output_dir, 'test_artifacts') + exp = self._Expected([ + '--output_dir=' + output_dir, 'exec', '--', + '--test_artifacts_dir=' + exp_artifacts_dir + ]) + self.assertEqual(result.gtest_parallel_args, exp) + self.assertEqual(result.output_dir, output_dir) + self.assertEqual(result.test_artifacts_dir, exp_artifacts_dir) + + def testNoDirsSpecified(self): + result = gtest_parallel_wrapper.ParseArgs(['exec']) + self.assertEqual(result.output_dir, None) + self.assertEqual(result.test_artifacts_dir, None) + + def testOutputDirSpecified(self): + result = gtest_parallel_wrapper.ParseArgs( + ['exec', '--output_dir', '/tmp/foo']) + self.assertEqual(result.output_dir, '/tmp/foo') + self.assertEqual(result.test_artifacts_dir, None) + + def testJsonTestResults(self): + result = gtest_parallel_wrapper.ParseArgs( + ['--isolated-script-test-output', '/tmp/foo', 'exec']) + expected = self._Expected(['--dump_json_test_results=/tmp/foo', 'exec']) + self.assertEqual(result.gtest_parallel_args, expected) + + def testShortArg(self): + result = gtest_parallel_wrapper.ParseArgs(['-d', '/tmp/foo', 'exec']) + expected = self._Expected(['--output_dir=/tmp/foo', 'exec']) + self.assertEqual(result.gtest_parallel_args, expected) + self.assertEqual(result.output_dir, '/tmp/foo') + + def testBoolArg(self): + result = gtest_parallel_wrapper.ParseArgs( + ['--gtest_also_run_disabled_tests', 'exec']) + expected = self._Expected(['--gtest_also_run_disabled_tests', 'exec']) + self.assertEqual(result.gtest_parallel_args, expected) + + def testNoArgs(self): + result = gtest_parallel_wrapper.ParseArgs(['exec']) + expected = self._Expected(['exec']) + self.assertEqual(result.gtest_parallel_args, expected) + + def testDocExample(self): + with TemporaryDirectory() as tmp_dir: + output_dir = os.path.join(tmp_dir, 'foo') + result = gtest_parallel_wrapper.ParseArgs([ + 'some_test', '--some_flag=some_value', '--another_flag', + '--output_dir=' + output_dir, '--store-test-artifacts', + '--isolated-script-test-output=SOME_DIR', + '--isolated-script-test-perf-output=SOME_OTHER_DIR', '--foo=bar', + '--baz' + ]) + expected_artifacts_dir = os.path.join(output_dir, 'test_artifacts') + expected = self._Expected([ + '--output_dir=' + output_dir, '--dump_json_test_results=SOME_DIR', + 'some_test', '--', '--test_artifacts_dir=' + expected_artifacts_dir, + '--some_flag=some_value', '--another_flag', + '--isolated_script_test_perf_output=SOME_OTHER_DIR', '--foo=bar', + '--baz' + ]) + self.assertEqual(result.gtest_parallel_args, expected) + + def testStandardWorkers(self): + """Check integer value is passed as-is.""" + result = gtest_parallel_wrapper.ParseArgs(['--workers', '17', 'exec']) + expected = self._Expected(['--workers=17', 'exec']) + self.assertEqual(result.gtest_parallel_args, expected) + + def testTwoWorkersPerCpuCore(self): + result = gtest_parallel_wrapper.ParseArgs(['--workers', '2x', 'exec']) + workers = 2 * multiprocessing.cpu_count() + expected = self._Expected(['--workers=%s' % workers, 'exec']) + self.assertEqual(result.gtest_parallel_args, expected) + + def testUseHalfTheCpuCores(self): + result = gtest_parallel_wrapper.ParseArgs(['--workers', '0.5x', 'exec']) + workers = max(multiprocessing.cpu_count() // 2, 1) + expected = self._Expected(['--workers=%s' % workers, 'exec']) + self.assertEqual(result.gtest_parallel_args, expected) + + +if __name__ == '__main__': + unittest.main() diff --git a/third_party/libwebrtc/tools_webrtc/ios/OWNERS b/third_party/libwebrtc/tools_webrtc/ios/OWNERS new file mode 100644 index 0000000000..cd06158b7f --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/ios/OWNERS @@ -0,0 +1 @@ +tkchin@webrtc.org diff --git a/third_party/libwebrtc/tools_webrtc/ios/build_ios_libs.py b/third_party/libwebrtc/tools_webrtc/ios/build_ios_libs.py new file mode 100755 index 0000000000..01a95b08c4 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/ios/build_ios_libs.py @@ -0,0 +1,340 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""WebRTC iOS XCFramework build script. +Each architecture is compiled separately before being merged together. +By default, the library is created in out_ios_libs/. (Change with -o.) +""" + +import argparse +import logging +import os +import shutil +import subprocess +import sys + +os.environ['PATH'] = '/usr/libexec' + os.pathsep + os.environ['PATH'] + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +SRC_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, '..', '..')) +sys.path.append(os.path.join(SRC_DIR, 'build')) +import find_depot_tools + +SDK_OUTPUT_DIR = os.path.join(SRC_DIR, 'out_ios_libs') +SDK_FRAMEWORK_NAME = 'WebRTC.framework' +SDK_DSYM_NAME = 'WebRTC.dSYM' +SDK_XCFRAMEWORK_NAME = 'WebRTC.xcframework' + +ENABLED_ARCHS = [ + 'device:arm64', 'simulator:arm64', 'simulator:x64', + 'catalyst:arm64', 'catalyst:x64', + 'arm64', 'x64' +] +DEFAULT_ARCHS = [ + 'device:arm64', 'simulator:arm64', 'simulator:x64' +] +IOS_DEPLOYMENT_TARGET = { + 'device': '12.0', + 'simulator': '12.0', + 'catalyst': '14.0' +} +LIBVPX_BUILD_VP9 = False + +sys.path.append(os.path.join(SCRIPT_DIR, '..', 'libs')) +from generate_licenses import LicenseBuilder + + +def _ParseArgs(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--build_config', + default='release', + choices=['debug', 'release'], + help='The build config. Can be "debug" or "release". ' + 'Defaults to "release".') + parser.add_argument('--arch', + nargs='+', + default=DEFAULT_ARCHS, + choices=ENABLED_ARCHS, + help='Architectures to build. Defaults to %(default)s.') + parser.add_argument( + '-c', + '--clean', + action='store_true', + default=False, + help='Removes the previously generated build output, if any.') + parser.add_argument('-p', + '--purify', + action='store_true', + default=False, + help='Purifies the previously generated build output by ' + 'removing the temporary results used when (re)building.') + parser.add_argument( + '-o', + '--output-dir', + type=os.path.abspath, + default=SDK_OUTPUT_DIR, + help='Specifies a directory to output the build artifacts to. ' + 'If specified together with -c, deletes the dir.') + parser.add_argument( + '-r', + '--revision', + type=int, + default=0, + help='Specifies a revision number to embed if building the framework.') + parser.add_argument('--verbose', + action='store_true', + default=False, + help='Debug logging.') + parser.add_argument('--use-goma', + action='store_true', + default=False, + help='Use goma to build.') + parser.add_argument( + '--extra-gn-args', + default=[], + nargs='*', + help='Additional GN args to be used during Ninja generation.') + + return parser.parse_args() + + +def _RunCommand(cmd): + logging.debug('Running: %r', cmd) + subprocess.check_call(cmd, cwd=SRC_DIR) + + +def _CleanArtifacts(output_dir): + if os.path.isdir(output_dir): + logging.info('Deleting %s', output_dir) + shutil.rmtree(output_dir) + + +def _CleanTemporary(output_dir, architectures): + if os.path.isdir(output_dir): + logging.info('Removing temporary build files.') + for arch in architectures: + arch_lib_path = os.path.join(output_dir, arch) + if os.path.isdir(arch_lib_path): + shutil.rmtree(arch_lib_path) + + +def _ParseArchitecture(architectures): + result = dict() + for arch in architectures: + if ":" in arch: + target_environment, target_cpu = arch.split(":") + else: + logging.warning('The environment for build is not specified.') + logging.warning('It is assumed based on cpu type.') + logging.warning('See crbug.com/1138425 for more details.') + if arch == "x64": + target_environment = "simulator" + else: + target_environment = "device" + target_cpu = arch + archs = result.get(target_environment) + if archs is None: + result[target_environment] = {target_cpu} + else: + archs.add(target_cpu) + + return result + + +def BuildWebRTC(output_dir, target_environment, target_arch, flavor, + gn_target_name, ios_deployment_target, libvpx_build_vp9, + use_goma, extra_gn_args): + gn_args = [ + 'target_os="ios"', + 'ios_enable_code_signing=false', + 'is_component_build=false', + 'rtc_include_tests=false', + ] + + # Add flavor option. + if flavor == 'debug': + gn_args.append('is_debug=true') + elif flavor == 'release': + gn_args.append('is_debug=false') + else: + raise ValueError('Unexpected flavor type: %s' % flavor) + + gn_args.append('target_environment="%s"' % target_environment) + + gn_args.append('target_cpu="%s"' % target_arch) + + gn_args.append('ios_deployment_target="%s"' % ios_deployment_target) + + gn_args.append('rtc_libvpx_build_vp9=' + + ('true' if libvpx_build_vp9 else 'false')) + + gn_args.append('use_lld=true') + gn_args.append('use_goma=' + ('true' if use_goma else 'false')) + gn_args.append('rtc_enable_objc_symbol_export=true') + + args_string = ' '.join(gn_args + extra_gn_args) + logging.info('Building WebRTC with args: %s', args_string) + + cmd = [ + sys.executable, + os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py'), + 'gen', + output_dir, + '--args=' + args_string, + ] + _RunCommand(cmd) + logging.info('Building target: %s', gn_target_name) + + cmd = [ + os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'ninja'), + '-C', + output_dir, + gn_target_name, + ] + if use_goma: + cmd.extend(['-j', '200']) + _RunCommand(cmd) + + +def main(): + args = _ParseArgs() + + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) + + if args.clean: + _CleanArtifacts(args.output_dir) + return 0 + + # architectures is typed as Dict[str, Set[str]], + # where key is for the environment (device or simulator) + # and value is for the cpu type. + architectures = _ParseArchitecture(args.arch) + gn_args = args.extra_gn_args + + if args.purify: + _CleanTemporary(args.output_dir, list(architectures.keys())) + return 0 + + gn_target_name = 'framework_objc' + gn_args.append('enable_dsyms=true') + gn_args.append('enable_stripping=true') + + # Build all architectures. + framework_paths = [] + all_lib_paths = [] + for (environment, archs) in list(architectures.items()): + framework_path = os.path.join(args.output_dir, environment) + framework_paths.append(framework_path) + lib_paths = [] + for arch in archs: + lib_path = os.path.join(framework_path, arch + '_libs') + lib_paths.append(lib_path) + BuildWebRTC(lib_path, environment, arch, args.build_config, + gn_target_name, IOS_DEPLOYMENT_TARGET[environment], + LIBVPX_BUILD_VP9, args.use_goma, gn_args) + all_lib_paths.extend(lib_paths) + + # Combine the slices. + dylib_path = os.path.join(SDK_FRAMEWORK_NAME, 'WebRTC') + # Dylibs will be combined, all other files are the same across archs. + shutil.rmtree(os.path.join(framework_path, SDK_FRAMEWORK_NAME), + ignore_errors=True) + shutil.copytree(os.path.join(lib_paths[0], SDK_FRAMEWORK_NAME), + os.path.join(framework_path, SDK_FRAMEWORK_NAME), + symlinks=True) + logging.info('Merging framework slices for %s.', environment) + dylib_paths = [os.path.join(path, dylib_path) for path in lib_paths] + out_dylib_path = os.path.join(framework_path, dylib_path) + if os.path.islink(out_dylib_path): + out_dylib_path = os.path.join(os.path.dirname(out_dylib_path), + os.readlink(out_dylib_path)) + try: + os.remove(out_dylib_path) + except OSError: + pass + cmd = ['lipo'] + dylib_paths + ['-create', '-output', out_dylib_path] + _RunCommand(cmd) + + # Merge the dSYM slices. + lib_dsym_dir_path = os.path.join(lib_paths[0], SDK_DSYM_NAME) + if os.path.isdir(lib_dsym_dir_path): + shutil.rmtree(os.path.join(framework_path, SDK_DSYM_NAME), + ignore_errors=True) + shutil.copytree(lib_dsym_dir_path, + os.path.join(framework_path, SDK_DSYM_NAME)) + logging.info('Merging dSYM slices.') + dsym_path = os.path.join(SDK_DSYM_NAME, 'Contents', 'Resources', 'DWARF', + 'WebRTC') + lib_dsym_paths = [os.path.join(path, dsym_path) for path in lib_paths] + out_dsym_path = os.path.join(framework_path, dsym_path) + try: + os.remove(out_dsym_path) + except OSError: + pass + cmd = ['lipo'] + lib_dsym_paths + ['-create', '-output', out_dsym_path] + _RunCommand(cmd) + + # Check for Mac-style WebRTC.framework/Resources/ (for Catalyst)... + resources_dir = os.path.join(framework_path, SDK_FRAMEWORK_NAME, + 'Resources') + if not os.path.exists(resources_dir): + # ...then fall back to iOS-style WebRTC.framework/ + resources_dir = os.path.dirname(resources_dir) + + # Modify the version number. + # Format should be <Branch cut MXX>.<Hotfix #>.<Rev #>. + # e.g. 55.0.14986 means + # branch cut 55, no hotfixes, and revision 14986. + infoplist_path = os.path.join(resources_dir, 'Info.plist') + cmd = [ + 'PlistBuddy', '-c', 'Print :CFBundleShortVersionString', + infoplist_path + ] + major_minor = subprocess.check_output(cmd).decode('utf-8').strip() + version_number = '%s.%s' % (major_minor, args.revision) + logging.info('Substituting revision number: %s', version_number) + cmd = [ + 'PlistBuddy', '-c', 'Set :CFBundleVersion ' + version_number, + infoplist_path + ] + _RunCommand(cmd) + _RunCommand(['plutil', '-convert', 'binary1', infoplist_path]) + + xcframework_dir = os.path.join(args.output_dir, SDK_XCFRAMEWORK_NAME) + if os.path.isdir(xcframework_dir): + shutil.rmtree(xcframework_dir) + + logging.info('Creating xcframework.') + cmd = ['xcodebuild', '-create-xcframework', '-output', xcframework_dir] + + # Apparently, xcodebuild needs absolute paths for input arguments + for framework_path in framework_paths: + cmd += [ + '-framework', + os.path.abspath(os.path.join(framework_path, SDK_FRAMEWORK_NAME)), + ] + dsym_full_path = os.path.join(framework_path, SDK_DSYM_NAME) + if os.path.exists(dsym_full_path): + cmd += ['-debug-symbols', os.path.abspath(dsym_full_path)] + + _RunCommand(cmd) + + # Generate the license file. + logging.info('Generate license file.') + gn_target_full_name = '//sdk:' + gn_target_name + builder = LicenseBuilder(all_lib_paths, [gn_target_full_name]) + builder.GenerateLicenseText( + os.path.join(args.output_dir, SDK_XCFRAMEWORK_NAME)) + + logging.info('Done.') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/third_party/libwebrtc/tools_webrtc/ios/build_ios_libs.sh b/third_party/libwebrtc/tools_webrtc/ios/build_ios_libs.sh new file mode 100755 index 0000000000..c8d659c7e6 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/ios/build_ios_libs.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Copyright 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +# This script has been rewritten in Python. Temporary "redirect": + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +exec "$SCRIPT_DIR/build_ios_libs.py" "$@" diff --git a/third_party/libwebrtc/tools_webrtc/ios/generate_modulemap.py b/third_party/libwebrtc/tools_webrtc/ios/generate_modulemap.py new file mode 100644 index 0000000000..1b61b8e3d1 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/ios/generate_modulemap.py @@ -0,0 +1,34 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import argparse +import sys + + +def GenerateModulemap(): + parser = argparse.ArgumentParser(description='Generate modulemap') + parser.add_argument("-o", "--out", type=str, help="Output file.") + parser.add_argument("-n", "--name", type=str, help="Name of binary.") + + args = parser.parse_args() + + with open(args.out, "w") as outfile: + module_template = 'framework module %s {\n' \ + ' umbrella header "%s.h"\n' \ + '\n' \ + ' export *\n' \ + ' module * { export * }\n' \ + '}\n' % (args.name, args.name) + outfile.write(module_template) + return 0 + + +if __name__ == '__main__': + sys.exit(GenerateModulemap()) diff --git a/third_party/libwebrtc/tools_webrtc/ios/generate_umbrella_header.py b/third_party/libwebrtc/tools_webrtc/ios/generate_umbrella_header.py new file mode 100644 index 0000000000..1fd1eed38e --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/ios/generate_umbrella_header.py @@ -0,0 +1,50 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import argparse +import datetime +import os +import sys +import textwrap + + +def GenerateUmbrellaHeader(): + parser = argparse.ArgumentParser(description='Generate umbrella header') + parser.add_argument("-o", "--out", type=str, help="Output file.") + parser.add_argument("-s", + "--sources", + default=[], + type=str, + nargs='+', + help="Headers to include.") + + args = parser.parse_args() + + with open(args.out, "w") as outfile: + outfile.write( + textwrap.dedent("""\ + /* + * Copyright %d The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */\n\n""" % datetime.datetime.now().year)) + + for s in args.sources: + outfile.write("#import <WebRTC/{}>\n".format(os.path.basename(s))) + + return 0 + + +if __name__ == '__main__': + sys.exit(GenerateUmbrellaHeader()) diff --git a/third_party/libwebrtc/tools_webrtc/ios/merge_ios_libs.py b/third_party/libwebrtc/tools_webrtc/ios/merge_ios_libs.py new file mode 100755 index 0000000000..111825155e --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/ios/merge_ios_libs.py @@ -0,0 +1,126 @@ +#!/usr/bin/env vpython3 + +# Copyright 2016 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""Script for merging generated iOS libraries.""" + +import sys +import argparse +import os +import re +import subprocess +from six.moves import range + + +# Valid arch subdir names. +VALID_ARCHS = ['arm_libs', 'arm64_libs', 'ia32_libs', 'x64_libs'] + + +def MergeLibs(lib_base_dir): + """Merges generated iOS libraries for different archs. + + Uses libtool to generate FAT archive files for each generated library. + + Args: + lib_base_dir: directory whose subdirectories are named by architecture and + contain the built libraries for that architecture + + Returns: + Exit code of libtool. + """ + output_dir_name = 'fat_libs' + archs = [arch for arch in os.listdir(lib_base_dir) if arch in VALID_ARCHS] + # For each arch, find (library name, libary path) for arch. We will merge + # all libraries with the same name. + libs = {} + for lib_dir in [os.path.join(lib_base_dir, arch) for arch in VALID_ARCHS]: + if not os.path.exists(lib_dir): + continue + for dirpath, _, filenames in os.walk(lib_dir): + for filename in filenames: + if not filename.endswith('.a'): + continue + entry = libs.get(filename, []) + entry.append(os.path.join(dirpath, filename)) + libs[filename] = entry + orphaned_libs = {} + valid_libs = {} + for library, paths in list(libs.items()): + if len(paths) < len(archs): + orphaned_libs[library] = paths + else: + valid_libs[library] = paths + for library, paths in list(orphaned_libs.items()): + components = library[:-2].split('_')[:-1] + found = False + # Find directly matching parent libs by stripping suffix. + while components and not found: + parent_library = '_'.join(components) + '.a' + if parent_library in valid_libs: + valid_libs[parent_library].extend(paths) + found = True + break + components = components[:-1] + # Find next best match by finding parent libs with the same prefix. + if not found: + base_prefix = library[:-2].split('_')[0] + for valid_lib, valid_paths in list(valid_libs.items()): + if valid_lib[:len(base_prefix)] == base_prefix: + valid_paths.extend(paths) + found = True + break + assert found + + # Create output directory. + output_dir_path = os.path.join(lib_base_dir, output_dir_name) + if not os.path.exists(output_dir_path): + os.mkdir(output_dir_path) + + # Use this so libtool merged binaries are always the same. + env = os.environ.copy() + env['ZERO_AR_DATE'] = '1' + + # Ignore certain errors. + libtool_re = re.compile(r'^.*libtool:.*file: .* has no symbols$') + + # Merge libraries using libtool. + libtool_returncode = 0 + for library, paths in list(valid_libs.items()): + cmd_list = [ + 'libtool', '-static', '-v', '-o', + os.path.join(output_dir_path, library) + ] + paths + libtoolout = subprocess.Popen(cmd_list, stderr=subprocess.PIPE, env=env) + _, err = libtoolout.communicate() + for line in err.splitlines(): + if not libtool_re.match(line): + print(line, file=sys.stderr) + # Unconditionally touch the output .a file on the command line if present + # and the command succeeded. A bit hacky. + libtool_returncode = libtoolout.returncode + if not libtool_returncode: + for i in range(len(cmd_list) - 1): + if cmd_list[i] == '-o' and cmd_list[i + 1].endswith('.a'): + os.utime(cmd_list[i + 1], None) + break + return libtool_returncode + + +def main(): + parser_description = 'Merge WebRTC libraries.' + parser = argparse.ArgumentParser(description=parser_description) + parser.add_argument('lib_base_dir', + help='Directory with built libraries. ', + type=str) + args = parser.parse_args() + lib_base_dir = args.lib_base_dir + MergeLibs(lib_base_dir) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/third_party/libwebrtc/tools_webrtc/ios/no_op.cc b/third_party/libwebrtc/tools_webrtc/ios/no_op.cc new file mode 100644 index 0000000000..7508b9dbb4 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/ios/no_op.cc @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// No-op main() to provide a dummy executable target. +int main() { + return 0; +} diff --git a/third_party/libwebrtc/tools_webrtc/ios/objc_app.plist b/third_party/libwebrtc/tools_webrtc/ios/objc_app.plist new file mode 100644 index 0000000000..c2fb0617f3 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/ios/objc_app.plist @@ -0,0 +1,24 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<dict> + <key>CFBundleDevelopmentRegion</key> + <string>en</string> + <key>CFBundleDisplayName</key> + <string>${PRODUCT_NAME}</string> + <key>CFBundleExecutable</key> + <string>${EXECUTABLE_NAME}</string> + <key>CFBundleIdentifier</key> + <string>com.Google.${PRODUCT_NAME:rfc1034identifier}</string> + <key>CFBundleInfoDictionaryVersion</key> + <string>6.0</string> + <key>CFBundleName</key> + <string>${PRODUCT_NAME}</string> + <key>CFBundlePackageType</key> + <string>APPL</string> + <key>CFBundleShortVersionString</key> + <string>1.0</string> + <key>CFBundleVersion</key> + <string>1.0</string> +</dict> +</plist> diff --git a/third_party/libwebrtc/tools_webrtc/iwyu/apply-iwyu b/third_party/libwebrtc/tools_webrtc/iwyu/apply-iwyu new file mode 100755 index 0000000000..794756ba47 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/iwyu/apply-iwyu @@ -0,0 +1,114 @@ +#!/bin/sh +# +# Run the include-what-you-use tool (iwyu) on a file in the webrtc source +# directory. +# +# The script uses a subsequent grep pass to remove #include files +# that are problematic to include. +# +# In order to handle include paths correctly, you need to provide +# a compile DB (aka compile_commands.json). +# You can create it in one of the following ways: +# "gn gen --export-compile-commands path/to/out" +# "tools/clang/scripts/generate_compdb.py -p path/to/out > compile_commands.json" +# If "out/Default" exists, the script will attempt to generate it for you. +# +# To get iwyu on Debian/glinux, do "sudo apt-get install iwyu". + +# Set this to 1 to get more debug information. +# Set this to 2 to also get a dump of the iwyu tool output. +DEBUG=0 + +set -e +if [ $DEBUG -gt 0 ]; then + set -x +fi + +IWYU_TOOL="${IWYU_TOOL:-/usr/bin/iwyu_tool}" +FIX_INCLUDE="${FIX_INCLUDE:-/usr/bin/fix_include}" +FIX_INCLUDE_ARGS='' +IWYU_TOOL_DIR="${IWYU_TOOL_DIR:-tools_webrtc/iwyu}" +COMPILE_COMMANDS='' + +error() { + echo "$*" >&2 + exit 1 +} + +usage() { + echo "Usage: $0 [ -c compile-commands-file.json ] [-r] file.cc" + echo "Runs the IWYU and fix-include on a CC file and its associated .h file" + echo "Arguments:" + echo " -c compile-commands: Compiler command file" + echo " -r : Remove non-required includes from .h file" + echo " -h : Print this help message" + echo "(default command file: out/Default/compile_commands.json - this" + echo "will be generated if not present" +} + +while getopts 'c:rh' opts; do + case "${opts}" in + c) COMPILE_COMMANDS="${OPTARG}" ;; + r) FIX_INCLUDE_ARGS="${FIX_INCLUDE_ARGS} --nosafe_headers" ;; + h) usage; exit 1 ;; + *) error "Unexpected option ${opts}" ;; + esac +done +shift $(expr $OPTIND - 1 ) + +if [[ -z "$COMPILE_COMMANDS" ]]; then + if [ -d "out/Default" ]; then + if [ ! -f "out/Default/compile_commands.json" ]; then + gn gen --export-compile-commands out/Default + fi + COMPILE_COMMANDS="out/Default/compile_commands.json" + else + error "compile_commands.json must be passed." + fi +fi + +FILE="$1" + +if [ ! -f $FILE_CC ]; then + error "File $FILE is not found" +fi + +# Find the .h file that IWYU will modify, if any. +FILE_CC=$FILE +if [ -f $(dirname $FILE)/$(basename -s .cc $FILE).h ]; then + FILE_H=$(dirname $FILE)/$(basename -s .cc $FILE).h +else + FILE_H="" +fi + +# IWYU has a confusing set of exit codes. Discard it. +"$IWYU_TOOL" -p "$COMPILE_COMMANDS" "$FILE_CC" -- -Xiwyu --no_fwd_decls \ + -Xiwyu --mapping_file=../../$IWYU_TOOL_DIR/mappings.imp \ + >& /tmp/includefixes$$ || echo "IWYU done, code $?" + +if grep 'fatal error' /tmp/includefixes$$; then + echo "iwyu run failed" + cat /tmp/includefixes$$ + rm /tmp/includefixes$$ + exit 1 +else + if [ $DEBUG -gt 1 ]; then + cat /tmp/includefixes$$ + fi + # In compile_commands.json, the file name is recorded + # as a relative path to the build directory. + pushd "$(dirname "$COMPILE_COMMANDS")" || error "pushd failed" + "$FIX_INCLUDE" $FIX_INCLUDE_ARGS < /tmp/includefixes$$ || echo "Some files modified" + popd + rm /tmp/includefixes$$ +fi + +grep -v -f tools_webrtc/iwyu/iwyu-filter-list $FILE_CC > $FILE_CC.new +mv $FILE_CC.new $FILE_CC + +if [ -n "$FILE_H" ]; then + grep -v -f tools_webrtc/iwyu/iwyu-filter-list $FILE_H > $FILE_H.new + mv $FILE_H.new $FILE_H +fi + +echo "Finished. Check diff, compile and git cl format before uploading." diff --git a/third_party/libwebrtc/tools_webrtc/iwyu/iwyu-filter-list b/third_party/libwebrtc/tools_webrtc/iwyu/iwyu-filter-list new file mode 100644 index 0000000000..b5b0fb0721 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/iwyu/iwyu-filter-list @@ -0,0 +1,9 @@ +# These are lines that apply-iwyu will prevent from being added to a +# file. They are lines that refer to files that are conditionally included +# in certain configurations. +#include <sys/socket.h> +#include <ext/alloc_traits.h> +#include <netinet/in.h> +#include <__memory/unique_ptr.h> +#include <__tree> +#include <iosfwd> diff --git a/third_party/libwebrtc/tools_webrtc/iwyu/mappings.imp b/third_party/libwebrtc/tools_webrtc/iwyu/mappings.imp new file mode 100644 index 0000000000..9e1855adbb --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/iwyu/mappings.imp @@ -0,0 +1,35 @@ +# +# Mappings file for IWYU in webrtc +# +# Documentation of syntax: +# https://github.com/include-what-you-use/include-what-you-use/blob/master/docs/IWYUMappings.md +# +# Remember that it needs include strings INCLUDING <> or "" inside the quotes. +# +[ +# Redirect to have gmock and gtest includes under our control +{ include: ['"gmock/gmock.h"', "private", '"test/gmock.h"', "public"] }, +{ include: ['"gtest/gtest.h"', "private", '"test/gtest.h"', "public"] }, + +# rtc_base/containers internal defs +{ include: ['"rtc_base/containers/flat_tree.h"', "private", '"rtc_base/containers/flat_set.h"', "public"] }, + +# Revectoring of JSON +{ include: ['"json/reader.h"', "private", '"rtc_base/strings/json.h"', "public"] }, +{ include: ['"json/value.h"', "private", '"rtc_base/strings/json.h"', "public"] }, + +# LIBSRTP overrides +{ include: ['"rdbx.h"', "private", '"third_party/libsrtp/include/srtp_priv.h"', "public"] }, +{ include: ['"auth.h"', "private", '"third_party/libsrtp/include/srtp_priv.h"', "public"] }, + +# pthread internals +{ include: ['<bits/pthread_types.h>', "private", '<pthread.h>', "public"] }, + +# Needed to agree with presubmit tests for includes (and not include <iosfwd>) +{ symbol: ["std::string", "public", "<string>", "public"] }, +{ symbol: ["std::move", "public", "<utility>", "public"] }, +{ symbol: ["std::make_unique", "public", "<memory>", "public"] }, +{ symbol: ["std::unique_ptr", "public", "<memory>", "public"] }, +# Needed to avoid <iosfwd> +{ symbol: ["std::ostringstream", "public", "<sstream>", "public"] }, +] diff --git a/third_party/libwebrtc/tools_webrtc/libs/__init__.py b/third_party/libwebrtc/tools_webrtc/libs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/libs/__init__.py diff --git a/third_party/libwebrtc/tools_webrtc/libs/generate_licenses.py b/third_party/libwebrtc/tools_webrtc/libs/generate_licenses.py new file mode 100755 index 0000000000..de4167a668 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/libs/generate_licenses.py @@ -0,0 +1,256 @@ +#!/usr/bin/env vpython3 + +# Copyright 2016 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""Generates license markdown for a prebuilt version of WebRTC. + +Licenses are taken from dependent libraries which are determined by +GN desc command `gn desc` on all targets specified via `--target` argument. + +One can see all dependencies by invoking this command: +$ gn.py desc --all --format=json <out_directory> <target> | \ + vpython3 -m json.tool +(see "deps" subarray) + +Libraries are mapped to licenses via LIB_TO_LICENSES_DICT dictionary. + +""" + +import sys +import argparse +import json +import logging +import os +import re +import subprocess +from html import escape + +# Third_party library to licences mapping. Keys are names of the libraries +# (right after the `third_party/` prefix) +LIB_TO_LICENSES_DICT = { + 'abseil-cpp': ['third_party/abseil-cpp/LICENSE'], + 'android_ndk': ['third_party/android_ndk/NOTICE'], + 'android_sdk': ['third_party/android_sdk/LICENSE'], + 'auto': [ + 'third_party/android_deps/libs/' + 'com_google_auto_service_auto_service/LICENSE' + ], + 'boringssl': ['third_party/boringssl/src/LICENSE'], + 'crc32c': ['third_party/crc32c/src/LICENSE'], + 'dav1d': ['third_party/dav1d/LICENSE'], + 'errorprone': [ + 'third_party/android_deps/libs/' + 'com_google_errorprone_error_prone_core/LICENSE' + ], + 'fiat': ['third_party/boringssl/src/third_party/fiat/LICENSE'], + 'guava': ['third_party/android_deps/libs/com_google_guava_guava/LICENSE'], + 'ijar': ['third_party/ijar/LICENSE'], + 'jsoncpp': ['third_party/jsoncpp/LICENSE'], + 'libaom': ['third_party/libaom/source/libaom/LICENSE'], + 'libc++': ['buildtools/third_party/libc++/trunk/LICENSE.TXT'], + 'libc++abi': ['buildtools/third_party/libc++abi/trunk/LICENSE.TXT'], + 'libevent': ['third_party/libevent/LICENSE'], + 'libjpeg_turbo': ['third_party/libjpeg_turbo/LICENSE.md'], + 'libsrtp': ['third_party/libsrtp/LICENSE'], + 'libunwind': ['buildtools/third_party/libunwind/trunk/LICENSE.TXT'], + 'libvpx': ['third_party/libvpx/source/libvpx/LICENSE'], + 'libyuv': ['third_party/libyuv/LICENSE'], + 'nasm': ['third_party/nasm/LICENSE'], + 'opus': ['third_party/opus/src/COPYING'], + 'pffft': ['third_party/pffft/LICENSE'], + 'protobuf': ['third_party/protobuf/LICENSE'], + 'rnnoise': ['third_party/rnnoise/COPYING'], + 'webrtc': ['LICENSE'], + 'zlib': ['third_party/zlib/LICENSE'], + 'base64': ['rtc_base/third_party/base64/LICENSE'], + 'sigslot': ['rtc_base/third_party/sigslot/LICENSE'], + 'portaudio': ['modules/third_party/portaudio/LICENSE'], + 'fft': ['modules/third_party/fft/LICENSE'], + 'g711': ['modules/third_party/g711/LICENSE'], + 'g722': ['modules/third_party/g722/LICENSE'], + 'ooura': ['common_audio/third_party/ooura/LICENSE'], + 'spl_sqrt_floor': ['common_audio/third_party/spl_sqrt_floor/LICENSE'], + + # TODO(bugs.webrtc.org/1110): Remove this hack. This is not a lib. + # For some reason it is listed as so in _GetThirdPartyLibraries. + 'android_deps': [], + # This is not a library but a collection of libraries. + 'androidx': [], + + # Compile time dependencies, no license needed: + 'ow2_asm': [], + 'jdk': [], +} + +# Third_party library _regex_ to licences mapping. Keys are regular expression +# with names of the libraries (right after the `third_party/` prefix) +LIB_REGEX_TO_LICENSES_DICT = { + 'android_deps:android_support_annotations.*': [ + 'third_party/android_deps/libs/' + + 'com_android_support_support_annotations/LICENSE' + ], + + # Internal dependencies, licenses are already included by other dependencies + 'android_deps:com_android_support_support_annotations.*': [], +} + + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +WEBRTC_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir, os.pardir)) +SRC_DIR = os.path.dirname(os.path.dirname(SCRIPT_DIR)) +sys.path.append(os.path.join(SRC_DIR, 'build')) +import find_depot_tools + +THIRD_PARTY_LIB_SIMPLE_NAME_REGEX = r'^.*/third_party/([\w\-+]+).*$' +THIRD_PARTY_LIB_REGEX_TEMPLATE = r'^.*/third_party/%s$' + + +class LicenseBuilder: + def __init__(self, + buildfile_dirs, + targets, + lib_to_licenses_dict=None, + lib_regex_to_licenses_dict=None): + if lib_to_licenses_dict is None: + lib_to_licenses_dict = LIB_TO_LICENSES_DICT + + if lib_regex_to_licenses_dict is None: + lib_regex_to_licenses_dict = LIB_REGEX_TO_LICENSES_DICT + + self.buildfile_dirs = buildfile_dirs + self.targets = targets + self.lib_to_licenses_dict = lib_to_licenses_dict + self.lib_regex_to_licenses_dict = lib_regex_to_licenses_dict + + self.common_licenses_dict = self.lib_to_licenses_dict.copy() + self.common_licenses_dict.update(self.lib_regex_to_licenses_dict) + + @staticmethod + def _ParseLibraryName(dep): + """Returns library name after third_party + + Input one of: + //a/b/third_party/libname:c + //a/b/third_party/libname:c(//d/e/f:g) + //a/b/third_party/libname/c:d(//e/f/g:h) + + Outputs libname or None if this is not a third_party dependency. + """ + groups = re.match(THIRD_PARTY_LIB_SIMPLE_NAME_REGEX, dep) + return groups.group(1) if groups else None + + def _ParseLibrary(self, dep): + """Returns library simple or regex name that matches `dep` after third_party + + This method matches `dep` dependency against simple names in + LIB_TO_LICENSES_DICT and regular expression names in + LIB_REGEX_TO_LICENSES_DICT keys + + Outputs matched dict key or None if this is not a third_party dependency. + """ + libname = LicenseBuilder._ParseLibraryName(dep) + + for lib_regex in self.lib_regex_to_licenses_dict: + if re.match(THIRD_PARTY_LIB_REGEX_TEMPLATE % lib_regex, dep): + return lib_regex + + return libname + + @staticmethod + def _RunGN(buildfile_dir, target): + cmd = [ + sys.executable, + os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py'), + 'desc', + '--all', + '--format=json', + os.path.abspath(buildfile_dir), + target, + ] + logging.debug('Running: %r', cmd) + output_json = subprocess.check_output(cmd, cwd=WEBRTC_ROOT).decode('UTF-8') + logging.debug('Output: %s', output_json) + return output_json + + def _GetThirdPartyLibraries(self, buildfile_dir, target): + output = json.loads(LicenseBuilder._RunGN(buildfile_dir, target)) + libraries = set() + for described_target in list(output.values()): + third_party_libs = (self._ParseLibrary(dep) + for dep in described_target['deps']) + libraries |= set(lib for lib in third_party_libs if lib) + return libraries + + def GenerateLicenseText(self, output_dir): + # Get a list of third_party libs from gn. For fat libraries we must consider + # all architectures, hence the multiple buildfile directories. + third_party_libs = set() + for buildfile in self.buildfile_dirs: + for target in self.targets: + third_party_libs |= self._GetThirdPartyLibraries(buildfile, target) + assert len(third_party_libs) > 0 + + missing_licenses = third_party_libs - set(self.common_licenses_dict.keys()) + if missing_licenses: + error_msg = 'Missing licenses for following third_party targets: %s' % \ + ', '.join(sorted(missing_licenses)) + logging.error(error_msg) + raise Exception(error_msg) + + # Put webrtc at the front of the list. + license_libs = sorted(third_party_libs) + license_libs.insert(0, 'webrtc') + + logging.info('List of licenses: %s', ', '.join(license_libs)) + + # Generate markdown. + output_license_file = open(os.path.join(output_dir, 'LICENSE.md'), 'w+') + for license_lib in license_libs: + if len(self.common_licenses_dict[license_lib]) == 0: + logging.info('Skipping compile time or internal dependency: %s', + license_lib) + continue # Compile time dependency + + output_license_file.write('# %s\n' % license_lib) + output_license_file.write('```\n') + for path in self.common_licenses_dict[license_lib]: + license_path = os.path.join(WEBRTC_ROOT, path) + with open(license_path, 'r') as license_file: + license_text = escape(license_file.read(), quote=True) + output_license_file.write(license_text) + output_license_file.write('\n') + output_license_file.write('```\n\n') + + output_license_file.close() + + +def main(): + parser = argparse.ArgumentParser(description='Generate WebRTC LICENSE.md') + parser.add_argument('--verbose', + action='store_true', + default=False, + help='Debug logging.') + parser.add_argument('--target', + required=True, + action='append', + default=[], + help='Name of the GN target to generate a license for') + parser.add_argument('output_dir', help='Directory to output LICENSE.md to.') + parser.add_argument('buildfile_dirs', + nargs='+', + help='Directories containing gn generated ninja files') + args = parser.parse_args() + + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) + + builder = LicenseBuilder(args.buildfile_dirs, args.target) + builder.GenerateLicenseText(args.output_dir) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/third_party/libwebrtc/tools_webrtc/libs/generate_licenses_test.py b/third_party/libwebrtc/tools_webrtc/libs/generate_licenses_test.py new file mode 100755 index 0000000000..6dfd8f3e22 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/libs/generate_licenses_test.py @@ -0,0 +1,120 @@ +#!/usr/bin/env vpython3 + +# pylint: disable=protected-access,unused-argument + +# Copyright 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import unittest +from mock import patch + +from generate_licenses import LicenseBuilder + + +class TestLicenseBuilder(unittest.TestCase): + @staticmethod + def _FakeRunGN(buildfile_dir, target): + return """ + { + "target1": { + "deps": [ + "//a/b/third_party/libname1:c", + "//a/b/third_party/libname2:c(//d/e/f:g)", + "//a/b/third_party/libname3/c:d(//e/f/g:h)", + "//a/b/not_third_party/c" + ] + } + } + """ + + def testParseLibraryName(self): + self.assertEqual( + LicenseBuilder._ParseLibraryName('//a/b/third_party/libname1:c'), + 'libname1') + self.assertEqual( + LicenseBuilder._ParseLibraryName('//a/b/third_party/libname2:c(d)'), + 'libname2') + self.assertEqual( + LicenseBuilder._ParseLibraryName('//a/b/third_party/libname3/c:d(e)'), + 'libname3') + self.assertEqual( + LicenseBuilder._ParseLibraryName('//a/b/not_third_party/c'), None) + + def testParseLibrarySimpleMatch(self): + builder = LicenseBuilder([], [], {}, {}) + self.assertEqual(builder._ParseLibrary('//a/b/third_party/libname:c'), + 'libname') + + def testParseLibraryRegExNoMatchFallbacksToDefaultLibname(self): + lib_dict = { + 'libname:foo.*': ['path/to/LICENSE'], + } + builder = LicenseBuilder([], [], lib_dict, {}) + self.assertEqual( + builder._ParseLibrary('//a/b/third_party/libname:bar_java'), 'libname') + + def testParseLibraryRegExMatch(self): + lib_regex_dict = { + 'libname:foo.*': ['path/to/LICENSE'], + } + builder = LicenseBuilder([], [], {}, lib_regex_dict) + self.assertEqual( + builder._ParseLibrary('//a/b/third_party/libname:foo_bar_java'), + 'libname:foo.*') + + def testParseLibraryRegExMatchWithSubDirectory(self): + lib_regex_dict = { + 'libname/foo:bar.*': ['path/to/LICENSE'], + } + builder = LicenseBuilder([], [], {}, lib_regex_dict) + self.assertEqual( + builder._ParseLibrary('//a/b/third_party/libname/foo:bar_java'), + 'libname/foo:bar.*') + + def testParseLibraryRegExMatchWithStarInside(self): + lib_regex_dict = { + 'libname/foo.*bar.*': ['path/to/LICENSE'], + } + builder = LicenseBuilder([], [], {}, lib_regex_dict) + self.assertEqual( + builder._ParseLibrary('//a/b/third_party/libname/fooHAHA:bar_java'), + 'libname/foo.*bar.*') + + @patch('generate_licenses.LicenseBuilder._RunGN', _FakeRunGN) + def testGetThirdPartyLibrariesWithoutRegex(self): + builder = LicenseBuilder([], [], {}, {}) + self.assertEqual(builder._GetThirdPartyLibraries('out/arm', 'target1'), + set(['libname1', 'libname2', 'libname3'])) + + @patch('generate_licenses.LicenseBuilder._RunGN', _FakeRunGN) + def testGetThirdPartyLibrariesWithRegex(self): + lib_regex_dict = { + 'libname2:c.*': ['path/to/LICENSE'], + } + builder = LicenseBuilder([], [], {}, lib_regex_dict) + self.assertEqual(builder._GetThirdPartyLibraries('out/arm', 'target1'), + set(['libname1', 'libname2:c.*', 'libname3'])) + + @patch('generate_licenses.LicenseBuilder._RunGN', _FakeRunGN) + def testGenerateLicenseTextFailIfUnknownLibrary(self): + lib_dict = { + 'simple_library': ['path/to/LICENSE'], + } + builder = LicenseBuilder(['dummy_dir'], ['dummy_target'], lib_dict, {}) + + with self.assertRaises(Exception) as context: + builder.GenerateLicenseText('dummy/dir') + + self.assertEqual( + context.exception.args[0], + 'Missing licenses for following third_party targets: ' + 'libname1, libname2, libname3') + + +if __name__ == '__main__': + unittest.main() diff --git a/third_party/libwebrtc/tools_webrtc/matlab/maxUnwrap.m b/third_party/libwebrtc/tools_webrtc/matlab/maxUnwrap.m new file mode 100644 index 0000000000..276c9523a4 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/matlab/maxUnwrap.m @@ -0,0 +1,25 @@ +function sequence = maxUnwrap(sequence, max) +% +% sequence = maxUnwrap(sequence, max) +% Unwraps when a wrap around is detected. +% +% Arguments +% +% sequence: The vector to unwrap. +% max: The maximum value that the sequence can take, +% and after which it will wrap over to 0. +% +% Return value +% +% sequence: The unwrapped vector. +% + +% Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. +% +% Use of this source code is governed by a BSD-style license +% that can be found in the LICENSE file in the root of the source +% tree. An additional intellectual property rights grant can be found +% in the file PATENTS. All contributing project authors may +% be found in the AUTHORS file in the root of the source tree. + +sequence = round((unwrap(2 * pi * sequence / max) * max) / (2 * pi)); diff --git a/third_party/libwebrtc/tools_webrtc/matlab/parseLog.m b/third_party/libwebrtc/tools_webrtc/matlab/parseLog.m new file mode 100644 index 0000000000..5d4c3f7bc1 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/matlab/parseLog.m @@ -0,0 +1,54 @@ +function parsed = parseLog(filename) +% +% parsed = parseLog(filename) +% Parses a DataLog text file, with the filename specified in the string +% filename, into a struct with each column name as a field, and with the +% column data stored as a vector in that field. +% +% Arguments +% +% filename: A string with the name of the file to parse. +% +% Return value +% +% parsed: A struct containing each column parsed from the input file +% as a field and with the column data stored as a vector in that +% field. +% + +% Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. +% +% Use of this source code is governed by a BSD-style license +% that can be found in the LICENSE file in the root of the source +% tree. An additional intellectual property rights grant can be found +% in the file PATENTS. All contributing project authors may +% be found in the AUTHORS file in the root of the source tree. + +table = importdata(filename, ',', 1); +if ~isstruct(table) + error('Malformed file, possibly empty or lacking data entries') +end + +colheaders = table.textdata; +if length(colheaders) == 1 + colheaders = regexp(table.textdata{1}, ',', 'split'); +end + +parsed = struct; +i = 1; +while i <= length(colheaders) + % Checking for a multi-value column. + m = regexp(colheaders{i}, '([\w\t]+)\[(\d+)\]', 'tokens'); + if ~isempty(m) + % Parse a multi-value column + n = str2double(m{1}{2}) - 1; + parsed.(strrep(m{1}{1}, ' ', '_')) = table.data(:, i:i+n); + i = i + n + 1; + elseif ~isempty(colheaders{i}) + % Parse a single-value column + parsed.(strrep(colheaders{i}, ' ', '_')) = table.data(:, i); + i = i + 1; + else + error('Empty column'); + end +end diff --git a/third_party/libwebrtc/tools_webrtc/matlab/rtpAnalyze.m b/third_party/libwebrtc/tools_webrtc/matlab/rtpAnalyze.m new file mode 100644 index 0000000000..c51af9cca5 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/matlab/rtpAnalyze.m @@ -0,0 +1,251 @@ +function rtpAnalyze( input_file ) +%RTP_ANALYZE Analyze RTP stream(s) from a txt file +% The function takes the output from the command line tool rtp_analyze +% and analyzes the stream(s) therein. First, process your rtpdump file +% through rtp_analyze (from command line): +% $ out/Debug/rtp_analyze my_file.rtp my_file.txt +% Then load it with this function (in Matlab): +% >> rtpAnalyze('my_file.txt') + +% Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. +% +% Use of this source code is governed by a BSD-style license +% that can be found in the LICENSE file in the root of the source +% tree. An additional intellectual property rights grant can be found +% in the file PATENTS. All contributing project authors may +% be found in the AUTHORS file in the root of the source tree. + +[SeqNo,TimeStamp,ArrTime,Size,PT,M,SSRC] = importfile(input_file); + +%% Filter out RTCP packets. +% These appear as RTP packets having payload types 72 through 76. +ix = not(ismember(PT, 72:76)); +fprintf('Removing %i RTCP packets\n', length(SeqNo) - sum(ix)); +SeqNo = SeqNo(ix); +TimeStamp = TimeStamp(ix); +ArrTime = ArrTime(ix); +Size = Size(ix); +PT = PT(ix); +M = M(ix); +SSRC = SSRC(ix); + +%% Find streams. +[uSSRC, ~, uix] = unique(SSRC); + +% If there are multiple streams, select one and purge the other +% streams from the data vectors. If there is only one stream, the +% vectors are good to use as they are. +if length(uSSRC) > 1 + for i=1:length(uSSRC) + uPT = unique(PT(uix == i)); + fprintf('%i: %s (%d packets, pt: %i', i, uSSRC{i}, ... + length(find(uix==i)), uPT(1)); + if length(uPT) > 1 + fprintf(', %i', uPT(2:end)); + end + fprintf(')\n'); + end + sel = input('Select stream number: '); + if sel < 1 || sel > length(uSSRC) + error('Out of range'); + end + ix = find(uix == sel); + % This is where the data vectors are trimmed. + SeqNo = SeqNo(ix); + TimeStamp = TimeStamp(ix); + ArrTime = ArrTime(ix); + Size = Size(ix); + PT = PT(ix); + M = M(ix); + SSRC = SSRC(ix); +end + +%% Unwrap SeqNo and TimeStamp. +SeqNoUW = maxUnwrap(SeqNo, 65535); +TimeStampUW = maxUnwrap(TimeStamp, 4294967295); + +%% Generate some stats for the stream. +fprintf('Statistics:\n'); +fprintf('SSRC: %s\n', SSRC{1}); +uPT = unique(PT); +if length(uPT) > 1 + warning('This tool cannot yet handle changes in codec sample rate'); +end +fprintf('Payload type(s): %i', uPT(1)); +if length(uPT) > 1 + fprintf(', %i', uPT(2:end)); +end +fprintf('\n'); +fprintf('Packets: %i\n', length(SeqNo)); +SortSeqNo = sort(SeqNoUW); +fprintf('Missing sequence numbers: %i\n', ... + length(find(diff(SortSeqNo) > 1))); +fprintf('Duplicated packets: %i\n', length(find(diff(SortSeqNo) == 0))); +reorderIx = findReorderedPackets(SeqNoUW); +fprintf('Reordered packets: %i\n', length(reorderIx)); +tsdiff = diff(TimeStampUW); +tsdiff = tsdiff(diff(SeqNoUW) == 1); +[utsdiff, ~, ixtsdiff] = unique(tsdiff); +fprintf('Common packet sizes:\n'); +for i = 1:length(utsdiff) + fprintf(' %i samples (%i%%)\n', ... + utsdiff(i), ... + round(100 * length(find(ixtsdiff == i))/length(ixtsdiff))); +end + +%% Trying to figure out sample rate. +fs_est = (TimeStampUW(end) - TimeStampUW(1)) / (ArrTime(end) - ArrTime(1)); +fs_vec = [8, 16, 32, 48]; +fs = 0; +for f = fs_vec + if abs((fs_est-f)/f) < 0.05 % 5% margin + fs = f; + break; + end +end +if fs == 0 + fprintf('Cannot determine sample rate. I get it to %.2f kHz\n', ... + fs_est); + fs = input('Please, input a sample rate (in kHz): '); +else + fprintf('Sample rate estimated to %i kHz\n', fs); +end + +SendTimeMs = (TimeStampUW - TimeStampUW(1)) / fs; + +fprintf('Stream duration at sender: %.1f seconds\n', ... + (SendTimeMs(end) - SendTimeMs(1)) / 1000); + +fprintf('Stream duration at receiver: %.1f seconds\n', ... + (ArrTime(end) - ArrTime(1)) / 1000); + +fprintf('Clock drift: %.2f%%\n', ... + 100 * ((ArrTime(end) - ArrTime(1)) / ... + (SendTimeMs(end) - SendTimeMs(1)) - 1)); + +fprintf('Sent average bitrate: %i kbps\n', ... + round(sum(Size) * 8 / (SendTimeMs(end)-SendTimeMs(1)))); + +fprintf('Received average bitrate: %i kbps\n', ... + round(sum(Size) * 8 / (ArrTime(end)-ArrTime(1)))); + +%% Plots. +delay = ArrTime - SendTimeMs; +delay = delay - min(delay); +delayOrdered = delay; +delayOrdered(reorderIx) = nan; % Set reordered packets to NaN. +delayReordered = delay(reorderIx); % Pick the reordered packets. +sendTimeMsReordered = SendTimeMs(reorderIx); + +% Sort time arrays in packet send order. +[~, sortix] = sort(SeqNoUW); +SendTimeMs = SendTimeMs(sortix); +Size = Size(sortix); +delayOrdered = delayOrdered(sortix); + +figure +plot(SendTimeMs / 1000, delayOrdered, ... + sendTimeMsReordered / 1000, delayReordered, 'r.'); +xlabel('Send time [s]'); +ylabel('Relative transport delay [ms]'); +title(sprintf('SSRC: %s', SSRC{1})); + +SendBitrateKbps = 8 * Size(1:end-1) ./ diff(SendTimeMs); +figure +plot(SendTimeMs(1:end-1)/1000, SendBitrateKbps); +xlabel('Send time [s]'); +ylabel('Send bitrate [kbps]'); +end + +%% Subfunctions. + +% findReorderedPackets returns the index to all packets that are considered +% old compared with the largest seen sequence number. The input seqNo must +% be unwrapped for this to work. +function reorderIx = findReorderedPackets(seqNo) +largestSeqNo = seqNo(1); +reorderIx = []; +for i = 2:length(seqNo) + if seqNo(i) < largestSeqNo + reorderIx = [reorderIx; i]; %#ok<AGROW> + else + largestSeqNo = seqNo(i); + end +end +end + +%% Auto-generated subfunction. +function [SeqNo,TimeStamp,SendTime,Size,PT,M,SSRC] = ... + importfile(filename, startRow, endRow) +%IMPORTFILE Import numeric data from a text file as column vectors. +% [SEQNO,TIMESTAMP,SENDTIME,SIZE,PT,M,SSRC] = IMPORTFILE(FILENAME) Reads +% data from text file FILENAME for the default selection. +% +% [SEQNO,TIMESTAMP,SENDTIME,SIZE,PT,M,SSRC] = IMPORTFILE(FILENAME, +% STARTROW, ENDROW) Reads data from rows STARTROW through ENDROW of text +% file FILENAME. +% +% Example: +% [SeqNo,TimeStamp,SendTime,Size,PT,M,SSRC] = +% importfile('rtpdump_recv.txt',2, 123); +% +% See also TEXTSCAN. + +% Auto-generated by MATLAB on 2015/05/28 09:55:50 + +%% Initialize variables. +if nargin<=2 + startRow = 2; + endRow = inf; +end + +%% Format string for each line of text: +% column1: double (%f) +% column2: double (%f) +% column3: double (%f) +% column4: double (%f) +% column5: double (%f) +% column6: double (%f) +% column7: text (%s) +% For more information, see the TEXTSCAN documentation. +formatSpec = '%5f%11f%11f%6f%6f%3f%s%[^\n\r]'; + +%% Open the text file. +fileID = fopen(filename,'r'); + +%% Read columns of data according to format string. +% This call is based on the structure of the file used to generate this +% code. If an error occurs for a different file, try regenerating the code +% from the Import Tool. +dataArray = textscan(fileID, formatSpec, endRow(1)-startRow(1)+1, ... + 'Delimiter', '', 'WhiteSpace', '', 'HeaderLines', startRow(1)-1, ... + 'ReturnOnError', false); +for block=2:length(startRow) + frewind(fileID); + dataArrayBlock = textscan(fileID, formatSpec, ... + endRow(block)-startRow(block)+1, 'Delimiter', '', 'WhiteSpace', ... + '', 'HeaderLines', startRow(block)-1, 'ReturnOnError', false); + for col=1:length(dataArray) + dataArray{col} = [dataArray{col};dataArrayBlock{col}]; + end +end + +%% Close the text file. +fclose(fileID); + +%% Post processing for unimportable data. +% No unimportable data rules were applied during the import, so no post +% processing code is included. To generate code which works for +% unimportable data, select unimportable cells in a file and regenerate the +% script. + +%% Allocate imported array to column variable names +SeqNo = dataArray{:, 1}; +TimeStamp = dataArray{:, 2}; +SendTime = dataArray{:, 3}; +Size = dataArray{:, 4}; +PT = dataArray{:, 5}; +M = dataArray{:, 6}; +SSRC = dataArray{:, 7}; +end + diff --git a/third_party/libwebrtc/tools_webrtc/mb/OWNERS b/third_party/libwebrtc/tools_webrtc/mb/OWNERS new file mode 100644 index 0000000000..48e6927746 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/mb/OWNERS @@ -0,0 +1 @@ +mbonadei@webrtc.org diff --git a/third_party/libwebrtc/tools_webrtc/mb/PRESUBMIT.py b/third_party/libwebrtc/tools_webrtc/mb/PRESUBMIT.py new file mode 100644 index 0000000000..dd957d0482 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/mb/PRESUBMIT.py @@ -0,0 +1,51 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + + +# Runs PRESUBMIT.py in py3 mode by git cl presubmit. +USE_PYTHON3 = True + + +def _CommonChecks(input_api, output_api): + results = [] + + # Run the MB unittests. + results.extend( + input_api.canned_checks.RunUnitTestsInDirectory(input_api, + output_api, + '.', + [r'^.+_unittest\.py$'], + skip_shebang_check=False, + run_on_python2=False)) + + # Validate the format of the mb_config.pyl file. + cmd = [input_api.python3_executable, 'mb.py', 'validate'] + kwargs = {'cwd': input_api.PresubmitLocalPath()} + results.extend(input_api.RunTests([ + input_api.Command(name='mb_validate', + cmd=cmd, kwargs=kwargs, + message=output_api.PresubmitError)])) + + results.extend( + input_api.canned_checks.CheckLongLines( + input_api, + output_api, + maxlen=80, + source_file_filter=lambda x: 'mb_config.pyl' in x.LocalPath())) + + return results + + +def CheckChangeOnUpload(input_api, output_api): + return _CommonChecks(input_api, output_api) + + +def CheckChangeOnCommit(input_api, output_api): + return _CommonChecks(input_api, output_api) diff --git a/third_party/libwebrtc/tools_webrtc/mb/README.md b/third_party/libwebrtc/tools_webrtc/mb/README.md new file mode 100644 index 0000000000..4e73a8e9fc --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/mb/README.md @@ -0,0 +1,22 @@ +# MB - The Meta-Build wrapper + +MB is a simple wrapper intended to provide a uniform interface to either +GYP or GN, such that users and bots can call one script and not need to +worry about whether a given bot is meant to use GN or GYP. + +It supports two main functions: + +1. "gen" - the main `gyp_chromium` / `gn gen` invocation that generates the + Ninja files needed for the build. + +2. "analyze" - the step that takes a list of modified files and a list of + desired targets and reports which targets will need to be rebuilt. + +We also use MB as a forcing function to collect all of the different +build configurations that we actually support for Chromium builds into +one place, in `//tools/mb/mb_config.pyl`. + +For more information, see: + +* [The User Guide](docs/user_guide.md) +* [The Design Spec](docs/design_spec.md) diff --git a/third_party/libwebrtc/tools_webrtc/mb/docs/README.md b/third_party/libwebrtc/tools_webrtc/mb/docs/README.md new file mode 100644 index 0000000000..f29007d9ed --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/mb/docs/README.md @@ -0,0 +1,4 @@ +# The MB (Meta-Build wrapper) documentation + +* The [User Guide](user_guide.md) +* The [Design Spec](design_spec.md) diff --git a/third_party/libwebrtc/tools_webrtc/mb/docs/design_spec.md b/third_party/libwebrtc/tools_webrtc/mb/docs/design_spec.md new file mode 100644 index 0000000000..0aaaf89f9b --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/mb/docs/design_spec.md @@ -0,0 +1,426 @@ +# The MB (Meta-Build wrapper) design spec + +[TOC] + +## Intro + +MB is intended to address two major aspects of the GYP -> GN transition +for Chromium: + +1. "bot toggling" - make it so that we can easily flip a given bot + back and forth between GN and GYP. + +2. "bot configuration" - provide a single source of truth for all of + the different configurations (os/arch/`gyp_define` combinations) of + Chromium that are supported. + +MB must handle at least the `gen` and `analyze` steps on the bots, i.e., +we need to wrap both the `gyp_chromium` invocation to generate the +Ninja files, and the `analyze` step that takes a list of modified files +and a list of targets to build and returns which targets are affected by +the files. + +For more information on how to actually use MB, see +[the user guide](user_guide.md). + +## Design + +MB is intended to be as simple as possible, and to defer as much work as +possible to GN or GYP. It should live as a very simple Python wrapper +that offers little in the way of surprises. + +### Command line + +It is structured as a single binary that supports a list of subcommands: + +* `mb gen -c linux_rel_bot //out/Release` +* `mb analyze -m tryserver.chromium.linux -b linux_rel /tmp/input.json /tmp/output.json` + +### Configurations + +`mb` will first look for a bot config file in a set of different locations +(initially just in //ios/build/bots). Bot config files are JSON files that +contain keys for 'GYP_DEFINES' (a list of strings that will be joined together +with spaces and passed to GYP, or a dict that will be similarly converted), +'gn_args' (a list of strings that will be joined together), and an +'mb_type' field that says whether to use GN or GYP. Bot config files +require the full list of settings to be given explicitly. + +If no matching bot config file is found, `mb` looks in the +`//tools/mb/mb_config.pyl` config file to determine whether to use GYP or GN +for a particular build directory, and what set of flags (`GYP_DEFINES` or `gn +args`) to use. + +A config can either be specified directly (useful for testing) or by specifying +the builder group name and builder name (useful on the bots so that they do not +need to specify a config directly and can be hidden from the details). + +See the [user guide](user_guide.md#mb_config.pyl) for details. + +### Handling the analyze step + +The interface to `mb analyze` is described in the +[user\_guide](user_guide.md#mb_analyze). + +The way analyze works can be subtle and complicated (see below). + +Since the interface basically mirrors the way the "analyze" step on the bots +invokes `gyp_chromium` today, when the config is found to be a gyp config, +the arguments are passed straight through. + +It implements the equivalent functionality in GN by calling `gn refs +[list of files] --type=executable --all --as=output` and filtering the +output to match the list of targets. + +## Analyze + +The goal of the `analyze` step is to speed up the cycle time of the try servers +by only building and running the tests affected by the files in a patch, rather +than everything that might be out of date. Doing this ends up being tricky. + +We start with the following requirements and observations: + +* In an ideal (un-resource-constrained) world, we would build and test + everything that a patch affected on every patch. This does not + necessarily mean that we would build 'all' on every patch (see below). + +* In the real world, however, we do not have an infinite number of machines, + and try jobs are not infinitely fast, so we need to balance the desire + to get maximum test coverage against the desire to have reasonable cycle + times, given the number of machines we have. + +* Also, since we run most try jobs against tip-of-tree Chromium, by + the time one job completes on the bot, new patches have probably landed, + rendering the build out of date. + +* This means that the next try job may have to do a build that is out of + date due to a combination of files affected by a given patch, and files + affected for unrelated reasons. We want to rebuild and test only the + targets affected by the patch, so that we don't blame or punish the + patch author for unrelated changes. + +So: + +1. We need a way to indicate which changed files we care about and which + we don't (the affected files of a patch). + +2. We need to know which tests we might potentially want to run, and how + those are mapped onto build targets. For some kinds of tests (like + GTest-based tests), the mapping is 1:1 - if you want to run base_unittests, + you need to build base_unittests. For others (like the telemetry and + layout tests), you might need to build several executables in order to + run the tests, and that mapping might best be captured by a *meta* + target (a GN group or a GYP 'none' target like `webkit_tests`) that + depends on the right list of files. Because the GN and GYP files know + nothing about test steps, we have to have some way of mapping back + and forth between test steps and build targets. That mapping + is *not* currently available to MB (or GN or GYP), and so we have to + enough information to make it possible for the caller to do the mapping. + +3. We might also want to know when test targets are affected by data files + that aren't compiled (python scripts, or the layout tests themselves). + There's no good way to do this in GYP, but GN supports this. + +4. We also want to ensure that particular targets still compile even if they + are not actually tested; consider testing the installers themselves, or + targets that don't yet have good test coverage. We might want to use meta + targets for this purpose as well. + +5. However, for some meta targets, we don't necessarily want to rebuild the + meta target itself, perhaps just the dependencies of the meta target that + are affected by the patch. For example, if you have a meta target like + `blink_tests` that might depend on ten different test binaries. If a patch + only affects one of them (say `wtf_unittests`), you don't want to + build `blink_tests`, because that might actually also build the other nine + targets. In other words, some meta targets are *prunable*. + +6. As noted above, in the ideal case we actually have enough resources and + things are fast enough that we can afford to build everything affected by a + patch, but listing every possible target explicitly would be painful. The + GYP and GN Ninja generators provide an 'all' target that captures (nearly, + see [crbug.com/503241](crbug.com/503241)) everything, but unfortunately + neither GN nor GYP actually represents 'all' as a meta target in the build + graph, so we will need to write code to handle that specially. + +7. In some cases, we will not be able to correctly analyze the build graph to + determine the impact of a patch, and need to bail out (e.g,. if you change a + build file itself, it may not be easy to tell how that affects the graph). + In that case we should simply build and run everything. + +The interaction between 2) and 5) means that we need to treat meta targets +two different ways, and so we need to know which targets should be +pruned in the sense of 5) and which targets should be returned unchanged +so that we can map them back to the appropriate tests. + +So, we need three things as input: + +* `files`: the list of files in the patch +* `test_targets`: the list of ninja targets which, if affected by a patch, + should be reported back so that we can map them back to the appropriate + tests to run. Any meta targets in this list should *not* be pruned. +* `additional_compile_targets`: the list of ninja targets we wish to compile + *in addition to* the list in `test_targets`. Any meta targets + present in this list should be pruned (we don't need to return the + meta targets because they aren't mapped back to tests, and we don't want + to build them because we might build too much). + +We can then return two lists as output: + +* `compile_targets`, which is a list of pruned targets to be + passed to Ninja to build. It is acceptable to replace a list of + pruned targets by a meta target if it turns out that all of the + dependendencies of the target are affected by the patch (i.e., + all ten binaries that blink_tests depends on), but doing so is + not required. +* `test_targets`, which is a list of unpruned targets to be mapped + back to determine which tests to run. + +There may be substantial overlap between the two lists, but there is +no guarantee that one is a subset of the other and the two cannot be +used interchangeably or merged together without losing information and +causing the wrong thing to happen. + +The implementation is responsible for recognizing 'all' as a magic string +and mapping it onto the list of all root nodes in the build graph. + +There may be files listed in the input that don't actually exist in the build +graph: this could be either the result of an error (the file should be in the +build graph, but isn't), or perfectly fine (the file doesn't affect the build +graph at all). We can't tell these two apart, so we should ignore missing +files. + +There may be targets listed in the input that don't exist in the build +graph; unlike missing files, this can only indicate a configuration error, +and so we should return which targets are missing so the caller can +treat this as an error, if so desired. + +Any of the three inputs may be an empty list: + +* It normally doesn't make sense to call analyze at all if no files + were modified, but in rare cases we can hit a race where we try to + test a patch after it has already been committed, in which case + the list of modified files is empty. We should return 'no dependency' + in that case. + +* Passing an empty list for one or the other of test_targets and + additional_compile_targets is perfectly sensible: in the former case, + it can indicate that you don't want to run any tests, and in the latter, + it can indicate that you don't want to do build anything else in + addition to the test targets. + +* It doesn't make sense to call analyze if you don't want to compile + anything at all, so passing [] for both test_targets and + additional_compile_targets should probably return an error. + +In the output case, an empty list indicates that there was nothing to +build, or that there were no affected test targets as appropriate. + +Note that passing no arguments to Ninja is equivalent to passing +`all` to Ninja (at least given how GN and GYP work); however, we +don't want to take advantage of this in most cases because we don't +actually want to build every out of date target, only the targets +potentially affected by the files. One could try to indicate +to analyze that we wanted to use no arguments instead of an empty +list, but using the existing fields for this seems fragile and/or +confusing, and adding a new field for this seems unwarranted at this time. + +There is an "error" field in case something goes wrong (like the +empty file list case, above, or an internal error in MB/GYP/GN). The +analyze code should also return an error code to the shell if appropriate +to indicate that the command failed. + +In the case where build files themselves are modified and analyze may +not be able to determine a correct answer (point 7 above, where we return +"Found dependency (all)"), we should also return the `test_targets` unmodified +and return the union of `test_targets` and `additional_compile_targets` for +`compile_targets`, to avoid confusion. + +### Examples + +Continuing the example given above, suppose we have the following build +graph: + +* `blink_tests` is a meta target that depends on `webkit_unit_tests`, + `wtf_unittests`, and `webkit_tests` and represents all of the targets + needed to fully test Blink. Each of those is a separate test step. +* `webkit_tests` is also a meta target; it depends on `content_shell` + and `image_diff`. +* `base_unittests` is a separate test binary. +* `wtf_unittests` depends on `Assertions.cpp` and `AssertionsTest.cpp`. +* `webkit_unit_tests` depends on `WebNode.cpp` and `WebNodeTest.cpp`. +* `content_shell` depends on `WebNode.cpp` and `Assertions.cpp`. +* `base_unittests` depends on `logging.cc` and `logging_unittest.cc`. + +#### Example 1 + +We wish to run 'wtf_unittests' and 'webkit_tests' on a bot, but not +compile any additional targets. + +If a patch touches WebNode.cpp, then analyze gets as input: + + { + "files": ["WebNode.cpp"], + "test_targets": ["wtf_unittests", "webkit_tests"], + "additional_compile_targets": [] + } + +and should return as output: + + { + "status": "Found dependency", + "compile_targets": ["webkit_unit_tests"], + "test_targets": ["webkit_tests"] + } + +Note how `webkit_tests` was pruned in compile_targets but not in test_targets. + +#### Example 2 + +Using the same patch as Example 1, assume we wish to run only `wtf_unittests`, +but additionally build everything needed to test Blink (`blink_tests`): + +We pass as input: + + { + "files": ["WebNode.cpp"], + "test_targets": ["wtf_unittests"], + "additional_compile_targets": ["blink_tests"] + } + +And should get as output: + + { + "status": "Found dependency", + "compile_targets": ["webkit_unit_tests"], + "test_targets": [] + } + +Here `blink_tests` was pruned in the output compile_targets, and +test_targets was empty, since blink_tests was not listed in the input +test_targets. + +#### Example 3 + +Build everything, but do not run any tests. + +Input: + + { + "files": ["WebNode.cpp"], + "test_targets": [], + "additional_compile_targets": ["all"] + } + +Output: + + { + "status": "Found dependency", + "compile_targets": ["webkit_unit_tests", "content_shell"], + "test_targets": [] + } + +#### Example 4 + +Same as Example 2, but a build file was modified instead of a source file. + +Input: + + { + "files": ["BUILD.gn"], + "test_targets": ["wtf_unittests"], + "additional_compile_targets": ["blink_tests"] + } + +Output: + + { + "status": "Found dependency (all)", + "compile_targets": ["webkit_unit_tests", "wtf_unittests"], + "test_targets": ["wtf_unittests"] + } + +test_targets was returned unchanged, compile_targets was pruned. + +## Random Requirements and Rationale + +This section is collection of semi-organized notes on why MB is the way +it is ... + +### in-tree or out-of-tree + +The first issue is whether or not this should exist as a script in +Chromium at all; an alternative would be to simply change the bot +configurations to know whether to use GYP or GN, and which flags to +pass. + +That would certainly work, but experience over the past two years +suggests a few things: + + * we should push as much logic as we can into the source repositories + so that they can be versioned and changed atomically with changes to + the product code; having to coordinate changes between src/ and + build/ is at best annoying and can lead to weird errors. + * the infra team would really like to move to providing + product-independent services (i.e., not have to do one thing for + Chromium, another for NaCl, a third for V8, etc.). + * we found that during the SVN->GIT migration the ability to flip bot + configurations between the two via changes to a file in chromium + was very useful. + +All of this suggests that the interface between bots and Chromium should +be a simple one, hiding as much of the chromium logic as possible. + +### Why not have MB be smarter about de-duping flags? + +This just adds complexity to the MB implementation, and duplicates logic +that GYP and GN already have to support anyway; in particular, it might +require MB to know how to parse GYP and GN values. The belief is that +if MB does *not* do this, it will lead to fewer surprises. + +It will not be hard to change this if need be. + +### Integration w/ gclient runhooks + +On the bots, we will disable `gyp_chromium` as part of runhooks (using +`GYP_CHROMIUM_NO_ACTION=1`), so that mb shows up as a separate step. + +At the moment, we expect most developers to either continue to use +`gyp_chromium` in runhooks or to disable at as above if they have no +use for GYP at all. We may revisit how this works once we encourage more +people to use GN full-time (i.e., we might take `gyp_chromium` out of +runhooks altogether). + +### Config per flag set or config per (os/arch/flag set)? + +Currently, mb_config.pyl does not specify the host_os, target_os, host_cpu, or +target_cpu values for every config that Chromium runs on, it only specifies +them for when the values need to be explicitly set on the command line. + +Instead, we have one config per unique combination of flags only. + +In other words, rather than having `linux_rel_bot`, `win_rel_bot`, and +`mac_rel_bot`, we just have `rel_bot`. + +This design allows us to determine easily all of the different sets +of flags that we need to support, but *not* which flags are used on which +host/target combinations. + +It may be that we should really track the latter. Doing so is just a +config file change, however. + +### Non-goals + +* MB is not intended to replace direct invocation of GN or GYP for + complicated build scenarios (aka ChromeOS), where multiple flags need + to be set to user-defined paths for specific toolchains (e.g., where + ChromeOS needs to specify specific board types and compilers). + +* MB is not intended at this time to be something developers use frequently, + or to add a lot of features to. We hope to be able to get rid of it once + the GYP->GN migration is done, and so we should not add things for + developers that can't easily be added to GN itself. + +* MB is not intended to replace the + [CR tool](https://code.google.com/p/chromium/wiki/CRUserManual). Not + only is it only intended to replace the gyp\_chromium part of `'gclient + runhooks'`, it is not really meant as a developer-facing tool. diff --git a/third_party/libwebrtc/tools_webrtc/mb/docs/user_guide.md b/third_party/libwebrtc/tools_webrtc/mb/docs/user_guide.md new file mode 100644 index 0000000000..8c66cd328c --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/mb/docs/user_guide.md @@ -0,0 +1,298 @@ +# The MB (Meta-Build wrapper) user guide + +[TOC] + +## Introduction + +`mb` is a simple python wrapper around the GYP and GN meta-build tools to +be used as part of the GYP->GN migration. + +It is intended to be used by bots to make it easier to manage the configuration +each bot builds (i.e., the configurations can be changed from chromium +commits), and to consolidate the list of all of the various configurations +that Chromium is built in. + +Ideally this tool will no longer be needed after the migration is complete. + +For more discussion of MB, see also [the design spec](design_spec.md). + +## MB subcommands + +### `mb analyze` + +`mb analyze` is reponsible for determining what targets are affected by +a list of files (e.g., the list of files in a patch on a trybot): + +``` +mb analyze -c chromium_linux_rel //out/Release input.json output.json +``` + +Either the `-c/--config` flag or the `-m/--builder-group` and `-b/--builder` +flags must be specified so that `mb` can figure out which config to use. + +The first positional argument must be a GN-style "source-absolute" path +to the build directory. + +The second positional argument is a (normal) path to a JSON file containing +a single object with the following fields: + + * `files`: an array of the modified filenames to check (as paths relative to + the checkout root). + * `test_targets`: an array of (ninja) build targets that needed to run the + tests we wish to run. An empty array will be treated as if there are + no tests that will be run. + * `additional_compile_targets`: an array of (ninja) build targets that + reflect the stuff we might want to build *in addition to* the list + passed in `test_targets`. Targets in this list will be treated + specially, in the following way: if a given target is a "meta" + (GN: group, GYP: none) target like 'blink_tests' or + 'chromium_builder_tests', or even the ninja-specific 'all' target, + then only the *dependencies* of the target that are affected by + the modified files will be rebuilt (not the target itself, which + might also cause unaffected dependencies to be rebuilt). An empty + list will be treated as if there are no additional targets to build. + Empty lists for both `test_targets` and `additional_compile_targets` + would cause no work to be done, so will result in an error. + * `targets`: a legacy field that resembled a union of `compile_targets` + and `test_targets`. Support for this field will be removed once the + bots have been updated to use compile_targets and test_targets instead. + +The third positional argument is a (normal) path to where mb will write +the result, also as a JSON object. This object may contain the following +fields: + + * `error`: this should only be present if something failed. + * `compile_targets`: the list of ninja targets that should be passed + directly to the corresponding ninja / compile.py invocation. This + list may contain entries that are *not* listed in the input (see + the description of `additional_compile_targets` above and + [design_spec.md](the design spec) for how this works). + * `invalid_targets`: a list of any targets that were passed in + either of the input lists that weren't actually found in the graph. + * `test_targets`: the subset of the input `test_targets` that are + potentially out of date, indicating that the matching test steps + should be re-run. + * `targets`: a legacy field that indicates the subset of the input `targets` + that depend on the input `files`. + * `build_targets`: a legacy field that indicates the minimal subset of + targets needed to build all of `targets` that were affected. + * `status`: a field containing one of three strings: + + * `"Found dependency"` (build the `compile_targets`) + * `"No dependency"` (i.e., no build needed) + * `"Found dependency (all)"` (`test_targets` is returned as-is; + `compile_targets` should contain the union of `test_targets` and + `additional_compile_targets`. In this case the targets do not + need to be pruned). + +See [design_spec.md](the design spec) for more details and examples; the +differences can be subtle. We won't even go into how the `targets` and +`build_targets` differ from each other or from `compile_targets` and +`test_targets`. + +The `-b/--builder`, `-c/--config`, `-f/--config-file`, `-m/--builder-group`, +`-q/--quiet`, and `-v/--verbose` flags work as documented for `mb gen`. + +### `mb audit` + +`mb audit` is used to track the progress of the GYP->GN migration. You can +use it to check a single builder group, or all the builder groups we care +about. +See `mb help audit` for more details (most people are not expected to care +about this). + +### `mb gen` + +`mb gen` is responsible for generating the Ninja files by invoking either GYP +or GN as appropriate. It takes arguments to specify a build config and +a directory, then runs GYP or GN as appropriate: + +``` +% mb gen -m tryserver.chromium.linux -b linux_rel //out/Release +% mb gen -c linux_rel_trybot //out/Release +``` + +Either the `-c/--config` flag or the `-m/--builder-group` and `-b/--builder` +flags must be specified so that `mb` can figure out which config to use. The +`--phase` flag must also be used with builders that have multiple +build/compile steps (and only with those builders). + +By default, MB will look for a bot config file under `//ios/build/bots` (see +[design_spec.md](the design spec) for details of how the bot config files +work). If no matching one is found, will then look in +`//tools/mb/mb_config.pyl` to look up the config information, but you can +specify a custom config file using the `-f/--config-file` flag. + +The path must be a GN-style "source-absolute" path (as above). + +You can pass the `-n/--dryrun` flag to mb gen to see what will happen without +actually writing anything. + +You can pass the `-q/--quiet` flag to get mb to be silent unless there is an +error, and pass the `-v/--verbose` flag to get mb to log all of the files +that are read and written, and all the commands that are run. + +If the build config will use the Goma distributed-build system, you can pass +the path to your Goma client in the `-g/--goma-dir` flag, and it will be +incorporated into the appropriate flags for GYP or GN as needed. + +If gen ends up using GYP, the path must have a valid GYP configuration as the +last component of the path (i.e., specify `//out/Release_x64`, not `//out`). +The gyp script defaults to `//build/gyp_chromium`, but can be overridden with +the `--gyp-script` flag, e.g. `--gyp-script=gypfiles/gyp_v8`. + +### `mb help` + +Produces help output on the other subcommands + +### `mb lookup` + +Prints what command will be run by `mb gen` (like `mb gen -n` but does +not require you to specify a path). + +The `-b/--builder`, `-c/--config`, `-f/--config-file`, `-m/--builder-group`, +`--phase`, `-q/--quiet`, and `-v/--verbose` flags work as documented for +`mb gen`. + +### `mb validate` + +Does internal checking to make sure the config file is syntactically +valid and that all of the entries are used properly. It does not validate +that the flags make sense, or that the builder names are legal or +comprehensive, but it does complain about configs and mixins that aren't +used. + +The `-f/--config-file` and `-q/--quiet` flags work as documented for +`mb gen`. + +This is mostly useful as a presubmit check and for verifying changes to +the config file. + +## Isolates and Swarming + +`mb gen` is also responsible for generating the `.isolate` and +`.isolated.gen.json` files needed to run test executables through swarming +in a GN build (in a GYP build, this is done as part of the compile step). + +If you wish to generate the isolate files, pass `mb gen` the +`--swarming-targets-file` command line argument; that arg should be a path +to a file containing a list of ninja build targets to compute the runtime +dependencies for (on Windows, use the ninja target name, not the file, so +`base_unittests`, not `base_unittests.exe`). + +MB will take this file, translate each build target to the matching GN +label (e.g., `base_unittests` -> `//base:base_unittests`, write that list +to a file called `runtime_deps` in the build directory, and pass that to +`gn gen $BUILD ... --runtime-deps-list-file=$BUILD/runtime_deps`. + +Once GN has computed the lists of runtime dependencies, MB will then +look up the command line for each target (currently this is hard-coded +in [mb.py](https://code.google.com/p/chromium/codesearch?q=mb.py#chromium/src/tools/mb/mb.py&q=mb.py%20GetIsolateCommand&sq=package:chromium&type=cs)), and write out the +matching `.isolate` and `.isolated.gen.json` files. + +## The `mb_config.pyl` config file + +The `mb_config.pyl` config file is intended to enumerate all of the +supported build configurations for Chromium. Generally speaking, you +should never need to (or want to) build a configuration that isn't +listed here, and so by using the configs in this file you can avoid +having to juggle long lists of GYP_DEFINES and gn args by hand. + +`mb_config.pyl` is structured as a file containing a single PYthon Literal +expression: a dictionary with three main keys, `builder_groups`, `configs` and +`mixins`. + +The `builder_groups` key contains a nested series of dicts containing mappings +of builder group -> builder -> config . This allows us to isolate the buildbot +recipes from the actual details of the configs. The config should either +be a single string value representing a key in the `configs` dictionary, +or a list of strings, each of which is a key in the `configs` dictionary; +the latter case is for builders that do multiple compiles with different +arguments in a single build, and must *only* be used for such builders +(where a --phase argument must be supplied in each lookup or gen call). + +The `configs` key points to a dictionary of named build configurations. + +There should be an key in this dict for every supported configuration +of Chromium, meaning every configuration we have a bot for, and every +configuration commonly used by develpers but that we may not have a bot +for. + +The value of each key is a list of "mixins" that will define what that +build_config does. Each item in the list must be an entry in the dictionary +value of the `mixins` key. + +Each mixin value is itself a dictionary that contains one or more of the +following keys: + + * `gyp_crosscompile`: a boolean; if true, GYP_CROSSCOMPILE=1 is set in + the environment and passed to GYP. + * `gyp_defines`: a string containing a list of GYP_DEFINES. + * `gn_args`: a string containing a list of values passed to gn --args. + * `mixins`: a list of other mixins that should be included. + * `type`: a string with either the value `gyp` or `gn`; + setting this indicates which meta-build tool to use. + +When `mb gen` or `mb analyze` executes, it takes a config name, looks it +up in the 'configs' dict, and then does a left-to-right expansion of the +mixins; gyp_defines and gn_args values are concatenated, and the type values +override each other. + +For example, if you had: + +``` +{ + 'configs`: { + 'linux_release_trybot': ['gyp_release', 'trybot'], + 'gn_shared_debug': None, + } + 'mixins': { + 'bot': { + 'gyp_defines': 'use_goma=1 dcheck_always_on=0', + 'gn_args': 'use_goma=true dcheck_always_on=false', + }, + 'debug': { + 'gn_args': 'is_debug=true', + }, + 'gn': {'type': 'gn'}, + 'gyp_release': { + 'mixins': ['release'], + 'type': 'gyp', + }, + 'release': { + 'gn_args': 'is_debug=false', + } + 'shared': { + 'gn_args': 'is_component_build=true', + 'gyp_defines': 'component=shared_library', + }, + 'trybot': { + 'gyp_defines': 'dcheck_always_on=1', + 'gn_args': 'dcheck_always_on=true', + } + } +} +``` + +and you ran `mb gen -c linux_release_trybot //out/Release`, it would +translate into a call to `gyp_chromium -G Release` with `GYP_DEFINES` set to +`"use_goma=true dcheck_always_on=false dcheck_always_on=true"`. + +(From that you can see that mb is intentionally dumb and does not +attempt to de-dup the flags, it lets gyp do that). + +## Debugging MB + +By design, MB should be simple enough that very little can go wrong. + +The most obvious issue is that you might see different commands being +run than you expect; running `'mb -v'` will print what it's doing and +run the commands; `'mb -n'` will print what it will do but *not* run +the commands. + +If you hit weirder things than that, add some print statements to the +python script, send a question to gn-dev@chromium.org, or +[file a bug](https://crbug.com/new) with the label +'mb' and cc: dpranke@chromium.org. + + diff --git a/third_party/libwebrtc/tools_webrtc/mb/mb b/third_party/libwebrtc/tools_webrtc/mb/mb new file mode 100755 index 0000000000..007d23cb64 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/mb/mb @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +base_dir=$(dirname "$0") + +PYTHONDONTWRITEBYTECODE=1 exec python3 "$base_dir/mb.py" "$@" diff --git a/third_party/libwebrtc/tools_webrtc/mb/mb.bat b/third_party/libwebrtc/tools_webrtc/mb/mb.bat new file mode 100755 index 0000000000..1252560a08 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/mb/mb.bat @@ -0,0 +1,6 @@ +@echo off +setlocal +:: This is required with cygwin only. +PATH=%~dp0;%PATH% +set PYTHONDONTWRITEBYTECODE=1 +call vpython3 "%~dp0mb.py" %* diff --git a/third_party/libwebrtc/tools_webrtc/mb/mb.py b/third_party/libwebrtc/tools_webrtc/mb/mb.py new file mode 100755 index 0000000000..915a5b83ed --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/mb/mb.py @@ -0,0 +1,161 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""MB - the Meta-Build wrapper around GN. + +MB is a wrapper script for GN that can be used to generate build files +for sets of canned configurations and analyze them. +""" + +import os +import sys + +_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +_SRC_DIR = os.path.dirname(os.path.dirname(_SCRIPT_DIR)) +sys.path.insert(0, _SRC_DIR) + +from tools.mb import mb + + +def _GetExecutable(target, platform): + executable_prefix = '.\\' if platform == 'win32' else './' + executable_suffix = '.exe' if platform == 'win32' else '' + return executable_prefix + target + executable_suffix + + +def main(args): + mbw = WebRTCMetaBuildWrapper() + return mbw.Main(args) + + +class WebRTCMetaBuildWrapper(mb.MetaBuildWrapper): + def __init__(self): + super().__init__() + # Make sure default_config and default_isolate_map are attributes of the + # parent class before changing their values. + # pylint: disable=access-member-before-definition + assert self.default_config + assert self.default_isolate_map + self.default_config = os.path.join(_SCRIPT_DIR, 'mb_config.pyl') + self.default_isolate_map = os.path.join(_SRC_DIR, 'infra', 'specs', + 'gn_isolate_map.pyl') + + def GetSwarmingCommand(self, target, vals): + isolate_map = self.ReadIsolateMap() + test_type = isolate_map[target]['type'] + + is_android = 'target_os="android"' in vals['gn_args'] + is_linux = self.platform.startswith('linux') and not is_android + is_ios = 'target_os="ios"' in vals['gn_args'] + + if test_type == 'nontest': + self.WriteFailureAndRaise('We should not be isolating %s.' % target, + output_path=None) + if test_type not in ('console_test_launcher', 'windowed_test_launcher', + 'non_parallel_console_test_launcher', 'raw', + 'additional_compile_target', 'junit_test', 'script'): + self.WriteFailureAndRaise('No command line for ' + '%s found (test type %s).' % + (target, test_type), + output_path=None) + + cmdline = [] + extra_files = [ + '../../.vpython3', + '../../testing/test_env.py', + ] + vpython_exe = 'vpython3' + + if isolate_map[target].get('script'): + cmdline += [ + vpython_exe, + '../../' + self.ToSrcRelPath(isolate_map[target]['script']) + ] + elif is_android: + cmdline += [ + vpython_exe, '../../build/android/test_wrapper/logdog_wrapper.py', + '--target', target, '--logdog-bin-cmd', '../../bin/logdog_butler', + '--logcat-output-file', '${ISOLATED_OUTDIR}/logcats', + '--store-tombstones' + ] + elif is_ios: + cmdline += [ + vpython_exe, '../../tools_webrtc/flags_compatibility.py', + 'bin/run_%s' % target + ] + extra_files.append('../../tools_webrtc/flags_compatibility.py') + elif test_type == 'raw': + cmdline += [vpython_exe, '../../tools_webrtc/flags_compatibility.py'] + extra_files.append('../../tools_webrtc/flags_compatibility.py') + cmdline.append(_GetExecutable(target, self.platform)) + else: + if isolate_map[target].get('use_webcam', False): + cmdline += [ + vpython_exe, '../../tools_webrtc/ensure_webcam_is_running.py' + ] + extra_files.append('../../tools_webrtc/ensure_webcam_is_running.py') + + # is_linux uses use_ozone and x11 by default. + use_x11 = is_linux + + xvfb = use_x11 and test_type == 'windowed_test_launcher' + if xvfb: + cmdline += [vpython_exe, '../../testing/xvfb.py'] + extra_files.append('../../testing/xvfb.py') + else: + cmdline += [vpython_exe, '../../testing/test_env.py'] + + extra_files += [ + '../../third_party/gtest-parallel/gtest-parallel', + '../../third_party/gtest-parallel/gtest_parallel.py', + '../../tools_webrtc/gtest-parallel-wrapper.py', + ] + output_dir = '${ISOLATED_OUTDIR}/test_logs' + timeout = isolate_map[target].get('timeout', 900) + cmdline += [ + '../../tools_webrtc/gtest-parallel-wrapper.py', + '--output_dir=%s' % output_dir, + '--gtest_color=no', + # We tell gtest-parallel to interrupt the test after 900 + # seconds, so it can exit cleanly and report results, + # instead of being interrupted by swarming and not + # reporting anything. + '--timeout=%s' % timeout, + ] + if test_type == 'non_parallel_console_test_launcher': + # Still use the gtest-parallel-wrapper.py script since we + # need it to run tests on swarming, but don't execute tests + # in parallel. + cmdline.append('--workers=1') + + asan = 'is_asan=true' in vals['gn_args'] + lsan = 'is_lsan=true' in vals['gn_args'] + msan = 'is_msan=true' in vals['gn_args'] + tsan = 'is_tsan=true' in vals['gn_args'] + sanitizer = asan or lsan or msan or tsan + if not sanitizer: + # Retry would hide most sanitizers detections. + cmdline.append('--retry_failed=3') + + cmdline.append(_GetExecutable(target, self.platform)) + + cmdline.extend([ + '--asan=%d' % asan, + '--lsan=%d' % lsan, + '--msan=%d' % msan, + '--tsan=%d' % tsan, + ]) + + cmdline += isolate_map[target].get('args', []) + + return cmdline, extra_files + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/third_party/libwebrtc/tools_webrtc/mb/mb_config.pyl b/third_party/libwebrtc/tools_webrtc/mb/mb_config.pyl new file mode 100644 index 0000000000..901bd4ab23 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/mb/mb_config.pyl @@ -0,0 +1,467 @@ +# Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +# FOR DETAILS ON THIS FILE SEE THE MAIN COPY IN //tools/mb/mb_config.pyl. +# This is configuration for standalone WebRTC bots. It is used to keep the bot +# configurations source-side instead of in the buildbot scripts. That makes it +# easy to try different configurations of GN args in tryjob patches. + +{ + # This is a map of buildbot builder group names -> buildbot builder names -> + # config names (where each config name is a key in the 'configs' dict, + # above). mb uses this dict to look up which config to use for a given bot. + # The builders should be sorted by the order they appear in the /builders + # page on the buildbots, *not* alphabetically. + 'builder_groups': { + # This is required because WebRTC mb.py overwrites the default configs + # and Chromium's mb.py checks the default config contains 'chromium'. + 'chromium': {}, + 'chromium.infra.codesearch': { + 'codesearch-gen-webrtc-android': { + 'android': 'android_debug_static_bot_arm', + }, + 'codesearch-gen-webrtc-linux': { + 'linux': 'codesearch_gen_linux_bot', + } + }, + 'client.webrtc': { + # Android + 'Android32 (M Nexus5X)': 'android_release_bot_arm', + 'Android32 (M Nexus5X)(dbg)': 'android_debug_static_bot_arm', + 'Android32 (more configs)': { + 'bwe_test_logging': 'bwe_test_logging_android_arm', + 'dummy_audio_file_devices_no_protobuf': + 'dummy_audio_file_devices_no_protobuf_android_arm', + 'rtti_no_sctp': 'rtti_no_sctp_android_arm', + }, + 'Android32 Builder arm': 'android_pure_release_bot_arm', + 'Android32 Builder x86': 'android_release_bot_x86', + 'Android32 Builder x86 (dbg)': 'android_debug_static_bot_x86', + 'Android64 (M Nexus5X)': 'android_release_bot_arm64', + 'Android64 (M Nexus5X)(dbg)': 'android_debug_static_bot_arm64', + 'Android64 Builder arm64': 'android_pure_release_bot_arm64', + 'Android64 Builder x64 (dbg)': 'android_debug_static_bot_x64', + + # Linux + # "More configs" bots will build all the following configs in sequence. + # This is using MB's "phases" feature. + 'Linux (more configs)': { + 'bwe_test_logging': 'bwe_test_logging_x64', + 'dummy_audio_file_devices_no_protobuf': + 'dummy_audio_file_devices_no_protobuf_x64', + 'rtti_no_sctp': 'rtti_no_sctp_x64', + }, + 'Linux Asan': 'asan_lsan_clang_release_bot_x64', + 'Linux MSan': 'msan_clang_release_bot_x64', + 'Linux Tsan v2': 'tsan_clang_release_bot_x64', + 'Linux UBSan': 'ubsan_clang_release_bot_x64', + 'Linux UBSan vptr': 'ubsan_vptr_clang_release_bot_x64', + 'Linux32 Debug': 'no_h264_debug_bot_x86', + 'Linux32 Debug (ARM)': 'debug_bot_arm', + 'Linux32 Release': 'release_bot_x86', + 'Linux32 Release (ARM)': 'release_bot_arm', + 'Linux64 Builder': 'pure_release_bot_x64', + 'Linux64 Debug': 'debug_bot_x64', + 'Linux64 Debug (ARM)': 'debug_bot_arm64', + 'Linux64 Release': 'release_bot_x64', + 'Linux64 Release (ARM)': 'release_bot_arm64', + 'Linux64 Release (Libfuzzer)': 'libfuzzer_asan_release_bot_x64', + + # Mac + 'Mac Asan': 'mac_asan_clang_release_bot_x64', + 'Mac64 Builder': 'pure_release_bot_x64', + 'Mac64 Debug': 'debug_bot_x64', + 'Mac64 Release': 'release_bot_x64', + 'MacARM64 M1 Release': 'release_bot_arm64', + 'MacArm64 Builder': 'release_bot_arm64', + + # Windows + 'Win (more configs)': { + 'bwe_test_logging': 'bwe_test_logging_x86', + 'dummy_audio_file_devices_no_protobuf': + 'dummy_audio_file_devices_no_protobuf_x86', + 'rtti_no_sctp': 'rtti_no_sctp_no_unicode_win_x86', + }, + 'Win32 Builder (Clang)': 'win_clang_pure_release_bot_x86', + 'Win32 Debug (Clang)': 'win_clang_debug_bot_x86', + 'Win32 Release (Clang)': 'win_clang_release_bot_x86', + 'Win64 ASan': 'win_asan_clang_release_bot_x64', + 'Win64 Debug (Clang)': 'win_clang_debug_bot_x64', + 'Win64 Release (Clang)': 'win_clang_release_bot_x64', + + # iOS + 'iOS64 Debug': 'ios_debug_bot_arm64', + 'iOS64 Release': 'ios_release_bot_arm64', + 'iOS64 Sim Debug (iOS 12)': 'ios_debug_bot_x64', + 'iOS64 Sim Debug (iOS 13)': 'ios_debug_bot_x64', + 'iOS64 Sim Debug (iOS 14)': 'ios_debug_bot_x64', + }, + 'client.webrtc.fyi': { + # Android + 'Android ASan (swarming)': 'android_asan_shared_release_bot_arm', + 'Android Perf (swarming)': 'android_pure_release_bot_arm', + + # Mac + 'Mac (swarming)': 'release_bot_x64', + + # Windows + 'Win (swarming)': 'release_bot_x86', + 'Win64 Debug (Win10)': 'debug_bot_x64', + 'Win64 Debug (Win8)': 'debug_bot_x64', + }, + 'client.webrtc.perf': { + # These are here because testers need to gn gen + ninja for the + # webrtc_dashboard_upload target (otherwise a tester would not need to + # build anything). + # TODO(http://crbug.com/1029452): Nuke these and isolate on builder + # instead? + 'Perf Android32 (M AOSP Nexus6)': 'release_bot_x64', + 'Perf Android32 (M Nexus5)': 'release_bot_x64', + 'Perf Android64 (M Nexus5X)': 'release_bot_x64', + 'Perf Android64 (O Pixel2)': 'release_bot_x64', + 'Perf Linux Bionic': 'release_bot_x64', + 'Perf Linux Trusty': 'release_bot_x64', + 'Perf Mac 11': 'release_bot_x64', + 'Perf Mac M1 Arm64 12': 'release_bot_x64', + 'Perf Win7': 'release_bot_x64', + }, + 'internal.client.webrtc': { + 'iOS64 Debug': 'ios_internal_debug_bot_arm64', + 'iOS64 Perf': 'ios_internal_pure_release_bot_arm64', + 'iOS64 Release': 'ios_internal_release_bot_arm64', + }, + 'tryserver.webrtc': { + # Android + 'android_arm64_dbg': 'android_release_bot_arm64', + 'android_arm64_rel': 'android_release_bot_arm64', + 'android_arm_dbg': 'android_debug_static_bot_arm', + 'android_arm_more_configs': { + 'bwe_test_logging': 'bwe_test_logging_android_arm', + 'dummy_audio_file_devices_no_protobuf': + 'dummy_audio_file_devices_no_protobuf_android_arm', + 'rtti_no_sctp': 'rtti_no_sctp_android_arm', + }, + 'android_arm_rel': 'android_release_bot_arm', + 'android_compile_arm64_dbg': 'android_debug_static_bot_arm64', + 'android_compile_arm64_rel': 'android_pure_release_bot_arm64', + 'android_compile_arm_dbg': 'android_debug_static_bot_arm', + 'android_compile_arm_rel': 'android_pure_release_bot_arm', + 'android_compile_x64_dbg': 'android_debug_static_bot_x64', + 'android_compile_x64_rel': 'android_release_bot_x64', + 'android_compile_x86_dbg': 'android_debug_static_bot_x86', + 'android_compile_x86_rel': 'android_release_bot_x86', + + # iOS + 'ios_compile_arm64_dbg': 'ios_debug_bot_arm64', + 'ios_compile_arm64_rel': 'ios_release_bot_arm64', + 'ios_sim_x64_dbg_ios12': 'ios_debug_bot_x64', + 'ios_sim_x64_dbg_ios13': 'ios_debug_bot_x64', + 'ios_sim_x64_dbg_ios14': 'ios_debug_bot_x64', + + # Linux + 'linux_asan': 'asan_lsan_clang_release_bot_x64', + 'linux_compile_arm64_dbg': 'debug_bot_arm64', + 'linux_compile_arm64_rel': 'release_bot_arm64', + 'linux_compile_arm_dbg': 'debug_bot_arm', + 'linux_compile_arm_rel': 'release_bot_arm', + 'linux_compile_dbg': 'debug_bot_x64', + 'linux_compile_rel': 'pure_release_bot_x64', + 'linux_compile_x86_dbg': 'debug_bot_x86', + 'linux_compile_x86_rel': 'pure_release_bot_x86', + 'linux_dbg': 'debug_bot_x64', + 'linux_libfuzzer_rel': 'libfuzzer_asan_release_bot_x64', + 'linux_more_configs': { + 'bwe_test_logging': 'bwe_test_logging_x64', + 'dummy_audio_file_devices_no_protobuf': + 'dummy_audio_file_devices_no_protobuf_x64', + 'rtti_no_sctp': 'rtti_no_sctp_x64', + }, + 'linux_msan': 'msan_clang_release_bot_x64', + 'linux_rel': 'release_bot_x64', + 'linux_tsan2': 'tsan_clang_release_bot_x64', + 'linux_ubsan': 'ubsan_clang_release_bot_x64', + 'linux_ubsan_vptr': 'ubsan_vptr_clang_release_bot_x64', + 'linux_x86_dbg': 'no_h264_debug_bot_x86', + 'linux_x86_rel': 'release_bot_x86', + + # Mac + 'mac_asan': 'mac_asan_clang_release_bot_x64', + 'mac_compile_dbg': 'debug_bot_x64', + 'mac_compile_rel': 'pure_release_bot_x64', + 'mac_dbg': 'debug_bot_x64', + 'mac_dbg_m1': 'debug_bot_arm64', + 'mac_rel': 'release_bot_x64', + 'mac_rel_m1': 'release_bot_arm64', + + # Windows + 'win_asan': 'win_asan_clang_release_bot_x64', + 'win_compile_x64_clang_dbg': 'win_clang_debug_bot_x64', + 'win_compile_x64_clang_rel': 'win_clang_release_bot_x64', + 'win_compile_x86_clang_dbg': 'win_clang_debug_bot_x86', + 'win_compile_x86_clang_rel': 'win_clang_release_bot_x86', + 'win_x64_clang_dbg': 'win_clang_debug_bot_x64', + 'win_x64_clang_dbg_win10': 'win_clang_debug_bot_x64', + 'win_x64_clang_rel': 'win_clang_release_bot_x64', + 'win_x86_clang_dbg': 'win_clang_debug_bot_x86', + 'win_x86_clang_rel': 'win_clang_release_bot_x86', + 'win_x86_more_configs': { + 'bwe_test_logging': 'bwe_test_logging_x86', + 'dummy_audio_file_devices_no_protobuf': + 'dummy_audio_file_devices_no_protobuf_x86', + 'rtti_no_sctp': 'rtti_no_sctp_no_unicode_win_x86', + }, + } + }, + + # This is the list of configs that you can pass to mb; each config + # represents a particular combination of gn args that we must support. + # A given config *may* be platform-specific but is not necessarily so (i.e., + # we might have mac, win, and linux bots all using the 'release_bot' config). + 'configs': { + 'android_asan_shared_release_bot_arm': + ['android', 'asan', 'clang', 'pure_release_bot', 'arm'], + 'android_debug_static_bot_arm': ['android', 'debug_static_bot', 'arm'], + 'android_debug_static_bot_arm64': ['android', 'debug_static_bot', 'arm64'], + 'android_debug_static_bot_x64': ['android', 'debug_static_bot', 'x64'], + 'android_debug_static_bot_x86': ['android', 'debug_static_bot', 'x86'], + 'android_pure_release_bot_arm': ['android', 'pure_release_bot', 'arm'], + 'android_pure_release_bot_arm64': ['android', 'pure_release_bot', 'arm64'], + 'android_release_bot_arm': ['android', 'release_bot', 'arm'], + 'android_release_bot_arm64': ['android', 'release_bot', 'arm64'], + 'android_release_bot_x64': ['android', 'release_bot', 'x64'], + 'android_release_bot_x86': ['android', 'release_bot', 'x86'], + 'asan_lsan_clang_release_bot_x64': + ['asan', 'lsan', 'clang', 'openh264', 'pure_release_bot', 'x64'], + 'bwe_test_logging_android_arm': + ['android', 'debug_static_bot', 'arm', 'bwe_test_logging'], + 'bwe_test_logging_x64': ['debug_bot', 'x64', 'bwe_test_logging'], + 'bwe_test_logging_x86': ['debug_bot', 'x86', 'bwe_test_logging'], + 'codesearch_gen_linux_bot': ['openh264', 'debug_bot', 'minimal_symbols'], + # TODO(kjellander): Restore Goma for this when crbug.com/726706 is fixed. + 'debug_bot_arm': ['openh264', 'debug', 'arm'], + 'debug_bot_arm64': ['openh264', 'debug_bot', 'arm64'], + 'debug_bot_x64': ['openh264', 'debug_bot', 'x64'], + 'debug_bot_x86': ['openh264', 'debug_bot', 'x86'], + 'dummy_audio_file_devices_no_protobuf_android_arm': [ + 'android', 'debug_static_bot', 'arm', 'dummy_audio_file_devices', + 'no_protobuf' + ], + 'dummy_audio_file_devices_no_protobuf_x64': + ['debug_bot', 'x64', 'dummy_audio_file_devices', 'no_protobuf'], + 'dummy_audio_file_devices_no_protobuf_x86': + ['debug_bot', 'x86', 'dummy_audio_file_devices', 'no_protobuf'], + 'ios_debug_bot_arm64': + ['ios', 'debug_bot', 'arm64', 'no_ios_code_signing', 'xctest'], + 'ios_debug_bot_x64': ['ios', 'debug_bot', 'x64', 'xctest'], + 'ios_internal_debug_bot_arm64': [ + 'ios', 'debug_bot', 'arm64', 'ios_code_signing_identity_description', + 'xctest' + ], + 'ios_internal_pure_release_bot_arm64': [ + 'ios', 'pure_release_bot', 'arm64', + 'ios_code_signing_identity_description', 'xctest' + ], + 'ios_internal_release_bot_arm64': [ + 'ios', 'release_bot', 'arm64', 'ios_code_signing_identity_description', + 'xctest' + ], + 'ios_release_bot_arm64': + ['ios', 'release_bot', 'arm64', 'no_ios_code_signing', 'xctest'], + 'libfuzzer_asan_release_bot_x64': [ + 'libfuzzer', 'asan', 'optimize_for_fuzzing', 'openh264', + 'pure_release_bot', 'x64' + ], + 'mac_asan_clang_release_bot_x64': [ + 'asan', + 'clang', + 'openh264', + 'pure_release_bot', + 'x64', + ], + 'msan_clang_release_bot_x64': + ['msan', 'clang', 'openh264', 'pure_release_bot', 'x64'], + 'no_h264_debug_bot_x86': ['debug_bot', 'x86'], + 'pure_release_bot_x64': ['openh264', 'pure_release_bot', 'x64'], + 'pure_release_bot_x86': ['openh264', 'pure_release_bot', 'x86'], + 'release_bot_arm': ['openh264', 'release_bot', 'arm'], + 'release_bot_arm64': ['openh264', 'release_bot', 'arm64'], + 'release_bot_x64': ['openh264', 'release_bot', 'x64'], + 'release_bot_x86': ['openh264', 'release_bot', 'x86'], + 'rtti_no_sctp_android_arm': [ + 'android', 'debug_static_bot', 'arm', 'rtti', 'no_sctp' + ], + 'rtti_no_sctp_no_unicode_win_x86': + ['debug_bot', 'x86', 'rtti', 'no_sctp', 'win_undef_unicode'], + 'rtti_no_sctp_x64': ['debug_bot', 'x64', 'rtti', 'no_sctp'], + 'tsan_clang_release_bot_x64': + ['tsan', 'clang', 'openh264', 'pure_release_bot', 'x64'], + 'ubsan_clang_release_bot_x64': + ['ubsan', 'clang', 'openh264', 'pure_release_bot', 'x64'], + 'ubsan_vptr_clang_release_bot_x64': + ['ubsan_vptr', 'clang', 'openh264', 'pure_release_bot', 'x64'], + 'win_asan_clang_release_bot_x64': [ + 'asan', + 'clang', + 'full_symbols', + 'openh264', + 'pure_release_bot', + 'x64', + 'win_fastlink', + ], + 'win_clang_debug_bot_x64': [ + 'clang', + 'openh264', + 'debug_bot', + 'x64', + ], + 'win_clang_debug_bot_x86': [ + 'clang', + 'openh264', + 'debug_bot', + 'x86', + ], + 'win_clang_pure_release_bot_x86': [ + 'clang', + 'openh264', + 'pure_release_bot', + 'x86', + ], + 'win_clang_release_bot_x64': [ + 'clang', + 'openh264', + 'release_bot', + 'x64', + ], + 'win_clang_release_bot_x86': [ + 'clang', + 'openh264', + 'release_bot', + 'x86', + ], + }, + + # This is a dict mapping a given 'mixin' name to a dict of settings that + # mb should use. See //tools/mb/docs/user_guide.md for more information. + 'mixins': { + 'android': { + 'gn_args': 'target_os="android"', + }, + 'arm': { + 'gn_args': 'target_cpu="arm"', + }, + 'arm64': { + 'gn_args': 'target_cpu="arm64"', + }, + 'asan': { + 'gn_args': 'is_asan=true', + }, + 'bwe_test_logging': { + 'gn_args': 'rtc_enable_bwe_test_logging=true', + }, + # is_clang=true by default, this is only to guard from upstream changes. + 'clang': { + 'gn_args': 'is_clang=true', + }, + 'dcheck_always_on': { + 'gn_args': 'dcheck_always_on=true', + }, + 'dcheck_off': { + 'gn_args': 'dcheck_always_on=false', + }, + 'debug': { + 'gn_args': 'is_debug=true', + }, + 'debug_bot': { + 'mixins': ['debug', 'goma'], + }, + 'debug_static_bot': { + 'mixins': ['debug', 'minimal_symbols', 'goma'], + }, + 'dummy_audio_file_devices': { + 'gn_args': 'rtc_use_dummy_audio_file_devices=true', + }, + 'full_symbols': { + 'gn_args': 'symbol_level=2', + }, + 'goma': { + 'gn_args': 'use_goma=true', + }, + 'ios': { + 'gn_args': 'target_os="ios"', + }, + 'ios_code_signing_identity_description': { + 'gn_args': 'ios_code_signing_identity_description="Apple Development"', + }, + 'libfuzzer': { + 'gn_args': 'use_libfuzzer=true', + }, + 'lsan': { + 'gn_args': 'is_lsan=true', + }, + 'minimal_symbols': { + 'gn_args': 'symbol_level=1', + }, + 'msan': { + 'gn_args': 'is_msan=true msan_track_origins=2', + }, + 'no_ios_code_signing': { + 'gn_args': 'ios_enable_code_signing=false', + }, + 'no_protobuf': { + 'gn_args': 'rtc_enable_protobuf=false', + }, + 'no_sctp': { + 'gn_args': 'rtc_enable_sctp=false', + }, + 'openh264': { + 'gn_args': 'ffmpeg_branding="Chrome" rtc_use_h264=true', + }, + 'optimize_for_fuzzing': { + 'gn_args': 'optimize_for_fuzzing=true', + }, + # The 'pure_release_bot' configuration is for release bots that are doing a + # 100% release build without DCHECKs while 'release_bot' is a partial + # release configs since `dcheck_always_on` is set to true. + 'pure_release_bot': { + 'mixins': ['release', 'goma', 'dcheck_off'], + }, + 'release': { + 'gn_args': 'is_debug=false', + }, + 'release_bot': { + 'mixins': ['pure_release_bot', 'dcheck_always_on'], + }, + 'rtti': { + 'gn_args': 'use_rtti=true', + }, + 'tsan': { + 'gn_args': 'is_tsan=true', + }, + 'ubsan': { + 'gn_args': 'is_ubsan=true is_ubsan_no_recover=true', + }, + 'ubsan_vptr': { + 'gn_args': 'is_ubsan_vptr=true is_ubsan_no_recover=true', + }, + 'win_fastlink': { + 'gn_args': 'is_win_fastlink=true', + }, + 'win_undef_unicode': { + 'gn_args': 'rtc_win_undef_unicode=true', + }, + 'x64': { + 'gn_args': 'target_cpu="x64"', + }, + 'x86': { + 'gn_args': 'target_cpu="x86"', + }, + 'xctest': { + 'gn_args': 'enable_run_ios_unittests_with_xctest=true', + }, + }, +} diff --git a/third_party/libwebrtc/tools_webrtc/mb/mb_unittest.py b/third_party/libwebrtc/tools_webrtc/mb/mb_unittest.py new file mode 100755 index 0000000000..109d2bcb2d --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/mb/mb_unittest.py @@ -0,0 +1,780 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Tests for mb.py.""" + +import ast +import os +import re +import sys +import tempfile +import unittest + +_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +_SRC_DIR = os.path.dirname(os.path.dirname(_SCRIPT_DIR)) +sys.path.insert(0, _SRC_DIR) + +from tools_webrtc.mb import mb + + +class FakeMBW(mb.WebRTCMetaBuildWrapper): + def __init__(self, win32=False): + super().__init__() + + # Override vars for test portability. + if win32: + self.chromium_src_dir = 'c:\\fake_src' + self.default_config = 'c:\\fake_src\\tools_webrtc\\mb\\mb_config.pyl' + self.default_isolate_map = ('c:\\fake_src\\testing\\buildbot\\' + 'gn_isolate_map.pyl') + self.platform = 'win32' + self.executable = 'c:\\python\\vpython3.exe' + self.sep = '\\' + self.cwd = 'c:\\fake_src\\out\\Default' + else: + self.chromium_src_dir = '/fake_src' + self.default_config = '/fake_src/tools_webrtc/mb/mb_config.pyl' + self.default_isolate_map = '/fake_src/testing/buildbot/gn_isolate_map.pyl' + self.executable = '/usr/bin/vpython3' + self.platform = 'linux2' + self.sep = '/' + self.cwd = '/fake_src/out/Default' + + self.files = {} + self.dirs = set() + self.calls = [] + self.cmds = [] + self.cross_compile = None + self.out = '' + self.err = '' + self.rmdirs = [] + + def ExpandUser(self, path): + # pylint: disable=no-self-use + return '$HOME/%s' % path + + def Exists(self, path): + abs_path = self._AbsPath(path) + return self.files.get(abs_path) is not None or abs_path in self.dirs + + def ListDir(self, path): + dir_contents = [] + for f in list(self.files.keys()) + list(self.dirs): + head, _ = os.path.split(f) + if head == path: + dir_contents.append(f) + return dir_contents + + def MaybeMakeDirectory(self, path): + abpath = self._AbsPath(path) + self.dirs.add(abpath) + + def PathJoin(self, *comps): + return self.sep.join(comps) + + def ReadFile(self, path): + try: + return self.files[self._AbsPath(path)] + except KeyError as e: + raise IOError('%s not found' % path) from e + + def WriteFile(self, path, contents, force_verbose=False): + if self.args.dryrun or self.args.verbose or force_verbose: + self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path)) + abpath = self._AbsPath(path) + self.files[abpath] = contents + + def Call(self, cmd, env=None, capture_output=True, stdin=None): + del env + del capture_output + del stdin + self.calls.append(cmd) + if self.cmds: + return self.cmds.pop(0) + return 0, '', '' + + def Print(self, *args, **kwargs): + sep = kwargs.get('sep', ' ') + end = kwargs.get('end', '\n') + f = kwargs.get('file', sys.stdout) + if f == sys.stderr: + self.err += sep.join(args) + end + else: + self.out += sep.join(args) + end + + def TempDir(self): + tmp_dir = os.path.join(tempfile.gettempdir(), 'mb_test') + self.dirs.add(tmp_dir) + return tmp_dir + + def TempFile(self, mode='w'): + del mode + return FakeFile(self.files) + + def RemoveFile(self, path): + abpath = self._AbsPath(path) + self.files[abpath] = None + + def RemoveDirectory(self, abs_path): + # Normalize the passed-in path to handle different working directories + # used during unit testing. + abs_path = self._AbsPath(abs_path) + self.rmdirs.append(abs_path) + files_to_delete = [f for f in self.files if f.startswith(abs_path)] + for f in files_to_delete: + self.files[f] = None + + def _AbsPath(self, path): + if not ((self.platform == 'win32' and path.startswith('c:')) or + (self.platform != 'win32' and path.startswith('/'))): + path = self.PathJoin(self.cwd, path) + if self.sep == '\\': + return re.sub(r'\\+', r'\\', path) + return re.sub('/+', '/', path) + + +class FakeFile: + # pylint: disable=invalid-name + def __init__(self, files): + self.name = '/tmp/file' + self.buf = '' + self.files = files + + def write(self, contents): + self.buf += contents + + def close(self): + self.files[self.name] = self.buf + + +TEST_CONFIG = """\ +{ + 'builder_groups': { + 'chromium': {}, + 'fake_group': { + 'fake_builder': 'rel_bot', + 'fake_debug_builder': 'debug_goma', + 'fake_args_bot': 'fake_args_bot', + 'fake_multi_phase': { 'phase_1': 'phase_1', 'phase_2': 'phase_2'}, + 'fake_android_bot': 'android_bot', + 'fake_args_file': 'args_file_goma', + 'fake_ios_error': 'ios_error', + }, + }, + 'configs': { + 'args_file_goma': ['fake_args_bot', 'goma'], + 'fake_args_bot': ['fake_args_bot'], + 'rel_bot': ['rel', 'goma', 'fake_feature1'], + 'debug_goma': ['debug', 'goma'], + 'phase_1': ['rel', 'phase_1'], + 'phase_2': ['rel', 'phase_2'], + 'android_bot': ['android'], + 'ios_error': ['error'], + }, + 'mixins': { + 'error': { + 'gn_args': 'error', + }, + 'fake_args_bot': { + 'args_file': '//build/args/bots/fake_group/fake_args_bot.gn', + }, + 'fake_feature1': { + 'gn_args': 'enable_doom_melon=true', + }, + 'goma': { + 'gn_args': 'use_goma=true', + }, + 'phase_1': { + 'gn_args': 'phase=1', + }, + 'phase_2': { + 'gn_args': 'phase=2', + }, + 'rel': { + 'gn_args': 'is_debug=false dcheck_always_on=false', + }, + 'debug': { + 'gn_args': 'is_debug=true', + }, + 'android': { + 'gn_args': 'target_os="android" dcheck_always_on=false', + } + }, +} +""" + + +def CreateFakeMBW(files=None, win32=False): + mbw = FakeMBW(win32=win32) + mbw.files.setdefault(mbw.default_config, TEST_CONFIG) + mbw.files.setdefault( + mbw.ToAbsPath('//testing/buildbot/gn_isolate_map.pyl'), '''{ + "foo_unittests": { + "label": "//foo:foo_unittests", + "type": "console_test_launcher", + "args": [], + }, + }''') + mbw.files.setdefault( + mbw.ToAbsPath('//build/args/bots/fake_group/fake_args_bot.gn'), + 'is_debug = false\ndcheck_always_on=false\n') + mbw.files.setdefault(mbw.ToAbsPath('//tools/mb/rts_banned_suites.json'), '{}') + if files: + for path, contents in list(files.items()): + mbw.files[path] = contents + if path.endswith('.runtime_deps'): + + def FakeCall(cmd, env=None, capture_output=True, stdin=None): + # pylint: disable=cell-var-from-loop + del cmd + del env + del capture_output + del stdin + mbw.files[path] = contents + return 0, '', '' + + # pylint: disable=invalid-name + mbw.Call = FakeCall + return mbw + + +class UnitTest(unittest.TestCase): + # pylint: disable=invalid-name + def check(self, + args, + mbw=None, + files=None, + out=None, + err=None, + ret=None, + env=None): + if not mbw: + mbw = CreateFakeMBW(files) + + try: + prev_env = os.environ.copy() + os.environ = env if env else prev_env + actual_ret = mbw.Main(args) + finally: + os.environ = prev_env + self.assertEqual( + actual_ret, ret, + "ret: %s, out: %s, err: %s" % (actual_ret, mbw.out, mbw.err)) + if out is not None: + self.assertEqual(mbw.out, out) + if err is not None: + self.assertEqual(mbw.err, err) + return mbw + + def test_gen_swarming(self): + files = { + '/tmp/swarming_targets': + 'foo_unittests\n', + '/fake_src/testing/buildbot/gn_isolate_map.pyl': + ("{'foo_unittests': {" + " 'label': '//foo:foo_unittests'," + " 'type': 'raw'," + " 'args': []," + "}}\n"), + '/fake_src/out/Default/foo_unittests.runtime_deps': ("foo_unittests\n"), + } + mbw = CreateFakeMBW(files) + self.check([ + 'gen', '-c', 'debug_goma', '--swarming-targets-file', + '/tmp/swarming_targets', '//out/Default' + ], + mbw=mbw, + ret=0) + self.assertIn('/fake_src/out/Default/foo_unittests.isolate', mbw.files) + self.assertIn('/fake_src/out/Default/foo_unittests.isolated.gen.json', + mbw.files) + + def test_gen_swarming_android(self): + test_files = { + '/tmp/swarming_targets': + 'foo_unittests\n', + '/fake_src/testing/buildbot/gn_isolate_map.pyl': + ("{'foo_unittests': {" + " 'label': '//foo:foo_unittests'," + " 'type': 'console_test_launcher'," + "}}\n"), + '/fake_src/out/Default/foo_unittests.runtime_deps': ("foo_unittests\n"), + } + mbw = self.check([ + 'gen', '-c', 'android_bot', '//out/Default', '--swarming-targets-file', + '/tmp/swarming_targets', '--isolate-map-file', + '/fake_src/testing/buildbot/gn_isolate_map.pyl' + ], + files=test_files, + ret=0) + + isolate_file = mbw.files['/fake_src/out/Default/foo_unittests.isolate'] + isolate_file_contents = ast.literal_eval(isolate_file) + files = isolate_file_contents['variables']['files'] + command = isolate_file_contents['variables']['command'] + + self.assertEqual( + files, + ['../../.vpython3', '../../testing/test_env.py', 'foo_unittests']) + self.assertEqual(command, [ + 'vpython3', + '../../build/android/test_wrapper/logdog_wrapper.py', + '--target', + 'foo_unittests', + '--logdog-bin-cmd', + '../../bin/logdog_butler', + '--logcat-output-file', + '${ISOLATED_OUTDIR}/logcats', + '--store-tombstones', + ]) + + def test_gen_swarming_android_junit_test(self): + test_files = { + '/tmp/swarming_targets': + 'foo_unittests\n', + '/fake_src/testing/buildbot/gn_isolate_map.pyl': + ("{'foo_unittests': {" + " 'label': '//foo:foo_unittests'," + " 'type': 'junit_test'," + "}}\n"), + '/fake_src/out/Default/foo_unittests.runtime_deps': ("foo_unittests\n"), + } + mbw = self.check([ + 'gen', '-c', 'android_bot', '//out/Default', '--swarming-targets-file', + '/tmp/swarming_targets', '--isolate-map-file', + '/fake_src/testing/buildbot/gn_isolate_map.pyl' + ], + files=test_files, + ret=0) + + isolate_file = mbw.files['/fake_src/out/Default/foo_unittests.isolate'] + isolate_file_contents = ast.literal_eval(isolate_file) + files = isolate_file_contents['variables']['files'] + command = isolate_file_contents['variables']['command'] + + self.assertEqual( + files, + ['../../.vpython3', '../../testing/test_env.py', 'foo_unittests']) + self.assertEqual(command, [ + 'vpython3', + '../../build/android/test_wrapper/logdog_wrapper.py', + '--target', + 'foo_unittests', + '--logdog-bin-cmd', + '../../bin/logdog_butler', + '--logcat-output-file', + '${ISOLATED_OUTDIR}/logcats', + '--store-tombstones', + ]) + + def test_gen_timeout(self): + test_files = { + '/tmp/swarming_targets': + 'foo_unittests\n', + '/fake_src/testing/buildbot/gn_isolate_map.pyl': + ("{'foo_unittests': {" + " 'label': '//foo:foo_unittests'," + " 'type': 'non_parallel_console_test_launcher'," + " 'timeout': 500," + "}}\n"), + '/fake_src/out/Default/foo_unittests.runtime_deps': ("foo_unittests\n"), + } + mbw = self.check([ + 'gen', '-c', 'debug_goma', '//out/Default', '--swarming-targets-file', + '/tmp/swarming_targets', '--isolate-map-file', + '/fake_src/testing/buildbot/gn_isolate_map.pyl' + ], + files=test_files, + ret=0) + + isolate_file = mbw.files['/fake_src/out/Default/foo_unittests.isolate'] + isolate_file_contents = ast.literal_eval(isolate_file) + files = isolate_file_contents['variables']['files'] + command = isolate_file_contents['variables']['command'] + + self.assertEqual(files, [ + '../../.vpython3', + '../../testing/test_env.py', + '../../third_party/gtest-parallel/gtest-parallel', + '../../third_party/gtest-parallel/gtest_parallel.py', + '../../tools_webrtc/gtest-parallel-wrapper.py', + 'foo_unittests', + ]) + self.assertEqual(command, [ + 'vpython3', + '../../testing/test_env.py', + '../../tools_webrtc/gtest-parallel-wrapper.py', + '--output_dir=${ISOLATED_OUTDIR}/test_logs', + '--gtest_color=no', + '--timeout=500', + '--workers=1', + '--retry_failed=3', + './foo_unittests', + '--asan=0', + '--lsan=0', + '--msan=0', + '--tsan=0', + ]) + + def test_gen_script(self): + test_files = { + '/tmp/swarming_targets': + 'foo_unittests_script\n', + '/fake_src/testing/buildbot/gn_isolate_map.pyl': + ("{'foo_unittests_script': {" + " 'label': '//foo:foo_unittests'," + " 'type': 'script'," + " 'script': '//foo/foo_unittests_script.py'," + "}}\n"), + '/fake_src/out/Default/foo_unittests_script.runtime_deps': + ("foo_unittests\n" + "foo_unittests_script.py\n"), + } + mbw = self.check([ + 'gen', '-c', 'debug_goma', '//out/Default', '--swarming-targets-file', + '/tmp/swarming_targets', '--isolate-map-file', + '/fake_src/testing/buildbot/gn_isolate_map.pyl' + ], + files=test_files, + ret=0) + + isolate_file = ( + mbw.files['/fake_src/out/Default/foo_unittests_script.isolate']) + isolate_file_contents = ast.literal_eval(isolate_file) + files = isolate_file_contents['variables']['files'] + command = isolate_file_contents['variables']['command'] + + self.assertEqual(files, [ + '../../.vpython3', + '../../testing/test_env.py', + 'foo_unittests', + 'foo_unittests_script.py', + ]) + self.assertEqual(command, [ + 'vpython3', + '../../foo/foo_unittests_script.py', + ]) + + def test_gen_raw(self): + test_files = { + '/tmp/swarming_targets': + 'foo_unittests\n', + '/fake_src/testing/buildbot/gn_isolate_map.pyl': + ("{'foo_unittests': {" + " 'label': '//foo:foo_unittests'," + " 'type': 'raw'," + "}}\n"), + '/fake_src/out/Default/foo_unittests.runtime_deps': ("foo_unittests\n"), + } + mbw = self.check([ + 'gen', '-c', 'debug_goma', '//out/Default', '--swarming-targets-file', + '/tmp/swarming_targets', '--isolate-map-file', + '/fake_src/testing/buildbot/gn_isolate_map.pyl' + ], + files=test_files, + ret=0) + + isolate_file = mbw.files['/fake_src/out/Default/foo_unittests.isolate'] + isolate_file_contents = ast.literal_eval(isolate_file) + files = isolate_file_contents['variables']['files'] + command = isolate_file_contents['variables']['command'] + + self.assertEqual(files, [ + '../../.vpython3', + '../../testing/test_env.py', + '../../tools_webrtc/flags_compatibility.py', + 'foo_unittests', + ]) + self.assertEqual(command, [ + 'vpython3', + '../../tools_webrtc/flags_compatibility.py', + './foo_unittests', + ]) + + def test_gen_non_parallel_console_test_launcher(self): + test_files = { + '/tmp/swarming_targets': + 'foo_unittests\n', + '/fake_src/testing/buildbot/gn_isolate_map.pyl': + ("{'foo_unittests': {" + " 'label': '//foo:foo_unittests'," + " 'type': 'non_parallel_console_test_launcher'," + "}}\n"), + '/fake_src/out/Default/foo_unittests.runtime_deps': ("foo_unittests\n"), + } + mbw = self.check([ + 'gen', '-c', 'debug_goma', '//out/Default', '--swarming-targets-file', + '/tmp/swarming_targets', '--isolate-map-file', + '/fake_src/testing/buildbot/gn_isolate_map.pyl' + ], + files=test_files, + ret=0) + + isolate_file = mbw.files['/fake_src/out/Default/foo_unittests.isolate'] + isolate_file_contents = ast.literal_eval(isolate_file) + files = isolate_file_contents['variables']['files'] + command = isolate_file_contents['variables']['command'] + + self.assertEqual(files, [ + '../../.vpython3', + '../../testing/test_env.py', + '../../third_party/gtest-parallel/gtest-parallel', + '../../third_party/gtest-parallel/gtest_parallel.py', + '../../tools_webrtc/gtest-parallel-wrapper.py', + 'foo_unittests', + ]) + self.assertEqual(command, [ + 'vpython3', + '../../testing/test_env.py', + '../../tools_webrtc/gtest-parallel-wrapper.py', + '--output_dir=${ISOLATED_OUTDIR}/test_logs', + '--gtest_color=no', + '--timeout=900', + '--workers=1', + '--retry_failed=3', + './foo_unittests', + '--asan=0', + '--lsan=0', + '--msan=0', + '--tsan=0', + ]) + + def test_isolate_windowed_test_launcher_linux(self): + test_files = { + '/tmp/swarming_targets': + 'foo_unittests\n', + '/fake_src/testing/buildbot/gn_isolate_map.pyl': + ("{'foo_unittests': {" + " 'label': '//foo:foo_unittests'," + " 'type': 'windowed_test_launcher'," + "}}\n"), + '/fake_src/out/Default/foo_unittests.runtime_deps': + ("foo_unittests\n" + "some_resource_file\n"), + } + mbw = self.check([ + 'gen', '-c', 'debug_goma', '//out/Default', '--swarming-targets-file', + '/tmp/swarming_targets', '--isolate-map-file', + '/fake_src/testing/buildbot/gn_isolate_map.pyl' + ], + files=test_files, + ret=0) + + isolate_file = mbw.files['/fake_src/out/Default/foo_unittests.isolate'] + isolate_file_contents = ast.literal_eval(isolate_file) + files = isolate_file_contents['variables']['files'] + command = isolate_file_contents['variables']['command'] + + self.assertEqual(files, [ + '../../.vpython3', + '../../testing/test_env.py', + '../../testing/xvfb.py', + '../../third_party/gtest-parallel/gtest-parallel', + '../../third_party/gtest-parallel/gtest_parallel.py', + '../../tools_webrtc/gtest-parallel-wrapper.py', + 'foo_unittests', + 'some_resource_file', + ]) + self.assertEqual(command, [ + 'vpython3', + '../../testing/xvfb.py', + '../../tools_webrtc/gtest-parallel-wrapper.py', + '--output_dir=${ISOLATED_OUTDIR}/test_logs', + '--gtest_color=no', + '--timeout=900', + '--retry_failed=3', + './foo_unittests', + '--asan=0', + '--lsan=0', + '--msan=0', + '--tsan=0', + ]) + + def test_gen_windowed_test_launcher_win(self): + files = { + 'c:\\fake_src\\out\\Default\\tmp\\swarming_targets': + 'unittests\n', + 'c:\\fake_src\\testing\\buildbot\\gn_isolate_map.pyl': + ("{'unittests': {" + " 'label': '//somewhere:unittests'," + " 'type': 'windowed_test_launcher'," + "}}\n"), + r'c:\fake_src\out\Default\unittests.exe.runtime_deps': + ("unittests.exe\n" + "some_dependency\n"), + } + mbw = CreateFakeMBW(files=files, win32=True) + self.check([ + 'gen', '-c', 'debug_goma', '--swarming-targets-file', + 'c:\\fake_src\\out\\Default\\tmp\\swarming_targets', + '--isolate-map-file', + 'c:\\fake_src\\testing\\buildbot\\gn_isolate_map.pyl', '//out/Default' + ], + mbw=mbw, + ret=0) + + isolate_file = mbw.files['c:\\fake_src\\out\\Default\\unittests.isolate'] + isolate_file_contents = ast.literal_eval(isolate_file) + files = isolate_file_contents['variables']['files'] + command = isolate_file_contents['variables']['command'] + + self.assertEqual(files, [ + '../../.vpython3', + '../../testing/test_env.py', + '../../third_party/gtest-parallel/gtest-parallel', + '../../third_party/gtest-parallel/gtest_parallel.py', + '../../tools_webrtc/gtest-parallel-wrapper.py', + 'some_dependency', + 'unittests.exe', + ]) + self.assertEqual(command, [ + 'vpython3', + '../../testing/test_env.py', + '../../tools_webrtc/gtest-parallel-wrapper.py', + '--output_dir=${ISOLATED_OUTDIR}/test_logs', + '--gtest_color=no', + '--timeout=900', + '--retry_failed=3', + r'.\unittests.exe', + '--asan=0', + '--lsan=0', + '--msan=0', + '--tsan=0', + ]) + + def test_gen_console_test_launcher(self): + test_files = { + '/tmp/swarming_targets': + 'foo_unittests\n', + '/fake_src/testing/buildbot/gn_isolate_map.pyl': + ("{'foo_unittests': {" + " 'label': '//foo:foo_unittests'," + " 'type': 'console_test_launcher'," + "}}\n"), + '/fake_src/out/Default/foo_unittests.runtime_deps': ("foo_unittests\n"), + } + mbw = self.check([ + 'gen', '-c', 'debug_goma', '//out/Default', '--swarming-targets-file', + '/tmp/swarming_targets', '--isolate-map-file', + '/fake_src/testing/buildbot/gn_isolate_map.pyl' + ], + files=test_files, + ret=0) + + isolate_file = mbw.files['/fake_src/out/Default/foo_unittests.isolate'] + isolate_file_contents = ast.literal_eval(isolate_file) + files = isolate_file_contents['variables']['files'] + command = isolate_file_contents['variables']['command'] + + self.assertEqual(files, [ + '../../.vpython3', + '../../testing/test_env.py', + '../../third_party/gtest-parallel/gtest-parallel', + '../../third_party/gtest-parallel/gtest_parallel.py', + '../../tools_webrtc/gtest-parallel-wrapper.py', + 'foo_unittests', + ]) + self.assertEqual(command, [ + 'vpython3', + '../../testing/test_env.py', + '../../tools_webrtc/gtest-parallel-wrapper.py', + '--output_dir=${ISOLATED_OUTDIR}/test_logs', + '--gtest_color=no', + '--timeout=900', + '--retry_failed=3', + './foo_unittests', + '--asan=0', + '--lsan=0', + '--msan=0', + '--tsan=0', + ]) + + def test_isolate_test_launcher_with_webcam(self): + test_files = { + '/tmp/swarming_targets': + 'foo_unittests\n', + '/fake_src/testing/buildbot/gn_isolate_map.pyl': + ("{'foo_unittests': {" + " 'label': '//foo:foo_unittests'," + " 'type': 'console_test_launcher'," + " 'use_webcam': True," + "}}\n"), + '/fake_src/out/Default/foo_unittests.runtime_deps': + ("foo_unittests\n" + "some_resource_file\n"), + } + mbw = self.check([ + 'gen', '-c', 'debug_goma', '//out/Default', '--swarming-targets-file', + '/tmp/swarming_targets', '--isolate-map-file', + '/fake_src/testing/buildbot/gn_isolate_map.pyl' + ], + files=test_files, + ret=0) + + isolate_file = mbw.files['/fake_src/out/Default/foo_unittests.isolate'] + isolate_file_contents = ast.literal_eval(isolate_file) + files = isolate_file_contents['variables']['files'] + command = isolate_file_contents['variables']['command'] + + self.assertEqual(files, [ + '../../.vpython3', + '../../testing/test_env.py', + '../../third_party/gtest-parallel/gtest-parallel', + '../../third_party/gtest-parallel/gtest_parallel.py', + '../../tools_webrtc/ensure_webcam_is_running.py', + '../../tools_webrtc/gtest-parallel-wrapper.py', + 'foo_unittests', + 'some_resource_file', + ]) + self.assertEqual(command, [ + 'vpython3', + '../../tools_webrtc/ensure_webcam_is_running.py', + 'vpython3', + '../../testing/test_env.py', + '../../tools_webrtc/gtest-parallel-wrapper.py', + '--output_dir=${ISOLATED_OUTDIR}/test_logs', + '--gtest_color=no', + '--timeout=900', + '--retry_failed=3', + './foo_unittests', + '--asan=0', + '--lsan=0', + '--msan=0', + '--tsan=0', + ]) + + def test_isolate(self): + files = { + '/fake_src/out/Default/toolchain.ninja': + "", + '/fake_src/testing/buildbot/gn_isolate_map.pyl': + ("{'foo_unittests': {" + " 'label': '//foo:foo_unittests'," + " 'type': 'non_parallel_console_test_launcher'," + "}}\n"), + '/fake_src/out/Default/foo_unittests.runtime_deps': ("foo_unittests\n"), + } + self.check( + ['isolate', '-c', 'debug_goma', '//out/Default', 'foo_unittests'], + files=files, + ret=0) + + # test running isolate on an existing build_dir + files['/fake_src/out/Default/args.gn'] = 'is_debug = true\n' + self.check(['isolate', '//out/Default', 'foo_unittests'], + files=files, + ret=0) + files['/fake_src/out/Default/mb_type'] = 'gn\n' + self.check(['isolate', '//out/Default', 'foo_unittests'], + files=files, + ret=0) + +if __name__ == '__main__': + unittest.main() diff --git a/third_party/libwebrtc/tools_webrtc/msan/suppressions.txt b/third_party/libwebrtc/tools_webrtc/msan/suppressions.txt new file mode 100644 index 0000000000..47a0dff16f --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/msan/suppressions.txt @@ -0,0 +1,15 @@ +# The rules in this file are only applied at compile time. +# Because the Chrome buildsystem does not automatically touch the files +# mentioned here, changing this file requires clobbering all MSan bots. +# +# Please think twice before you add or remove these rules. + +# This is a stripped down copy of Chromium's ignorelist.txt, to enable +# adding WebRTC-specific ignorelist entries. + +# Uninit in zlib. http://crbug.com/116277 +fun:*MOZ_Z_deflate* + +# Uninit in H264. http://crbug.com/webrtc/11702 +src:*/third_party/openh264/src/codec/processing/src/vaacalc/vaacalcfuncs.cpp + diff --git a/third_party/libwebrtc/tools_webrtc/network_emulator/config.py b/third_party/libwebrtc/tools_webrtc/network_emulator/config.py new file mode 100644 index 0000000000..9a18bdce45 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/network_emulator/config.py @@ -0,0 +1,36 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""Configuration class for network emulation.""" + + +class ConnectionConfig: + """Configuration containing the characteristics of a network connection.""" + + def __init__(self, num, name, receive_bw_kbps, send_bw_kbps, delay_ms, + packet_loss_percent, queue_slots): + self.num = num + self.name = name + self.receive_bw_kbps = receive_bw_kbps + self.send_bw_kbps = send_bw_kbps + self.delay_ms = delay_ms + self.packet_loss_percent = packet_loss_percent + self.queue_slots = queue_slots + + def __str__(self): + """String representing the configuration. + + Returns: + A string formatted and padded like this example: + 12 Name 375 kbps 375 kbps 10 145 ms 0.1 % + """ + left_aligned_name = self.name.ljust(24, ' ') + return '%2s %24s %5s kbps %5s kbps %4s %5s ms %3s %%' % ( + self.num, left_aligned_name, self.receive_bw_kbps, self.send_bw_kbps, + self.queue_slots, self.delay_ms, self.packet_loss_percent) diff --git a/third_party/libwebrtc/tools_webrtc/network_emulator/emulate.py b/third_party/libwebrtc/tools_webrtc/network_emulator/emulate.py new file mode 100755 index 0000000000..a35ccd36c8 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/network_emulator/emulate.py @@ -0,0 +1,209 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""Script for constraining traffic on the local machine.""" + +import logging +import optparse +import socket +import sys + +import config +import network_emulator + +_DEFAULT_LOG_LEVEL = logging.INFO + +# Default port range to apply network constraints on. +_DEFAULT_PORT_RANGE = (32768, 65535) + +# The numbers below are gathered from Google stats from the presets of the Apple +# developer tool called Network Link Conditioner. +_PRESETS = [ + config.ConnectionConfig(1, 'Generic, Bad', 95, 95, 250, 2, 100), + config.ConnectionConfig(2, 'Generic, Average', 375, 375, 145, 0.1, 100), + config.ConnectionConfig(3, 'Generic, Good', 1000, 1000, 35, 0, 100), + config.ConnectionConfig(4, '3G, Average Case', 780, 330, 100, 0, 100), + config.ConnectionConfig(5, '3G, Good', 850, 420, 90, 0, 100), + config.ConnectionConfig(6, '3G, Lossy Network', 780, 330, 100, 1, 100), + config.ConnectionConfig(7, 'Cable Modem', 6000, 1000, 2, 0, 10), + config.ConnectionConfig(8, 'DSL', 2000, 256, 5, 0, 10), + config.ConnectionConfig(9, 'Edge, Average Case', 240, 200, 400, 0, 100), + config.ConnectionConfig(10, 'Edge, Good', 250, 200, 350, 0, 100), + config.ConnectionConfig(11, 'Edge, Lossy Network', 240, 200, 400, 1, 100), + config.ConnectionConfig(12, 'Wifi, Average Case', 40000, 33000, 1, 0, 100), + config.ConnectionConfig(13, 'Wifi, Good', 45000, 40000, 1, 0, 100), + config.ConnectionConfig(14, 'Wifi, Lossy', 40000, 33000, 1, 0, 100), +] +_PRESETS_DICT = dict((p.num, p) for p in _PRESETS) + +_DEFAULT_PRESET_ID = 2 +_DEFAULT_PRESET = _PRESETS_DICT[_DEFAULT_PRESET_ID] + + +class NonStrippingEpilogOptionParser(optparse.OptionParser): + """Custom parser to let us show the epilog without weird line breaking.""" + + def format_epilog(self, formatter): + return self.epilog + + +def _GetExternalIp(): + """Finds out the machine's external IP by connecting to google.com.""" + external_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + external_socket.connect(('google.com', 80)) + return external_socket.getsockname()[0] + + +def _ParseArgs(): + """Define and parse the command-line arguments.""" + presets_string = '\n'.join(str(p) for p in _PRESETS) + parser = NonStrippingEpilogOptionParser(epilog=( + '\nAvailable presets:\n' + ' Bandwidth (kbps) Packet\n' + 'ID Name Receive Send Queue Delay loss \n' + '-- ---- --------- -------- ----- ------- ------\n' + '%s\n' % presets_string)) + parser.add_option('-p', + '--preset', + type='int', + default=_DEFAULT_PRESET_ID, + help=('ConnectionConfig configuration, specified by ID. ' + 'Default: %default')) + parser.add_option('-r', + '--receive-bw', + type='int', + default=_DEFAULT_PRESET.receive_bw_kbps, + help=('Receive bandwidth in kilobit/s. Default: %default')) + parser.add_option('-s', + '--send-bw', + type='int', + default=_DEFAULT_PRESET.send_bw_kbps, + help=('Send bandwidth in kilobit/s. Default: %default')) + parser.add_option('-d', + '--delay', + type='int', + default=_DEFAULT_PRESET.delay_ms, + help=('Delay in ms. Default: %default')) + parser.add_option('-l', + '--packet-loss', + type='float', + default=_DEFAULT_PRESET.packet_loss_percent, + help=('Packet loss in %. Default: %default')) + parser.add_option('-q', + '--queue', + type='int', + default=_DEFAULT_PRESET.queue_slots, + help=('Queue size as number of slots. Default: %default')) + parser.add_option('--port-range', + default='%s,%s' % _DEFAULT_PORT_RANGE, + help=('Range of ports for constrained network. Specify as ' + 'two comma separated integers. Default: %default')) + parser.add_option('--target-ip', + default=None, + help=('The interface IP address to apply the rules for. ' + 'Default: the external facing interface IP address.')) + parser.add_option('-v', + '--verbose', + action='store_true', + default=False, + help=('Turn on verbose output. Will print all \'ipfw\' ' + 'commands that are executed.')) + + options = parser.parse_args()[0] + + # Find preset by ID, if specified. + if options.preset and options.preset not in _PRESETS_DICT: + parser.error('Invalid preset: %s' % options.preset) + + # Simple validation of the IP address, if supplied. + if options.target_ip: + try: + socket.inet_aton(options.target_ip) + except socket.error: + parser.error('Invalid IP address specified: %s' % options.target_ip) + + # Convert port range into the desired tuple format. + try: + if isinstance(options.port_range, str): + options.port_range = tuple( + int(port) for port in options.port_range.split(',')) + if len(options.port_range) != 2: + parser.error('Invalid port range specified, please specify two ' + 'integers separated by a comma.') + except ValueError: + parser.error('Invalid port range specified.') + + _InitLogging(options.verbose) + return options + + +def _InitLogging(verbose): + """Setup logging.""" + log_level = _DEFAULT_LOG_LEVEL + if verbose: + log_level = logging.DEBUG + logging.basicConfig(level=log_level, format='%(message)s') + + +def main(): + options = _ParseArgs() + + # Build a configuration object. Override any preset configuration settings if + # a value of a setting was also given as a flag. + connection_config = _PRESETS_DICT[options.preset] + if options.receive_bw is not _DEFAULT_PRESET.receive_bw_kbps: + connection_config.receive_bw_kbps = options.receive_bw + if options.send_bw is not _DEFAULT_PRESET.send_bw_kbps: + connection_config.send_bw_kbps = options.send_bw + if options.delay is not _DEFAULT_PRESET.delay_ms: + connection_config.delay_ms = options.delay + if options.packet_loss is not _DEFAULT_PRESET.packet_loss_percent: + connection_config.packet_loss_percent = options.packet_loss + if options.queue is not _DEFAULT_PRESET.queue_slots: + connection_config.queue_slots = options.queue + emulator = network_emulator.NetworkEmulator(connection_config, + options.port_range) + try: + emulator.CheckPermissions() + except network_emulator.NetworkEmulatorError as e: + logging.error('Error: %s\n\nCause: %s', e.fail_msg, e.error) + return -1 + + if not options.target_ip: + external_ip = _GetExternalIp() + else: + external_ip = options.target_ip + + logging.info('Constraining traffic to/from IP: %s', external_ip) + try: + emulator.Emulate(external_ip) + logging.info( + 'Started network emulation with the following configuration:\n' + ' Receive bandwidth: %s kbps (%s kB/s)\n' + ' Send bandwidth : %s kbps (%s kB/s)\n' + ' Delay : %s ms\n' + ' Packet loss : %s %%\n' + ' Queue slots : %s', connection_config.receive_bw_kbps, + connection_config.receive_bw_kbps / 8, connection_config.send_bw_kbps, + connection_config.send_bw_kbps / 8, connection_config.delay_ms, + connection_config.packet_loss_percent, connection_config.queue_slots) + logging.info('Affected traffic: IP traffic on ports %s-%s', + options.port_range[0], options.port_range[1]) + input('Press Enter to abort Network Emulation...') + logging.info('Flushing all Dummynet rules...') + network_emulator.Cleanup() + logging.info('Completed Network Emulation.') + return 0 + except network_emulator.NetworkEmulatorError as e: + logging.error('Error: %s\n\nCause: %s', e.fail_msg, e.error) + return -2 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/third_party/libwebrtc/tools_webrtc/network_emulator/network_emulator.py b/third_party/libwebrtc/tools_webrtc/network_emulator/network_emulator.py new file mode 100644 index 0000000000..a7776a5f92 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/network_emulator/network_emulator.py @@ -0,0 +1,195 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""Script for constraining traffic on the local machine.""" + +import ctypes +import logging +import os +import subprocess +import sys + + +class NetworkEmulatorError(BaseException): + """Exception raised for errors in the network emulator. + + Attributes: + fail_msg: User defined error message. + cmd: Command for which the exception was raised. + returncode: Return code of running the command. + stdout: Output of running the command. + stderr: Error output of running the command. + """ + + def __init__(self, + fail_msg, + cmd=None, + returncode=None, + output=None, + error=None): + BaseException.__init__(self, fail_msg) + self.fail_msg = fail_msg + self.cmd = cmd + self.returncode = returncode + self.output = output + self.error = error + + +class NetworkEmulator: + """A network emulator that can constrain the network using Dummynet.""" + + def __init__(self, connection_config, port_range): + """Constructor. + + Args: + connection_config: A config.ConnectionConfig object containing the + characteristics for the connection to be emulation. + port_range: Tuple containing two integers defining the port range. + """ + self._pipe_counter = 0 + self._rule_counter = 0 + self._port_range = port_range + self._connection_config = connection_config + + def Emulate(self, target_ip): + """Starts a network emulation by setting up Dummynet rules. + + Args: + target_ip: The IP address of the interface that shall be that have the + network constraints applied to it. + """ + receive_pipe_id = self._CreateDummynetPipe( + self._connection_config.receive_bw_kbps, + self._connection_config.delay_ms, + self._connection_config.packet_loss_percent, + self._connection_config.queue_slots) + logging.debug('Created receive pipe: %s', receive_pipe_id) + send_pipe_id = self._CreateDummynetPipe( + self._connection_config.send_bw_kbps, self._connection_config.delay_ms, + self._connection_config.packet_loss_percent, + self._connection_config.queue_slots) + logging.debug('Created send pipe: %s', send_pipe_id) + + # Adding the rules will start the emulation. + incoming_rule_id = self._CreateDummynetRule(receive_pipe_id, 'any', + target_ip, self._port_range) + logging.debug('Created incoming rule: %s', incoming_rule_id) + outgoing_rule_id = self._CreateDummynetRule(send_pipe_id, target_ip, 'any', + self._port_range) + logging.debug('Created outgoing rule: %s', outgoing_rule_id) + + @staticmethod + def CheckPermissions(): + """Checks if permissions are available to run Dummynet commands. + + Raises: + NetworkEmulatorError: If permissions to run Dummynet commands are not + available. + """ + try: + if os.getuid() != 0: + raise NetworkEmulatorError('You must run this script with sudo.') + except AttributeError as permission_error: + + # AttributeError will be raised on Windows. + if ctypes.windll.shell32.IsUserAnAdmin() == 0: + raise NetworkEmulatorError('You must run this script with administrator' + ' privileges.') from permission_error + + def _CreateDummynetRule(self, pipe_id, from_address, to_address, port_range): + """Creates a network emulation rule and returns its ID. + + Args: + pipe_id: integer ID of the pipe. + from_address: The IP address to match source address. May be an IP or + 'any'. + to_address: The IP address to match destination address. May be an IP or + 'any'. + port_range: The range of ports the rule shall be applied on. Must be + specified as a tuple of with two integers. + Returns: + The ID of the rule, starting at 100. The rule ID increments with 100 for + each rule being added. + """ + self._rule_counter += 100 + add_part = [ + 'add', self._rule_counter, 'pipe', pipe_id, 'ip', 'from', from_address, + 'to', to_address + ] + _RunIpfwCommand(add_part + ['src-port', '%s-%s' % port_range], + 'Failed to add Dummynet src-port rule.') + _RunIpfwCommand(add_part + ['dst-port', '%s-%s' % port_range], + 'Failed to add Dummynet dst-port rule.') + return self._rule_counter + + def _CreateDummynetPipe(self, bandwidth_kbps, delay_ms, packet_loss_percent, + queue_slots): + """Creates a Dummynet pipe and return its ID. + + Args: + bandwidth_kbps: Bandwidth. + delay_ms: Delay for a one-way trip of a packet. + packet_loss_percent: Float value of packet loss, in percent. + queue_slots: Size of the queue. + Returns: + The ID of the pipe, starting at 1. + """ + self._pipe_counter += 1 + cmd = [ + 'pipe', self._pipe_counter, 'config', 'bw', + str(bandwidth_kbps / 8) + 'KByte/s', 'delay', + '%sms' % delay_ms, 'plr', (packet_loss_percent / 100.0), 'queue', + queue_slots + ] + error_message = 'Failed to create Dummynet pipe. ' + if sys.platform.startswith('linux'): + error_message += ('Make sure you have loaded the ipfw_mod.ko module to ' + 'your kernel (sudo insmod /path/to/ipfw_mod.ko).') + _RunIpfwCommand(cmd, error_message) + return self._pipe_counter + + +def Cleanup(): + """Stops the network emulation by flushing all Dummynet rules. + + Notice that this will flush any rules that may have been created previously + before starting the emulation. + """ + _RunIpfwCommand(['-f', 'flush'], 'Failed to flush Dummynet rules!') + _RunIpfwCommand(['-f', 'pipe', 'flush'], 'Failed to flush Dummynet pipes!') + + +def _RunIpfwCommand(command, fail_msg=None): + """Executes a command and prefixes the appropriate command for + Windows or Linux/UNIX. + + Args: + command: Command list to execute. + fail_msg: Message describing the error in case the command fails. + + Raises: + NetworkEmulatorError: If command fails a message is set by the fail_msg + parameter. + """ + if sys.platform == 'win32': + ipfw_command = ['ipfw.exe'] + else: + ipfw_command = ['sudo', '-n', 'ipfw'] + + cmd_list = ipfw_command[:] + [str(x) for x in command] + cmd_string = ' '.join(cmd_list) + logging.debug('Running command: %s', cmd_string) + process = subprocess.Popen(cmd_list, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output, error = process.communicate() + if process.returncode != 0: + raise NetworkEmulatorError(fail_msg, cmd_string, process.returncode, output, + error) + return output.strip() diff --git a/third_party/libwebrtc/tools_webrtc/perf/BUILD.gn b/third_party/libwebrtc/tools_webrtc/perf/BUILD.gn new file mode 100644 index 0000000000..484f9565b5 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/perf/BUILD.gn @@ -0,0 +1,17 @@ +# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../webrtc.gni") + +if (rtc_enable_protobuf) { + group("webrtc_dashboard_upload") { + data = [ "process_perf_results.py" ] + data_deps = + [ "//third_party/catapult/tracing/tracing/proto:histogram_proto" ] + } +} diff --git a/third_party/libwebrtc/tools_webrtc/perf/catapult_uploader.py b/third_party/libwebrtc/tools_webrtc/perf/catapult_uploader.py new file mode 100644 index 0000000000..d07c287f28 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/perf/catapult_uploader.py @@ -0,0 +1,310 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import datetime +import json +import subprocess +import time +import zlib + +from typing import Optional +import dataclasses +import httplib2 + +from tracing.value import histogram +from tracing.value import histogram_set +from tracing.value.diagnostics import generic_set +from tracing.value.diagnostics import reserved_infos + + +@dataclasses.dataclass +class UploaderOptions(): + """Required information to upload perf metrics. + + Attributes: + perf_dashboard_machine_group: The "master" the bots are grouped under. + This string is the group in the the perf dashboard path + group/bot/perf_id/metric/subtest. + bot: The bot running the test (e.g. webrtc-win-large-tests). + test_suite: The key for the test in the dashboard (i.e. what you select + in the top-level test suite selector in the dashboard + webrtc_git_hash: webrtc.googlesource.com commit hash. + commit_position: Commit pos corresponding to the git hash. + build_page_url: URL to the build page for this build. + dashboard_url: Which dashboard to use. + input_results_file: A HistogramSet proto file coming from WebRTC tests. + output_json_file: Where to write the output (for debugging). + wait_timeout_sec: Maximum amount of time in seconds that the script will + wait for the confirmation. + wait_polling_period_sec: Status will be requested from the Dashboard + every wait_polling_period_sec seconds. + """ + perf_dashboard_machine_group: str + bot: str + test_suite: str + webrtc_git_hash: str + commit_position: int + build_page_url: str + dashboard_url: str + input_results_file: str + output_json_file: Optional[str] = None + wait_timeout_sec: datetime.timedelta = datetime.timedelta(seconds=1200) + wait_polling_period_sec: datetime.timedelta = datetime.timedelta(seconds=120) + + +def _GenerateOauthToken(): + args = ['luci-auth', 'token'] + p = subprocess.Popen(args, + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + if p.wait() == 0: + output = p.stdout.read() + return output.strip() + raise RuntimeError( + 'Error generating authentication token.\nStdout: %s\nStderr:%s' % + (p.stdout.read(), p.stderr.read())) + + +def _CreateHeaders(oauth_token): + return {'Authorization': 'Bearer %s' % oauth_token} + + +def _SendHistogramSet(url, histograms): + """Make a HTTP POST with the given JSON to the Performance Dashboard. + + Args: + url: URL of Performance Dashboard instance, e.g. + "https://chromeperf.appspot.com". + histograms: a histogram set object that contains the data to be sent. + """ + headers = _CreateHeaders(_GenerateOauthToken()) + + serialized = json.dumps(_ApplyHacks(histograms.AsDicts()), indent=4) + + if url.startswith('http://localhost'): + # The catapult server turns off compression in developer mode. + data = serialized + else: + data = zlib.compress(serialized.encode('utf-8')) + + print('Sending %d bytes to %s.' % (len(data), url + '/add_histograms')) + + http = httplib2.Http() + response, content = http.request(url + '/add_histograms', + method='POST', + body=data, + headers=headers) + return response, content + + +def _WaitForUploadConfirmation(url, upload_token, wait_timeout, + wait_polling_period): + """Make a HTTP GET requests to the Performance Dashboard untill upload + status is known or the time is out. + + Args: + url: URL of Performance Dashboard instance, e.g. + "https://chromeperf.appspot.com". + upload_token: String that identifies Performance Dashboard and can be used + for the status check. + wait_timeout: (datetime.timedelta) Maximum time to wait for the + confirmation. + wait_polling_period: (datetime.timedelta) Performance Dashboard will be + polled every wait_polling_period amount of time. + """ + assert wait_polling_period <= wait_timeout + + headers = _CreateHeaders(_GenerateOauthToken()) + http = httplib2.Http() + + oauth_refreshed = False + response = None + resp_json = None + current_time = datetime.datetime.now() + end_time = current_time + wait_timeout + next_poll_time = current_time + wait_polling_period + while datetime.datetime.now() < end_time: + current_time = datetime.datetime.now() + if next_poll_time > current_time: + time.sleep((next_poll_time - current_time).total_seconds()) + next_poll_time = datetime.datetime.now() + wait_polling_period + + response, content = http.request(url + '/uploads/' + upload_token, + method='GET', + headers=headers) + + print('Upload state polled. Response: %r.' % content) + + if not oauth_refreshed and response.status == 403: + print('Oauth token refreshed. Continue polling.') + headers = _CreateHeaders(_GenerateOauthToken()) + oauth_refreshed = True + continue + + if response.status != 200: + break + + resp_json = json.loads(content) + if resp_json['state'] == 'COMPLETED' or resp_json['state'] == 'FAILED': + break + + return response, resp_json + + +# Because of an issues on the Dashboard side few measurements over a large set +# can fail to upload. That would lead to the whole upload to be marked as +# failed. Check it, so it doesn't increase flakiness of our tests. +# TODO(crbug.com/1145904): Remove check after fixed. +def _CheckFullUploadInfo(url, upload_token, + min_measurements_amount=50, + max_failed_measurements_percent=0.03): + """Make a HTTP GET requests to the Performance Dashboard to get full info + about upload (including measurements). Checks if upload is correct despite + not having status "COMPLETED". + + Args: + url: URL of Performance Dashboard instance, e.g. + "https://chromeperf.appspot.com". + upload_token: String that identifies Performance Dashboard and can be used + for the status check. + min_measurements_amount: minimal amount of measurements that the upload + should have to start tolerating failures in particular measurements. + max_failed_measurements_percent: maximal percent of failured measurements + to tolerate. + """ + headers = _CreateHeaders(_GenerateOauthToken()) + http = httplib2.Http() + + response, content = http.request(url + '/uploads/' + upload_token + + '?additional_info=measurements', + method='GET', + headers=headers) + + if response.status != 200: + print('Failed to reach the dashboard to get full upload info.') + return False + + resp_json = json.loads(content) + print('Full upload info: %s.' % json.dumps(resp_json, indent=4)) + + if 'measurements' in resp_json: + measurements_cnt = len(resp_json['measurements']) + not_completed_state_cnt = len( + [m for m in resp_json['measurements'] if m['state'] != 'COMPLETED']) + + if (measurements_cnt >= min_measurements_amount + and (not_completed_state_cnt / + (measurements_cnt * 1.0) <= max_failed_measurements_percent)): + print(('Not all measurements were confirmed to upload. ' + 'Measurements count: %d, failed to upload or timed out: %d' % + (measurements_cnt, not_completed_state_cnt))) + return True + + return False + + +# TODO(https://crbug.com/1029452): HACKHACK +# Remove once we have doubles in the proto and handle -infinity correctly. +def _ApplyHacks(dicts): + def _NoInf(value): + if value == float('inf'): + return histogram.JS_MAX_VALUE + if value == float('-inf'): + return -histogram.JS_MAX_VALUE + return value + + for d in dicts: + if 'running' in d: + d['running'] = [_NoInf(value) for value in d['running']] + if 'sampleValues' in d: + d['sampleValues'] = [_NoInf(value) for value in d['sampleValues']] + + return dicts + + +def _LoadHistogramSetFromProto(options): + hs = histogram_set.HistogramSet() + with open(options.input_results_file, 'rb') as f: + hs.ImportProto(f.read()) + + return hs + + +def _AddBuildInfo(histograms, options): + common_diagnostics = { + reserved_infos.MASTERS: options.perf_dashboard_machine_group, + reserved_infos.BOTS: options.bot, + reserved_infos.POINT_ID: options.commit_position, + reserved_infos.BENCHMARKS: options.test_suite, + reserved_infos.WEBRTC_REVISIONS: str(options.webrtc_git_hash), + reserved_infos.BUILD_URLS: options.build_page_url, + } + + for k, v in list(common_diagnostics.items()): + histograms.AddSharedDiagnosticToAllHistograms(k.name, + generic_set.GenericSet([v])) + + +def _DumpOutput(histograms, output_file): + with open(output_file, 'w') as f: + json.dump(_ApplyHacks(histograms.AsDicts()), f, indent=4) + + +def UploadToDashboardImpl(options): + histograms = _LoadHistogramSetFromProto(options) + _AddBuildInfo(histograms, options) + + if options.output_json_file: + _DumpOutput(histograms, options.output_json_file) + + response, content = _SendHistogramSet(options.dashboard_url, histograms) + + if response.status != 200: + print(('Upload failed with %d: %s\n\n%s' % + (response.status, response.reason, content))) + return 1 + + upload_token = json.loads(content).get('token') + if not upload_token: + print(('Received 200 from dashboard. ', + 'Not waiting for the upload status confirmation.')) + return 0 + + response, resp_json = _WaitForUploadConfirmation( + options.dashboard_url, upload_token, options.wait_timeout_sec, + options.wait_polling_period_sec) + + if ((resp_json and resp_json['state'] == 'COMPLETED') + or _CheckFullUploadInfo(options.dashboard_url, upload_token)): + print('Upload completed.') + return 0 + + if response.status != 200: + print(('Upload status poll failed with %d: %s' % + (response.status, response.reason))) + return 1 + + if resp_json['state'] == 'FAILED': + print('Upload failed.') + return 1 + + print(('Upload wasn\'t completed in a given time: %s seconds.' % + options.wait_timeout_sec)) + return 1 + + +def UploadToDashboard(options): + try: + exit_code = UploadToDashboardImpl(options) + except RuntimeError as e: + print(e) + return 1 + return exit_code diff --git a/third_party/libwebrtc/tools_webrtc/perf/catapult_uploader_test.py b/third_party/libwebrtc/tools_webrtc/perf/catapult_uploader_test.py new file mode 100644 index 0000000000..ba42554412 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/perf/catapult_uploader_test.py @@ -0,0 +1,122 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import os +import sys +import unittest + +from unittest.mock import MagicMock + + +# This tests requires the webrtc_dashboard_upload target to be built before +# running the tests. +def _ConfigurePythonPath(): + # We just yank the python scripts we require into the PYTHONPATH. You could + # also imagine a solution where we use for instance + # protobuf:py_proto_runtime to copy catapult and protobuf code to out/. + # This is the convention in Chromium and WebRTC python scripts. We do need + # to build histogram_pb2 however, so that's why we add out/ to sys.path + # below. + # + # It would be better if there was an equivalent to py_binary in GN, but + # there's not. + script_dir = os.path.dirname(os.path.realpath(__file__)) + checkout_root = os.path.abspath(os.path.join(script_dir, os.pardir, + os.pardir)) + + sys.path.insert( + 0, os.path.join(checkout_root, 'third_party', 'catapult', 'tracing')) + sys.path.insert( + 0, os.path.join(checkout_root, 'third_party', 'protobuf', 'python')) + + # The webrtc_dashboard_upload gn rule will build the protobuf stub for + # python, so put it in the path for this script before we attempt to import + # it. + histogram_proto_path = os.path.join(os.path.join('../../out/Default'), + 'pyproto', 'tracing', 'tracing', 'proto') + sys.path.insert(0, histogram_proto_path) + + # Fail early in case the proto hasn't been built. + from tracing.proto import histogram_proto + if not histogram_proto.HAS_PROTO: + raise ImportError('Could not find histogram_pb2. You need to build the ' + 'webrtc_dashboard_upload target before invoking this ' + 'script. Expected to find ' + 'histogram_pb2.py in %s.' % histogram_proto_path) + + +def _CreateHistogram(name='hist', + master=None, + bot=None, + benchmark=None, + benchmark_description=None, + commit_position=None, + samples=None): + hists = [catapult_uploader.histogram.Histogram(name, 'count')] + if samples: + for s in samples: + hists[0].AddSample(s) + histograms = catapult_uploader.histogram_set.HistogramSet(hists) + if master: + histograms.AddSharedDiagnosticToAllHistograms( + catapult_uploader.reserved_infos.MASTERS.name, + catapult_uploader.generic_set.GenericSet([master])) + if bot: + histograms.AddSharedDiagnosticToAllHistograms( + catapult_uploader.reserved_infos.BOTS.name, + catapult_uploader.generic_set.GenericSet([bot])) + if commit_position: + histograms.AddSharedDiagnosticToAllHistograms( + catapult_uploader.reserved_infos.CHROMIUM_COMMIT_POSITIONS.name, + catapult_uploader.generic_set.GenericSet([commit_position])) + if benchmark: + histograms.AddSharedDiagnosticToAllHistograms( + catapult_uploader.reserved_infos.BENCHMARKS.name, + catapult_uploader.generic_set.GenericSet([benchmark])) + if benchmark_description: + histograms.AddSharedDiagnosticToAllHistograms( + catapult_uploader.reserved_infos.BENCHMARK_DESCRIPTIONS.name, + catapult_uploader.generic_set.GenericSet([benchmark_description])) + return histograms + + +class CatapultUploaderTest(unittest.TestCase): + def setUp(self): + mock = MagicMock(return_value=[200, None]) + catapult_uploader.httplib2.Http.request = mock + + self.histogram = _CreateHistogram( + master='master', + bot='bot', + benchmark='benchmark', + commit_position=123, + benchmark_description='Benchmark description.', + samples=[1, 2, 3]) + + def testSendHistogramsSet(self): + url = 'http://notlocalhost' + # pylint: disable=protected-access + response, content = catapult_uploader._SendHistogramSet(url, self.histogram) + self.assertEqual(response, 200) + self.assertEqual(content, None) + + def testSendHistogramsSetLocalhost(self): + url = 'http://localhost' + # pylint: disable=protected-access + response, content = catapult_uploader._SendHistogramSet(url, self.histogram) + self.assertEqual(response, 200) + self.assertEqual(content, None) + + +if (__name__) == '__main__': + _ConfigurePythonPath() + import catapult_uploader + + unittest.main() diff --git a/third_party/libwebrtc/tools_webrtc/perf/process_perf_results.py b/third_party/libwebrtc/tools_webrtc/perf/process_perf_results.py new file mode 100644 index 0000000000..e91b1f66e9 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/perf/process_perf_results.py @@ -0,0 +1,123 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""Adds build info to perf results and uploads them. + +The tests don't know which bot executed the tests or at what revision, so we +need to take their output and enrich it with this information. We load the proto +from the tests, add the build information as shared diagnostics and then +upload it to the dashboard. + +This script can't be in recipes, because we can't access the catapult APIs from +there. It needs to be here source-side. +""" + +import argparse +import json +import os +import sys + +from pathlib import Path + +# Even if protobuf is not used directly, this allows transitive imports +# of the protobuf library to use the vpython wheel specified in the root +# level .vpython (see bugs.webrtc.org/12211 for context). +import google.protobuf # pylint: disable=unused-import + + +def _ConfigurePythonPath(outdir): + # We just yank the python scripts we require into the PYTHONPATH. You could + # also imagine a solution where we use for instance + # protobuf:py_proto_runtime to copy catapult and protobuf code to out/. + # This is the convention in Chromium and WebRTC python scripts. We do need + # to build histogram_pb2 however, so that's why we add out/ to sys.path + # below. + # + # It would be better if there was an equivalent to py_binary in GN, but + # there's not. + script_dir = os.path.dirname(os.path.realpath(__file__)) + checkout_root = os.path.abspath(os.path.join(script_dir, os.pardir, + os.pardir)) + + sys.path.insert( + 0, os.path.join(checkout_root, 'third_party', 'catapult', 'tracing')) + sys.path.insert( + 0, os.path.join(checkout_root, 'third_party', 'protobuf', 'python')) + + # The webrtc_dashboard_upload gn rule will build the protobuf stub for + # python, so put it in the path for this script before we attempt to import + # it. + histogram_proto_path = os.path.join(outdir, 'pyproto', 'tracing', 'tracing', + 'proto') + sys.path.insert(0, histogram_proto_path) + + # Fail early in case the proto hasn't been built. + from tracing.proto import histogram_proto + if not histogram_proto.HAS_PROTO: + print('Could not find histogram_pb2. You need to build the ' + 'webrtc_dashboard_upload target before invoking this ' + 'script. Expected to find ' + 'histogram_pb2.py in %s.' % histogram_proto_path) + return 1 + return 0 + + +def _UploadToDasboard(args): + build_properties = json.loads(args.build_properties) + exit_code = _ConfigurePythonPath(build_properties['outdir']) + if exit_code != 0: + return exit_code + + import catapult_uploader + + perftest_outputs = [ + f.absolute() for f in Path(args.task_output_dir).rglob('perftest-output*') + if f.is_file() + ] + for perftest_output in perftest_outputs: + uploader_options = catapult_uploader.UploaderOptions( + perf_dashboard_machine_group=( + build_properties['perf_dashboard_machine_group']), + bot=build_properties['bot'], + webrtc_git_hash=build_properties['webrtc_git_hash'], + commit_position=build_properties['commit_position'], + build_page_url=build_properties['build_page_url'], + dashboard_url=build_properties['dashboard_url'], + test_suite=args.test_suite, + input_results_file=perftest_output, + ) + exit_code = catapult_uploader.UploadToDashboard(uploader_options) + if exit_code != 0: + return exit_code + return 0 + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--build-properties', help=argparse.SUPPRESS) + parser.add_argument('--summary-json', help=argparse.SUPPRESS) + parser.add_argument('--task-output-dir', help=argparse.SUPPRESS) + parser.add_argument('--test-suite', help=argparse.SUPPRESS) + parser.add_argument('-o', '--output-json', help=argparse.SUPPRESS) + parser.add_argument('json_files', nargs='*', help=argparse.SUPPRESS) + args = parser.parse_args() + + exit_code = _UploadToDasboard(args) + if exit_code != 0: + with open(args.output_json, 'w') as f: + json.dump({ + "global_tags": ["UNRELIABLE_RESULTS"], + "missing_shards": [0] + }, f) + return exit_code + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/third_party/libwebrtc/tools_webrtc/perf/process_perf_results_py2.py b/third_party/libwebrtc/tools_webrtc/perf/process_perf_results_py2.py new file mode 100644 index 0000000000..14b6858093 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/perf/process_perf_results_py2.py @@ -0,0 +1,25 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""Calls process_perf_results.py with a python 3 interpreter.""" + +import sys +import subprocess + + +# TODO(crbug.com/webrtc/13835): Delete this file and use +# process_perf_results.py instead. +def main(): + cmd = sys.argv[0].replace('_py2', '') + print('Calling "%s" with py3 in case this script was called with py2.' % cmd) + return subprocess.call(['vpython3', cmd] + sys.argv[1:]) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/third_party/libwebrtc/tools_webrtc/perf/process_perf_results_test.py b/third_party/libwebrtc/tools_webrtc/perf/process_perf_results_test.py new file mode 100644 index 0000000000..3aa5afd75c --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/perf/process_perf_results_test.py @@ -0,0 +1,70 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import os +import sys + +import unittest +from unittest import mock + +_SCRIPT_DIR = os.path.dirname(__file__) +_SRC_DIR = os.path.normpath(os.path.join(_SCRIPT_DIR, '..', '..')) + +sys.path.insert(0, os.path.join(_SRC_DIR, 'third_party', 'protobuf', 'python')) +import process_perf_results + + +class ProcessPerfResultsTest(unittest.TestCase): + def testConfigurePythonPath(self): + # pylint: disable=protected-access + self.assertEqual( + 0, + process_perf_results._ConfigurePythonPath( + os.path.join(_SRC_DIR, 'out/Default'))) + + def testUploadToDasboard(self): + outdir = os.path.join(_SRC_DIR, 'out/Default') + args = mock.Mock( + build_properties='{' + '"outdir":"' + outdir + '", ' + + '"perf_dashboard_machine_group":"mock_machine_group", ' + + '"bot":"mock_bot", ' + '"webrtc_git_hash":"mock_webrtc_git_hash", ' + + '"commit_position":"123456", ' + + '"build_page_url":"mock_build_page_url", ' + + '"dashboard_url":"mock_dashboard_url"' + '}', + summary_json='mock_sumary_json', + task_output_dir='mock_task_output_dir', + test_suite='mock_test_suite', + ) + perftest_output = mock.Mock( + absolute=lambda: 'dummy_path/perftest-output.pb', + is_file=lambda: True, + ) + with mock.patch('pathlib.Path.rglob') as mocked_rglob: + with mock.patch('catapult_uploader.UploadToDashboard') as mocked_upload: + mocked_rglob.return_value = [perftest_output] + mocked_upload.return_value = 0 + # pylint: disable=protected-access + self.assertEqual(0, process_perf_results._UploadToDasboard(args)) + + import catapult_uploader + mocked_upload.assert_called_once_with( + catapult_uploader.UploaderOptions( + perf_dashboard_machine_group='mock_machine_group', + bot='mock_bot', + test_suite='mock_test_suite', + webrtc_git_hash='mock_webrtc_git_hash', + commit_position='123456', + build_page_url='mock_build_page_url', + dashboard_url='mock_dashboard_url', + input_results_file=perftest_output.absolute())) + + +if (__name__) == '__main__': + unittest.main() diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/__init__.py b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/__init__.py diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/build_helpers.py b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/build_helpers.py new file mode 100644 index 0000000000..86fc1a0bbd --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/build_helpers.py @@ -0,0 +1,134 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""This script helps to invoke gn and ninja +which lie in depot_tools repository.""" + +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile + + +def FindSrcDirPath(): + """Returns the abs path to the src/ dir of the project.""" + src_dir = os.path.dirname(os.path.abspath(__file__)) + while os.path.basename(src_dir) != 'src': + src_dir = os.path.normpath(os.path.join(src_dir, os.pardir)) + return src_dir + + +SRC_DIR = FindSrcDirPath() +sys.path.append(os.path.join(SRC_DIR, 'build')) +import find_depot_tools + + +def RunGnCommand(args, root_dir=None): + """Runs `gn` with provided args and return error if any.""" + try: + command = [ + sys.executable, + os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py') + ] + args + subprocess.check_output(command, cwd=root_dir) + except subprocess.CalledProcessError as err: + return err.output + return None + + +# GN_ERROR_RE matches the summary of an error output by `gn check`. +# Matches "ERROR" and following lines until it sees an empty line or a line +# containing just underscores. +GN_ERROR_RE = re.compile(r'^ERROR .+(?:\n.*[^_\n].*$)+', re.MULTILINE) + + +def RunGnCheck(root_dir=None): + """Runs `gn gen --check` with default args to detect mismatches between + #includes and dependencies in the BUILD.gn files, as well as general build + errors. + + Returns a list of error summary strings. + """ + out_dir = tempfile.mkdtemp('gn') + try: + error = RunGnCommand(['gen', '--check', out_dir], root_dir) + finally: + shutil.rmtree(out_dir, ignore_errors=True) + return GN_ERROR_RE.findall(error.decode('utf-8')) if error else [] + + +def RunNinjaCommand(args, root_dir=None): + """Runs ninja quietly. Any failure (e.g. clang not found) is + silently discarded, since this is unlikely an error in submitted CL.""" + command = [os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'ninja')] + args + p = subprocess.Popen(command, + cwd=root_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, _ = p.communicate() + return out + + +def GetClangTidyPath(): + """POC/WIP! Use the one we have, even it doesn't match clang's version.""" + tidy = ('third_party/android_ndk/toolchains/' + 'llvm/prebuilt/linux-x86_64/bin/clang-tidy') + return os.path.join(SRC_DIR, tidy) + + +def GetCompilationDb(root_dir=None): + """Run ninja compdb tool to get proper flags, defines and include paths.""" + # The compdb tool expect a rule. + commands = json.loads(RunNinjaCommand(['-t', 'compdb', 'cxx'], root_dir)) + # Turns 'file' field into a key. + return {v['file']: v for v in commands} + + +def GetCompilationCommand(filepath, gn_args, work_dir): + """Get the whole command used to compile one cc file. + Typically, clang++ with flags, defines and include paths. + + Args: + filepath: path to .cc file. + gen_args: build configuration for gn. + work_dir: build dir. + + Returns: + Command as a list, ready to be consumed by subprocess.Popen. + """ + gn_errors = RunGnCommand(['gen'] + gn_args + [work_dir]) + if gn_errors: + raise RuntimeError('FYI, cannot complete check due to gn error:\n%s\n' + 'Please open a bug.' % gn_errors) + + # Needed for single file compilation. + commands = GetCompilationDb(work_dir) + + # Path as referenced by ninja. + rel_path = os.path.relpath(os.path.abspath(filepath), work_dir) + + # Gather defines, include path and flags (such as -std=c++11). + try: + compilation_entry = commands[rel_path] + except KeyError as not_found: + raise ValueError('%s: Not found in compilation database.\n' + 'Please check the path.' % filepath) from not_found + command = compilation_entry['command'].split() + + # Remove troublesome flags. May trigger an error otherwise. + if '-MMD' in command: + command.remove('-MMD') + if '-MF' in command: + index = command.index('-MF') + del command[index:index + 2] # Remove filename as well. + + return command diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/build_helpers_test.py b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/build_helpers_test.py new file mode 100755 index 0000000000..42b94d6c29 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/build_helpers_test.py @@ -0,0 +1,32 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import re +import os +import unittest + +import build_helpers + +TESTDATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'testdata') + + +class GnCheckTest(unittest.TestCase): + + def testCircularDependencyError(self): + test_dir = os.path.join(TESTDATA_DIR, 'circular_dependency') + expected_error = re.compile('ERROR Dependency cycle') + gn_output = build_helpers.RunGnCheck(test_dir) + self.assertEqual(1, len(gn_output)) + self.assertRegex(gn_output[0], expected_error) + + +if __name__ == '__main__': + unittest.main() diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/check_orphan_headers.py b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/check_orphan_headers.py new file mode 100644 index 0000000000..ae5b5baf2f --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/check_orphan_headers.py @@ -0,0 +1,116 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import os +import re + +# TARGET_RE matches a GN target, and extracts the target name and the contents. +TARGET_RE = re.compile( + r'(?P<indent>\s*)\w+\("(?P<target_name>\w+)"\) {' + r'(?P<target_contents>.*?)' + r'(?P=indent)}', re.MULTILINE | re.DOTALL) + +# SOURCES_RE matches a block of sources inside a GN target. +SOURCES_RE = re.compile( + r'(sources|public|common_objc_headers) \+?= \[(?P<sources>.*?)\]', + re.MULTILINE | re.DOTALL) + +SOURCE_FILE_RE = re.compile(r'.*\"(?P<source_file>.*)\"') + + +class NoBuildGnFoundError(Exception): + pass + + +class WrongFileTypeError(Exception): + pass + + +def _ReadFile(file_path): + """Returns the content of file_path in a string. + + Args: + file_path: the path of the file to read. + Returns: + A string with the content of the file. + """ + with open(file_path) as f: + return f.read() + + +def GetBuildGnPathFromFilePath(file_path, file_exists_check, root_dir_path): + """Returns the BUILD.gn file responsible for file_path. + + Args: + file_path: the absolute path to the .h file to check. + file_exists_check: a function that defines how to check if a file exists + on the file system. + root_dir_path: the absolute path of the root of project. + + Returns: + A string with the absolute path to the BUILD.gn file responsible to include + file_path in a target. + """ + if not file_path.endswith('.h'): + raise WrongFileTypeError( + 'File {} is not an header file (.h)'.format(file_path)) + candidate_dir = os.path.dirname(file_path) + while candidate_dir.startswith(root_dir_path): + candidate_build_gn_path = os.path.join(candidate_dir, 'BUILD.gn') + if file_exists_check(candidate_build_gn_path): + return candidate_build_gn_path + candidate_dir = os.path.abspath(os.path.join(candidate_dir, os.pardir)) + raise NoBuildGnFoundError( + 'No BUILD.gn file found for file: `{}`'.format(file_path)) + + +def IsHeaderInBuildGn(header_path, build_gn_path): + """Returns True if the header is listed in the BUILD.gn file. + + Args: + header_path: the absolute path to the header to check. + build_gn_path: the absolute path to the header to check. + + Returns: + bool: True if the header_path is an header that is listed in + at least one GN target in the BUILD.gn file specified by + the argument build_gn_path. + """ + target_abs_path = os.path.dirname(build_gn_path) + build_gn_content = _ReadFile(build_gn_path) + headers_in_build_gn = GetHeadersInBuildGnFileSources(build_gn_content, + target_abs_path) + return header_path in headers_in_build_gn + + +def GetHeadersInBuildGnFileSources(file_content, target_abs_path): + """Returns a set with all the .h files in the file_content. + + Args: + file_content: a string with the content of the BUILD.gn file. + target_abs_path: the absolute path to the directory where the + BUILD.gn file lives. + + Returns: + A set with all the headers (.h file) in the file_content. + The set contains absolute paths. + """ + headers_in_sources = set([]) + for target_match in TARGET_RE.finditer(file_content): + target_contents = target_match.group('target_contents') + for sources_match in SOURCES_RE.finditer(target_contents): + sources = sources_match.group('sources') + for source_file_match in SOURCE_FILE_RE.finditer(sources): + source_file = source_file_match.group('source_file') + if source_file.endswith('.h'): + source_file_tokens = source_file.split('/') + headers_in_sources.add( + os.path.join(target_abs_path, *source_file_tokens)) + return headers_in_sources diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/check_orphan_headers_test.py b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/check_orphan_headers_test.py new file mode 100755 index 0000000000..957d7b814a --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/check_orphan_headers_test.py @@ -0,0 +1,113 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import os +import sys +import unittest + +import check_orphan_headers + +def _GetRootBasedOnPlatform(): + if sys.platform.startswith('win'): + return 'C:\\' + return '/' + + +def _GetPath(*path_chunks): + return os.path.join(_GetRootBasedOnPlatform(), *path_chunks) + + +class GetBuildGnPathFromFilePathTest(unittest.TestCase): + def testGetBuildGnFromSameDirectory(self): + file_path = _GetPath('home', 'projects', 'webrtc', 'base', 'foo.h') + expected_build_path = _GetPath('home', 'projects', 'webrtc', 'base', + 'BUILD.gn') + file_exists = lambda p: p == _GetPath('home', 'projects', 'webrtc', 'base', + 'BUILD.gn') + src_dir_path = _GetPath('home', 'projects', 'webrtc') + self.assertEqual( + expected_build_path, + check_orphan_headers.GetBuildGnPathFromFilePath(file_path, file_exists, + src_dir_path)) + + def testGetBuildPathFromParentDirectory(self): + file_path = _GetPath('home', 'projects', 'webrtc', 'base', 'foo.h') + expected_build_path = _GetPath('home', 'projects', 'webrtc', 'BUILD.gn') + file_exists = lambda p: p == _GetPath('home', 'projects', 'webrtc', + 'BUILD.gn') + src_dir_path = _GetPath('home', 'projects', 'webrtc') + self.assertEqual( + expected_build_path, + check_orphan_headers.GetBuildGnPathFromFilePath(file_path, file_exists, + src_dir_path)) + + def testExceptionIfNoBuildGnFilesAreFound(self): + with self.assertRaises(check_orphan_headers.NoBuildGnFoundError): + file_path = _GetPath('home', 'projects', 'webrtc', 'base', 'foo.h') + file_exists = lambda p: False + src_dir_path = _GetPath('home', 'projects', 'webrtc') + check_orphan_headers.GetBuildGnPathFromFilePath(file_path, file_exists, + src_dir_path) + + def testExceptionIfFilePathIsNotAnHeader(self): + with self.assertRaises(check_orphan_headers.WrongFileTypeError): + file_path = _GetPath('home', 'projects', 'webrtc', 'base', 'foo.cc') + file_exists = lambda p: False + src_dir_path = _GetPath('home', 'projects', 'webrtc') + check_orphan_headers.GetBuildGnPathFromFilePath(file_path, file_exists, + src_dir_path) + + +class GetHeadersInBuildGnFileSourcesTest(unittest.TestCase): + def testEmptyFileReturnsEmptySet(self): + self.assertEqual( + set([]), + check_orphan_headers.GetHeadersInBuildGnFileSources('', '/a/b')) + + def testReturnsSetOfHeadersFromFileContent(self): + file_content = """ + # Some comments + if (is_android) { + import("//a/b/c.gni") + import("//d/e/f.gni") + } + source_set("foo") { + sources = ["foo.h"] + deps = [":bar"] + } + rtc_static_library("bar") { + # Public headers should also be included. + public = [ + "public_foo.h", + ] + sources = [ + "bar.h", + "bar.cc", + ] + deps = [":bar"] + } + source_set("baz_foo") { + sources = ["baz/foo.h"] + } + """ + target_abs_path = _GetPath('a', 'b') + self.assertEqual( + set([ + _GetPath('a', 'b', 'foo.h'), + _GetPath('a', 'b', 'bar.h'), + _GetPath('a', 'b', 'public_foo.h'), + _GetPath('a', 'b', 'baz', 'foo.h'), + ]), + check_orphan_headers.GetHeadersInBuildGnFileSources( + file_content, target_abs_path)) + + +if __name__ == '__main__': + unittest.main() diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/check_package_boundaries.py b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/check_package_boundaries.py new file mode 100644 index 0000000000..6477a17c39 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/check_package_boundaries.py @@ -0,0 +1,133 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import argparse +import collections +import os +import re +import sys + +# TARGET_RE matches a GN target, and extracts the target name and the contents. +TARGET_RE = re.compile( + r'(?P<indent>\s*)\w+\("(?P<target_name>\w+)"\) {' + r'(?P<target_contents>.*?)' + r'(?P=indent)}', re.MULTILINE | re.DOTALL) + +# SOURCES_RE matches a block of sources inside a GN target. +SOURCES_RE = re.compile(r'sources \+?= \[(?P<sources>.*?)\]', + re.MULTILINE | re.DOTALL) + +ERROR_MESSAGE = ("{build_file_path} in target '{target_name}':\n" + " Source file '{source_file}'\n" + " crosses boundary of package '{subpackage}'.") + + +class PackageBoundaryViolation( + collections.namedtuple( + 'PackageBoundaryViolation', + 'build_file_path target_name source_file subpackage')): + def __str__(self): + return ERROR_MESSAGE.format(**self._asdict()) + + +def _BuildSubpackagesPattern(packages, query): + """Returns a regular expression that matches source files inside subpackages + of the given query.""" + query += os.path.sep + length = len(query) + pattern = r'\s*"(?P<source_file>(?P<subpackage>' + pattern += '|'.join( + re.escape(package[length:].replace(os.path.sep, '/')) + for package in packages if package.startswith(query)) + pattern += r')/[\w\./]*)"' + return re.compile(pattern) + + +def _ReadFileAndPrependLines(file_path): + """Reads the contents of a file.""" + with open(file_path) as f: + return "".join(f.readlines()) + + +def _CheckBuildFile(build_file_path, packages): + """Iterates over all the targets of the given BUILD.gn file, and verifies that + the source files referenced by it don't belong to any of it's subpackages. + Returns an iterator over PackageBoundaryViolations for this package. + """ + package = os.path.dirname(build_file_path) + subpackages_re = _BuildSubpackagesPattern(packages, package) + + build_file_contents = _ReadFileAndPrependLines(build_file_path) + for target_match in TARGET_RE.finditer(build_file_contents): + target_name = target_match.group('target_name') + target_contents = target_match.group('target_contents') + for sources_match in SOURCES_RE.finditer(target_contents): + sources = sources_match.group('sources') + for subpackages_match in subpackages_re.finditer(sources): + subpackage = subpackages_match.group('subpackage') + source_file = subpackages_match.group('source_file') + if subpackage: + yield PackageBoundaryViolation(build_file_path, target_name, + source_file, subpackage) + + +def CheckPackageBoundaries(root_dir, build_files=None): + packages = [ + root for root, _, files in os.walk(root_dir) if 'BUILD.gn' in files + ] + + if build_files is not None: + for build_file_path in build_files: + assert build_file_path.startswith(root_dir) + else: + build_files = [os.path.join(package, 'BUILD.gn') for package in packages] + + messages = [] + for build_file_path in build_files: + messages.extend(_CheckBuildFile(build_file_path, packages)) + return messages + + +def main(argv): + parser = argparse.ArgumentParser( + description='Script that checks package boundary violations in GN ' + 'build files.') + + parser.add_argument('root_dir', + metavar='ROOT_DIR', + help='The root directory that contains all BUILD.gn ' + 'files to be processed.') + parser.add_argument('build_files', + metavar='BUILD_FILE', + nargs='*', + help='A list of BUILD.gn files to be processed. If no ' + 'files are given, all BUILD.gn files under ROOT_DIR ' + 'will be processed.') + parser.add_argument('--max_messages', + type=int, + default=None, + help='If set, the maximum number of violations to be ' + 'displayed.') + + args = parser.parse_args(argv) + + messages = CheckPackageBoundaries(args.root_dir, args.build_files) + messages = messages[:args.max_messages] + + for i, message in enumerate(messages): + if i > 0: + print() + print(message) + + return bool(messages) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/check_package_boundaries_test.py b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/check_package_boundaries_test.py new file mode 100755 index 0000000000..611af3a236 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/check_package_boundaries_test.py @@ -0,0 +1,71 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import ast +import os +import unittest + +import check_package_boundaries + +MSG_FORMAT = 'ERROR:check_package_boundaries.py: Unexpected %s.' +TESTDATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'testdata') + + +def ReadPylFile(file_path): + with open(file_path) as f: + return ast.literal_eval(f.read()) + + +class UnitTest(unittest.TestCase): + def _RunTest(self, test_dir, check_all_build_files=False): + build_files = [os.path.join(test_dir, 'BUILD.gn')] + if check_all_build_files: + build_files = None + + messages = [] + for violation in check_package_boundaries.CheckPackageBoundaries( + test_dir, build_files): + build_file_path = os.path.relpath(violation.build_file_path, test_dir) + build_file_path = build_file_path.replace(os.path.sep, '/') + messages.append(violation._replace(build_file_path=build_file_path)) + + expected_messages = ReadPylFile(os.path.join(test_dir, 'expected.pyl')) + self.assertListEqual(sorted(expected_messages), sorted(messages)) + + def testNoErrors(self): + self._RunTest(os.path.join(TESTDATA_DIR, 'no_errors')) + + def testMultipleErrorsSingleTarget(self): + self._RunTest(os.path.join(TESTDATA_DIR, 'multiple_errors_single_target')) + + def testMultipleErrorsMultipleTargets(self): + self._RunTest(os.path.join(TESTDATA_DIR, + 'multiple_errors_multiple_targets')) + + def testCommonPrefix(self): + self._RunTest(os.path.join(TESTDATA_DIR, 'common_prefix')) + + def testAllBuildFiles(self): + self._RunTest(os.path.join(TESTDATA_DIR, 'all_build_files'), True) + + def testSanitizeFilename(self): + # The `dangerous_filename` test case contains a directory with '++' in its + # name. If it's not properly escaped, a regex error would be raised. + self._RunTest(os.path.join(TESTDATA_DIR, 'dangerous_filename'), True) + + def testRelativeFilename(self): + test_dir = os.path.join(TESTDATA_DIR, 'all_build_files') + with self.assertRaises(AssertionError): + check_package_boundaries.CheckPackageBoundaries(test_dir, ["BUILD.gn"]) + + +if __name__ == '__main__': + unittest.main() diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/all_build_files/BUILD.gn b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/all_build_files/BUILD.gn new file mode 100644 index 0000000000..46bd2bec8f --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/all_build_files/BUILD.gn @@ -0,0 +1,14 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +target("dummy_target_ok") { + sources = [ + "dummy_source.cc", + "dummy_source.h", + ] +} diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/all_build_files/expected.pyl b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/all_build_files/expected.pyl new file mode 100644 index 0000000000..07f98e9a6e --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/all_build_files/expected.pyl @@ -0,0 +1,16 @@ +[('subpackage2/BUILD.gn', + 'error_2', + 'subsubpackage2/dummy_subsubpackage2.cc', + 'subsubpackage2'), + ('subpackage2/BUILD.gn', + 'error_2', + 'subsubpackage2/dummy_subsubpackage2.h', + 'subsubpackage2'), + ('subpackage1/BUILD.gn', + 'error_1', + 'subsubpackage1/dummy_subsubpackage1.cc', + 'subsubpackage1'), + ('subpackage1/BUILD.gn', + 'error_1', + 'subsubpackage1/dummy_subsubpackage1.h', + 'subsubpackage1')] diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/all_build_files/subpackage1/BUILD.gn b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/all_build_files/subpackage1/BUILD.gn new file mode 100644 index 0000000000..2653a2b607 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/all_build_files/subpackage1/BUILD.gn @@ -0,0 +1,15 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +target("error_1") { + sources = [ + "subpackage1.h", + "subsubpackage1/dummy_subsubpackage1.cc", + "subsubpackage1/dummy_subsubpackage1.h", + ] +} diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/all_build_files/subpackage1/subsubpackage1/BUILD.gn b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/all_build_files/subpackage1/subsubpackage1/BUILD.gn new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/all_build_files/subpackage1/subsubpackage1/BUILD.gn diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/all_build_files/subpackage2/BUILD.gn b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/all_build_files/subpackage2/BUILD.gn new file mode 100644 index 0000000000..290036145d --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/all_build_files/subpackage2/BUILD.gn @@ -0,0 +1,15 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +target("error_2") { + sources = [ + "subpackage2.h", + "subsubpackage2/dummy_subsubpackage2.cc", + "subsubpackage2/dummy_subsubpackage2.h", + ] +} diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/all_build_files/subpackage2/subsubpackage2/BUILD.gn b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/all_build_files/subpackage2/subsubpackage2/BUILD.gn new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/all_build_files/subpackage2/subsubpackage2/BUILD.gn diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/circular_dependency/.gn b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/circular_dependency/.gn new file mode 100644 index 0000000000..9fe0b4226c --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/circular_dependency/.gn @@ -0,0 +1 @@ +buildconfig = "//BUILDCONFIG.gn" diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/circular_dependency/BUILD.gn b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/circular_dependency/BUILD.gn new file mode 100644 index 0000000000..86095b2f79 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/circular_dependency/BUILD.gn @@ -0,0 +1,10 @@ +toolchain("toolchain") { +} + +static_library("foo") { + deps = [ ":bar" ] +} + +static_library("bar") { + deps = [ ":foo" ] +} diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/circular_dependency/BUILDCONFIG.gn b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/circular_dependency/BUILDCONFIG.gn new file mode 100644 index 0000000000..48c2a464b2 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/circular_dependency/BUILDCONFIG.gn @@ -0,0 +1 @@ +set_default_toolchain(":toolchain") diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/common_prefix/BUILD.gn b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/common_prefix/BUILD.gn new file mode 100644 index 0000000000..dd5145bc68 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/common_prefix/BUILD.gn @@ -0,0 +1,15 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +rtc_library("webrtc") { + sources = [ + "call.h", + "dummy_source.h", + ] + deps = [ "call" ] +} diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/common_prefix/call/BUILD.gn b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/common_prefix/call/BUILD.gn new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/common_prefix/call/BUILD.gn diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/common_prefix/expected.pyl b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/common_prefix/expected.pyl new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/common_prefix/expected.pyl @@ -0,0 +1 @@ +[] diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/dangerous_filename/BUILD.gn b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/dangerous_filename/BUILD.gn new file mode 100644 index 0000000000..d4e728431b --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/dangerous_filename/BUILD.gn @@ -0,0 +1,13 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +# "libc++" is considered a "dangerous filename" because it's an invalid regex. + +target("dummy_target") { + sources = [ "libc++/dummy_subpackage_file.h" ] +} diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/dangerous_filename/expected.pyl b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/dangerous_filename/expected.pyl new file mode 100644 index 0000000000..34f23f8a01 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/dangerous_filename/expected.pyl @@ -0,0 +1,4 @@ +[("BUILD.gn", + "dummy_target", + "libc++/dummy_subpackage_file.h", + "libc++")] diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/dangerous_filename/libc++/BUILD.gn b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/dangerous_filename/libc++/BUILD.gn new file mode 100644 index 0000000000..b7c1f8aac2 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/dangerous_filename/libc++/BUILD.gn @@ -0,0 +1,11 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +group("dummy_subpackage") { + sources = [ "dummy_subpackage.h" ] +} diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/multiple_errors_multiple_targets/BUILD.gn b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/multiple_errors_multiple_targets/BUILD.gn new file mode 100644 index 0000000000..db8c426588 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/multiple_errors_multiple_targets/BUILD.gn @@ -0,0 +1,28 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +target("dummy_target_ok") { + sources = [ + "dummy_source.cc", + "dummy_source.h", + ] +} + +target("error_1") { + sources = [ + "subpackage1/dummy_subpackage1.cc", + "subpackage1/dummy_subpackage1.h", + ] +} + +target("error_2") { + sources = [ + "subpackage1/dummy_subpackage2.cc", + "subpackage1/dummy_subpackage2.h", + ] +} diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/multiple_errors_multiple_targets/expected.pyl b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/multiple_errors_multiple_targets/expected.pyl new file mode 100644 index 0000000000..9b9ad01c6c --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/multiple_errors_multiple_targets/expected.pyl @@ -0,0 +1,16 @@ +[('BUILD.gn', + 'error_1', + 'subpackage1/dummy_subpackage1.cc', + 'subpackage1'), + ('BUILD.gn', + 'error_1', + 'subpackage1/dummy_subpackage1.h', + 'subpackage1'), + ('BUILD.gn', + 'error_2', + 'subpackage1/dummy_subpackage2.cc', + 'subpackage1'), + ('BUILD.gn', + 'error_2', + 'subpackage1/dummy_subpackage2.h', + 'subpackage1')] diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/multiple_errors_multiple_targets/subpackage1/BUILD.gn b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/multiple_errors_multiple_targets/subpackage1/BUILD.gn new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/multiple_errors_multiple_targets/subpackage1/BUILD.gn diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/multiple_errors_multiple_targets/subpackage2/BUILD.gn b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/multiple_errors_multiple_targets/subpackage2/BUILD.gn new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/multiple_errors_multiple_targets/subpackage2/BUILD.gn diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/multiple_errors_single_target/BUILD.gn b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/multiple_errors_single_target/BUILD.gn new file mode 100644 index 0000000000..eb5ee40846 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/multiple_errors_single_target/BUILD.gn @@ -0,0 +1,14 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +target("dummy_target") { + sources = [ + "subpackage/dummy_subpackage_file.cc", + "subpackage/dummy_subpackage_file.h", + ] +} diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/multiple_errors_single_target/expected.pyl b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/multiple_errors_single_target/expected.pyl new file mode 100644 index 0000000000..012d3bd1c2 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/multiple_errors_single_target/expected.pyl @@ -0,0 +1,8 @@ +[("BUILD.gn", + "dummy_target", + "subpackage/dummy_subpackage_file.cc", + "subpackage"), + ("BUILD.gn", + "dummy_target", + "subpackage/dummy_subpackage_file.h", + "subpackage")] diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/multiple_errors_single_target/subpackage/BUILD.gn b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/multiple_errors_single_target/subpackage/BUILD.gn new file mode 100644 index 0000000000..700508155b --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/multiple_errors_single_target/subpackage/BUILD.gn @@ -0,0 +1,14 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +group("dummy_supbackage") { + sources = [ + "dummy_subpackage.cc", + "dummy_subpackage.h", + ] +} diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/no_errors/BUILD.gn b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/no_errors/BUILD.gn new file mode 100644 index 0000000000..559d333cb2 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/no_errors/BUILD.gn @@ -0,0 +1,16 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +group("testdata") { + deps = [ ":dummy_target" ] +} + +static_library("dummy_target") { + sources = [ "dummy.cc" ] + deps = [ "subdir" ] +} diff --git a/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/no_errors/expected.pyl b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/no_errors/expected.pyl new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/presubmit_checks_lib/testdata/no_errors/expected.pyl @@ -0,0 +1 @@ +[] diff --git a/third_party/libwebrtc/tools_webrtc/sanitizers/README b/third_party/libwebrtc/tools_webrtc/sanitizers/README new file mode 100644 index 0000000000..68c3015f49 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/sanitizers/README @@ -0,0 +1,3 @@ +This directory contains suppressions for sanitizer tools. +They're overriding the default ones for Chromium using our +webrtc/supplement.gypi file. diff --git a/third_party/libwebrtc/tools_webrtc/sanitizers/lsan_suppressions_webrtc.cc b/third_party/libwebrtc/tools_webrtc/sanitizers/lsan_suppressions_webrtc.cc new file mode 100644 index 0000000000..064b2804ab --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/sanitizers/lsan_suppressions_webrtc.cc @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This file contains the WebRTC suppressions for LeakSanitizer. +// You can also pass additional suppressions via LSAN_OPTIONS: +// LSAN_OPTIONS=suppressions=/path/to/suppressions. Please refer to +// http://dev.chromium.org/developers/testing/leaksanitizer for more info. + +#if defined(LEAK_SANITIZER) + +// Please make sure the code below declares a single string variable +// kLSanDefaultSuppressions which contains LSan suppressions delimited by +// newlines. See http://dev.chromium.org/developers/testing/leaksanitizer +// for the instructions on writing suppressions. +char kLSanDefaultSuppressions[] = + + // ============ Leaks in third-party code shared with Chromium ============= + // These entries are copied from build/sanitizers/lsan_suppressions.cc in + // Chromium. Please don't add new entries here unless they're present in + // there. + + // False positives in libfontconfig. http://crbug.com/39050 + "leak:libfontconfig\n" + + // Leaks in Nvidia's libGL. + "leak:libGL.so\n" + + // XRandR has several one time leaks. + "leak:libxrandr\n" + + // xrandr leak. http://crbug.com/119677 + "leak:XRRFindDisplay\n" + + // ========== Leaks in third-party code not shared with Chromium =========== + + // None known so far. + + // ================ Leaks in WebRTC code ================ + // PLEASE DO NOT ADD SUPPRESSIONS FOR NEW LEAKS. + // Instead, commits that introduce memory leaks should be reverted. + // Suppressing the leak is acceptable in some cases when reverting is + // impossible, i.e. when enabling leak detection for the first time for a + // test target with pre-existing leaks. + + // rtc_unittest + // https://code.google.com/p/webrtc/issues/detail?id=3827 for details. + "leak:rtc::unstarted_task_test_DoNotDeleteTask2_Test::TestBody\n" + "leak:rtc::HttpServer::HandleConnection\n" + "leak:rtc::HttpServer::Connection::onHttpHeaderComplete\n" + "leak:rtc::HttpResponseData::set_success\n" + "leak:rtc::HttpData::changeHeader\n" + // https://code.google.com/p/webrtc/issues/detail?id=4149 for details. + "leak:StartDNSLookup\n" + + // rtc_media_unittests + "leak:cricket::FakeNetworkInterface::SetOption\n" + "leak:CodecTest_TestCodecOperators_Test::TestBody\n" + "leak:VideoEngineTest*::ConstrainNewCodecBody\n" + "leak:VideoMediaChannelTest*::AddRemoveRecvStreams\n" + "leak:WebRtcVideoCapturerTest_TestCapture_Test::TestBody\n" + "leak:WebRtcVideoEngineTestFake_MultipleSendStreamsWithOneCapturer_Test::" + "TestBody\n" + "leak:WebRtcVideoEngineTestFake_SetBandwidthInConference_Test::TestBody\n" + "leak:WebRtcVideoEngineTestFake_SetSendCodecsRejectBadFormat_Test::" + "TestBody\n" + + // peerconnection_unittests + // https://code.google.com/p/webrtc/issues/detail?id=2528 + "leak:cricket::FakeVideoMediaChannel::~FakeVideoMediaChannel\n" + "leak:cricket::MediaSessionDescriptionFactory::CreateAnswer\n" + "leak:cricket::MediaSessionDescriptionFactory::CreateOffer\n" + "leak:DtmfSenderTest_InsertEmptyTonesToCancelPreviousTask_Test::TestBody\n" + "leak:sigslot::_signal_base2*::~_signal_base2\n" + "leak:testing::internal::CmpHelperEQ\n" + "leak:webrtc::AudioDeviceLinuxALSA::InitMicrophone\n" + "leak:webrtc::AudioDeviceLinuxALSA::InitSpeaker\n" + "leak:webrtc::CreateIceCandidate\n" + "leak:webrtc::WebRtcIdentityRequestObserver::OnSuccess\n" + "leak:webrtc::WebRtcSessionDescriptionFactory::InternalCreateAnswer\n" + "leak:webrtc::WebRtcSessionDescriptionFactory::InternalCreateOffer\n" + "leak:PeerConnectionInterfaceTest_SsrcInOfferAnswer_Test::TestBody\n" + "leak:PeerConnectionInterfaceTest_CloseAndTestMethods_Test::TestBody\n" + "leak:WebRtcSdpTest::TestDeserializeRtcpFb\n" + "leak:WebRtcSdpTest::TestSerialize\n" + "leak:WebRtcSdpTest_SerializeSessionDescriptionWithDataChannelAndBandwidth_" + "Test::TestBody\n" + "leak:WebRtcSdpTest_SerializeSessionDescriptionWithBandwidth_Test::" + "TestBody\n" + "leak:WebRtcSessionTest::SetLocalDescriptionExpectError\n" + "leak:WebRtcSessionTest_TestAVOfferWithAudioOnlyAnswer_Test::TestBody\n" + + // PLEASE READ ABOVE BEFORE ADDING NEW SUPPRESSIONS. + + // End of suppressions. + ; // Please keep this semicolon. + +#endif // LEAK_SANITIZER diff --git a/third_party/libwebrtc/tools_webrtc/sanitizers/tsan_suppressions_webrtc.cc b/third_party/libwebrtc/tools_webrtc/sanitizers/tsan_suppressions_webrtc.cc new file mode 100644 index 0000000000..a3a4bdbdd0 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/sanitizers/tsan_suppressions_webrtc.cc @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This file contains the WebRTC suppressions for ThreadSanitizer. +// Please refer to +// http://dev.chromium.org/developers/testing/threadsanitizer-tsan-v2 +// for more info. + +#if defined(THREAD_SANITIZER) + +// Please make sure the code below declares a single string variable +// kTSanDefaultSuppressions contains TSan suppressions delimited by newlines. +// See http://dev.chromium.org/developers/testing/threadsanitizer-tsan-v2 +// for the instructions on writing suppressions. +char kTSanDefaultSuppressions[] = + + // WebRTC specific suppressions. + + // Split up suppressions covered previously by thread.cc and + // messagequeue.cc. + "race:vp8cx_remove_encoder_threads\n" + "race:third_party/libvpx/source/libvpx/vp9/common/vp9_scan.h\n" + + // rtc_unittests + // https://code.google.com/p/webrtc/issues/detail?id=2080 + "race:rtc_base/logging.cc\n" + + // Potential deadlocks detected after roll in r6516. + // https://code.google.com/p/webrtc/issues/detail?id=3509 + "deadlock:webrtc::test::UdpSocketManagerPosixImpl::RemoveSocket\n" + + // TODO(pbos): Trace events are racy due to lack of proper POD atomics. + // https://code.google.com/p/webrtc/issues/detail?id=2497 + "race:*trace_event_unique_catstatic*\n" + + // Race between InitCpuFlags and TestCpuFlag in libyuv. + // https://code.google.com/p/libyuv/issues/detail?id=508 + "race:InitCpuFlags\n" + + // Test-only race due to PeerConnection::session() being virtual for + // testing. The stats collector may call session() before or after the + // destructor begins executing, which modifies the vtable. + "race:*RTCStatsIntegrationTest_GetsStatsWhileDestroyingPeerConnections_" + "Test::TestBody\n" + + // http://crbug.com/244856 + "race:libpulsecommon*.so\n" + + // https://crbug.com/1158622 + "race:absl::synchronization_internal::Waiter::Post\n" + + // End of suppressions. + ; // Please keep this semicolon. + +#endif // THREAD_SANITIZER diff --git a/third_party/libwebrtc/tools_webrtc/sslroots/README.md b/third_party/libwebrtc/tools_webrtc/sslroots/README.md new file mode 100644 index 0000000000..0184a412a9 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/sslroots/README.md @@ -0,0 +1,21 @@ +# Generate rtc_base/ssl_roots.h + +This directory contains a script to generate the content of +[rtc_base/ssl_roots.h][ssl-roots-header], to update the SSL roots shipped +by WebRTC follow this instructions: + +1. Download roots.pem from [pki.goog][pki-goog] or [curl.se][mozilla-cacert] + +2. Launch the script: + +``` +$ vpython3 tools_webrtc/sslroots/generate_sslroots.py <the pem file> +``` + +3. Step 2 should have generated an ssl_roots.h file right next to the pem file. + +4. Overwrite rtc_base/ssl_roots.h with the newly generated one. + +[ssl-roots-header]: https://cs.chromium.org/chromium/src/third_party/webrtc/rtc_base/ssl_roots.h +[pki-goog]: https://www.google.com/url?q=https://pki.google.com/roots.pem +[mozila-cacert]: https://curl.se/ca/cacert.pem diff --git a/third_party/libwebrtc/tools_webrtc/sslroots/generate_sslroots.py b/third_party/libwebrtc/tools_webrtc/sslroots/generate_sslroots.py new file mode 100644 index 0000000000..e2da02935d --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/sslroots/generate_sslroots.py @@ -0,0 +1,226 @@ +#!/usr/bin/env vpython3 + +# -*- coding:utf-8 -*- +# Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""This is a tool to transform a crt file into a C/C++ header. + +Usage: +python3 generate_sslroots.py certfile.pem [--verbose | -v] [--full_cert | -f] + +Arguments: + -v Print output while running. + -f Add public key and certificate name. Default is to skip and reduce + generated file size. + +The supported cert files are: + - Google: https://pki.goog/roots.pem + - Mozilla: https://curl.se/docs/caextract.html +""" + +import subprocess +from optparse import OptionParser +import os +import re + +_GENERATED_FILE = 'ssl_roots.h' +_PREFIX = '__generated__' +_EXTENSION = '.crt' +_SUBJECT_NAME_ARRAY = 'subject_name' +_SUBJECT_NAME_VARIABLE = 'SubjectName' +_PUBLIC_KEY_ARRAY = 'public_key' +_PUBLIC_KEY_VARIABLE = 'PublicKey' +_CERTIFICATE_ARRAY = 'certificate' +_CERTIFICATE_VARIABLE = 'Certificate' +_CERTIFICATE_SIZE_VARIABLE = 'CertificateSize' +_INT_TYPE = 'size_t' +_CHAR_TYPE = 'unsigned char* const' +_VERBOSE = 'verbose' +_MOZILLA_BUNDLE_CHECK = '## Certificate data from Mozilla as of:' + + +def main(): + """The main entrypoint.""" + parser = OptionParser('usage %prog FILE') + parser.add_option('-v', '--verbose', dest='verbose', action='store_true') + parser.add_option('-f', '--full_cert', dest='full_cert', action='store_true') + options, args = parser.parse_args() + if len(args) < 1: + parser.error('No crt file specified.') + return + root_dir, bundle_type = _SplitCrt(args[0], options) + _GenCFiles(root_dir, options, bundle_type) + _Cleanup(root_dir) + + +def _SplitCrt(source_file, options): + sub_file_blocks = [] + label_name = '' + prev_line = None + root_dir = os.path.dirname(os.path.abspath(source_file)) + '/' + _PrintOutput(root_dir, options) + lines = None + with open(source_file) as f: + lines = f.readlines() + mozilla_bundle = any(l.startswith(_MOZILLA_BUNDLE_CHECK) for l in lines) + for line in lines: + if line.startswith('#'): + if mozilla_bundle: + continue + if line.startswith('# Label: '): + sub_file_blocks.append(line) + label = re.search(r'\".*\"', line) + temp_label = label.group(0) + end = len(temp_label) - 1 + label_name = _SafeName(temp_label[1:end]) + if mozilla_bundle and line.startswith('==='): + sub_file_blocks.append(line) + label_name = _SafeName(prev_line) + elif line.startswith('-----END CERTIFICATE-----'): + sub_file_blocks.append(line) + new_file_name = root_dir + _PREFIX + label_name + _EXTENSION + _PrintOutput('Generating: ' + new_file_name, options) + new_file = open(new_file_name, 'w') + for out_line in sub_file_blocks: + new_file.write(out_line) + new_file.close() + sub_file_blocks = [] + else: + sub_file_blocks.append(line) + prev_line = line + return root_dir, 'Mozilla' if mozilla_bundle else 'Google' + + +def _GenCFiles(root_dir, options, bundle_type): + output_header_file = open(root_dir + _GENERATED_FILE, 'w') + output_header_file.write(_CreateOutputHeader(bundle_type)) + if options.full_cert: + subject_name_list = _CreateArraySectionHeader(_SUBJECT_NAME_VARIABLE, + _CHAR_TYPE, options) + public_key_list = _CreateArraySectionHeader(_PUBLIC_KEY_VARIABLE, + _CHAR_TYPE, options) + certificate_list = _CreateArraySectionHeader(_CERTIFICATE_VARIABLE, + _CHAR_TYPE, options) + certificate_size_list = _CreateArraySectionHeader(_CERTIFICATE_SIZE_VARIABLE, + _INT_TYPE, options) + + for _, _, files in os.walk(root_dir): + for current_file in files: + if current_file.startswith(_PREFIX): + prefix_length = len(_PREFIX) + length = len(current_file) - len(_EXTENSION) + label = current_file[prefix_length:length] + filtered_output, cert_size = _CreateCertSection(root_dir, current_file, + label, options) + output_header_file.write(filtered_output + '\n\n\n') + if options.full_cert: + subject_name_list += _AddLabelToArray(label, _SUBJECT_NAME_ARRAY) + public_key_list += _AddLabelToArray(label, _PUBLIC_KEY_ARRAY) + certificate_list += _AddLabelToArray(label, _CERTIFICATE_ARRAY) + certificate_size_list += (' %s,\n') % (cert_size) + + if options.full_cert: + subject_name_list += _CreateArraySectionFooter() + output_header_file.write(subject_name_list) + public_key_list += _CreateArraySectionFooter() + output_header_file.write(public_key_list) + certificate_list += _CreateArraySectionFooter() + output_header_file.write(certificate_list) + certificate_size_list += _CreateArraySectionFooter() + output_header_file.write(certificate_size_list) + output_header_file.write(_CreateOutputFooter()) + output_header_file.close() + + +def _Cleanup(root_dir): + for f in os.listdir(root_dir): + if f.startswith(_PREFIX): + os.remove(root_dir + f) + + +def _CreateCertSection(root_dir, source_file, label, options): + command = 'openssl x509 -in %s%s -noout -C' % (root_dir, source_file) + _PrintOutput(command, options) + output = subprocess.getstatusoutput(command)[1] + renamed_output = output.replace('unsigned char XXX_', + 'const unsigned char ' + label + '_') + filtered_output = '' + cert_block = '^const unsigned char.*?};$' + prog = re.compile(cert_block, re.IGNORECASE | re.MULTILINE | re.DOTALL) + if not options.full_cert: + filtered_output = prog.sub('', renamed_output, count=2) + else: + filtered_output = renamed_output + + cert_size_block = r'\d\d\d+' + prog2 = re.compile(cert_size_block, re.MULTILINE | re.VERBOSE) + result = prog2.findall(renamed_output) + cert_size = result[len(result) - 1] + + return filtered_output, cert_size + + +def _CreateOutputHeader(bundle_type): + output = ('/*\n' + ' * Copyright 2004 The WebRTC Project Authors. All rights ' + 'reserved.\n' + ' *\n' + ' * Use of this source code is governed by a BSD-style license\n' + ' * that can be found in the LICENSE file in the root of the ' + 'source\n' + ' * tree. An additional intellectual property rights grant can be ' + 'found\n' + ' * in the file PATENTS. All contributing project authors may\n' + ' * be found in the AUTHORS file in the root of the source tree.\n' + ' */\n\n' + '#ifndef RTC_BASE_SSL_ROOTS_H_\n' + '#define RTC_BASE_SSL_ROOTS_H_\n\n' + '// This file is the root certificates in C form.\n\n' + '// It was generated with the following script:\n' + '// tools_webrtc/sslroots/generate_sslroots.py' + ' %s_CA_bundle.pem\n\n' + '// clang-format off\n' + '// Don\'t bother formatting generated code,\n' + '// also it would breaks subject/issuer lines.\n\n' % bundle_type) + return output + + +def _CreateOutputFooter(): + return '// clang-format on\n\n#endif // RTC_BASE_SSL_ROOTS_H_\n' + + +def _CreateArraySectionHeader(type_name, type_type, options): + output = ('const %s kSSLCert%sList[] = {\n') % (type_type, type_name) + _PrintOutput(output, options) + return output + + +def _AddLabelToArray(label, type_name): + return ' %s_%s,\n' % (label, type_name) + + +def _CreateArraySectionFooter(): + return '};\n\n' + + +def _SafeName(original_file_name): + bad_chars = ' -./\\()áéÃőú\r\n' + replacement_chars = '' + for _ in bad_chars: + replacement_chars += '_' + translation_table = str.maketrans(bad_chars, replacement_chars) + return original_file_name.translate(translation_table) + + +def _PrintOutput(output, options): + if options.verbose: + print(output) + + +if __name__ == '__main__': + main() diff --git a/third_party/libwebrtc/tools_webrtc/ubsan/suppressions.txt b/third_party/libwebrtc/tools_webrtc/ubsan/suppressions.txt new file mode 100644 index 0000000000..dc76f38c20 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/ubsan/suppressions.txt @@ -0,0 +1,24 @@ +############################################################################# +# UBSan ignorelist. +# +# This is a WebRTC-specific replacement of Chromium's ignorelist.txt. +# Only exceptions for third party libraries go here. WebRTC's code should use +# the RTC_NO_SANITIZE macro. Please think twice before adding new exceptions. + +############################################################################# +# YASM does some funny things that UBsan doesn't like. +# https://crbug.com/489901 +src:*/third_party/yasm/* + +# OpenH264 triggers some errors that are out of our control. +src:*/third_party/ffmpeg/libavcodec/* +src:*/third_party/openh264/* + +# TODO(bugs.webrtc.org/11110). +# Remove those once upstream code has been cleaned. +src:*/third_party/abseil-cpp/absl/debugging/* +src:*/third_party/libvpx/source/libvpx/vp8/* + +############################################################################# +# Ignore system libraries. +src:*/usr/* diff --git a/third_party/libwebrtc/tools_webrtc/ubsan/vptr_suppressions.txt b/third_party/libwebrtc/tools_webrtc/ubsan/vptr_suppressions.txt new file mode 100644 index 0000000000..617ba88f98 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/ubsan/vptr_suppressions.txt @@ -0,0 +1,27 @@ +############################################################################# +# UBSan vptr ignorelist. +# Function and type based blacklisting use a mangled name, and it is especially +# tricky to represent C++ types. For now, any possible changes by name manglings +# are simply represented as wildcard expressions of regexp, and thus it might be +# over-blacklisted. +# +# Please think twice before you add or remove these rules. +# +# This is a stripped down copy of Chromium's vptr_blacklist.txt, to enable +# adding WebRTC-specific ignorelist entries. + +############################################################################# +# Using raw pointer values. +# +# A raw pointer value (16) is used to infer the field offset by +# GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET. + +src:*/third_party/protobuf/src/google/protobuf/compiler/plugin.pb.cc +src:*/third_party/protobuf/src/google/protobuf/compiler/cpp/cpp_message.cc +src:*/third_party/protobuf/src/google/protobuf/descriptor.pb.cc + +############################################################################# +# UBsan goes into an infinite recursion when __dynamic_cast instrumented with +# "vptr". See crbug.com/609786. + +src:*/third_party/libc\+\+abi/trunk/src/private_typeinfo.cpp diff --git a/third_party/libwebrtc/tools_webrtc/version_updater/update_version.py b/third_party/libwebrtc/tools_webrtc/version_updater/update_version.py new file mode 100644 index 0000000000..2a693cd630 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/version_updater/update_version.py @@ -0,0 +1,168 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Script to auto-update the WebRTC source version in call/version.cc""" + +import argparse +import datetime +import logging +import os +import re +import subprocess +import sys + + +def FindSrcDirPath(): + """Returns the abs path to the src/ dir of the project.""" + src_dir = os.path.dirname(os.path.abspath(__file__)) + while os.path.basename(src_dir) != 'src': + src_dir = os.path.normpath(os.path.join(src_dir, os.pardir)) + return src_dir + + +UPDATE_BRANCH_NAME = 'webrtc_version_update' +CHECKOUT_SRC_DIR = FindSrcDirPath() + +NOTIFY_EMAIL = 'webrtc-trooper@webrtc.org' + + +def _RemovePreviousUpdateBranch(): + active_branch, branches = _GetBranches() + if active_branch == UPDATE_BRANCH_NAME: + active_branch = 'main' + if UPDATE_BRANCH_NAME in branches: + logging.info('Removing previous update branch (%s)', UPDATE_BRANCH_NAME) + subprocess.check_call(['git', 'checkout', active_branch]) + subprocess.check_call(['git', 'branch', '-D', UPDATE_BRANCH_NAME]) + logging.info('No branch to remove') + + +def _GetLastAuthor(): + """Returns a string with the author of the last commit.""" + author = subprocess.check_output( + ['git', 'log', '-1', '--pretty=format:"%an"'], + universal_newlines=True).splitlines() + return author + + +def _GetBranches(): + """Returns a tuple (active, branches). + + 'active' is a string with name of the currently active branch, while + 'branches' is the list of all branches. + """ + lines = subprocess.check_output(['git', 'branch'], + universal_newlines=True).splitlines() + branches = [] + active = '' + for line in lines: + if '*' in line: + # The assumption is that the first char will always be the '*'. + active = line[1:].strip() + branches.append(active) + else: + branch = line.strip() + if branch: + branches.append(branch) + return active, branches + + +def _CreateUpdateBranch(): + logging.info('Creating update branch: %s', UPDATE_BRANCH_NAME) + subprocess.check_call(['git', 'checkout', '-b', UPDATE_BRANCH_NAME]) + + +def _UpdateWebRTCVersion(filename): + with open(filename, 'rb') as f: + content = f.read().decode('utf-8') + d = datetime.datetime.utcnow() + # pylint: disable=line-too-long + new_content = re.sub( + r'WebRTC source stamp [0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}', + r'WebRTC source stamp %02d-%02d-%02dT%02d:%02d:%02d' % + (d.year, d.month, d.day, d.hour, d.minute, d.second), + content, + flags=re.MULTILINE) + # pylint: enable=line-too-long + with open(filename, 'wb') as f: + f.write(new_content.encode('utf-8')) + + +def _IsTreeClean(): + stdout = subprocess.check_output(['git', 'status', '--porcelain'], + universal_newlines=True) + if len(stdout) == 0: + return True + return False + + +def _LocalCommit(): + logging.info('Committing changes locally.') + d = datetime.datetime.utcnow() + + commit_msg = ('Update WebRTC code version (%02d-%02d-%02dT%02d:%02d:%02d).' + '\n\nBug: None') + commit_msg = commit_msg % (d.year, d.month, d.day, d.hour, d.minute, d.second) + subprocess.check_call(['git', 'add', '--update', '.']) + subprocess.check_call(['git', 'commit', '-m', commit_msg]) + + +def _UploadCL(commit_queue_mode): + """Upload the committed changes as a changelist to Gerrit. + + commit_queue_mode: + - 2: Submit to commit queue. + - 1: Run trybots but do not submit to CQ. + - 0: Skip CQ, upload only. + """ + cmd = [ + 'git', 'cl', 'upload', '--force', '--bypass-hooks', '--bypass-watchlist' + ] + if commit_queue_mode >= 2: + logging.info('Sending the CL to the CQ...') + cmd.extend(['-o', 'label=Bot-Commit+1']) + cmd.extend(['-o', 'label=Commit-Queue+2']) + cmd.extend(['--send-mail', '--cc', NOTIFY_EMAIL]) + elif commit_queue_mode >= 1: + logging.info('Starting CQ dry run...') + cmd.extend(['-o', 'label=Commit-Queue+1']) + subprocess.check_call(cmd) + + +def main(): + logging.basicConfig(level=logging.INFO) + p = argparse.ArgumentParser() + p.add_argument('--clean', + action='store_true', + default=False, + help='Removes any previous local update branch.') + opts = p.parse_args() + + if opts.clean: + _RemovePreviousUpdateBranch() + + if _GetLastAuthor() == 'webrtc-version-updater': + logging.info('Last commit is a version change, skipping CL.') + return 0 + + version_filename = os.path.join(CHECKOUT_SRC_DIR, 'call', 'version.cc') + _CreateUpdateBranch() + _UpdateWebRTCVersion(version_filename) + if _IsTreeClean(): + logging.info('No WebRTC version change detected, skipping CL.') + else: + _LocalCommit() + logging.info('Uploading CL...') + _UploadCL(2) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/README b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/README new file mode 100644 index 0000000000..66bdd8a7e2 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/README @@ -0,0 +1,4 @@ +Use ../download_tools.py to download the files in this directory. Note that +they are downloaded from gs://chrome-webrtc-resources, which is a +google-internal bucket. If you're a non-Googler you'll have to download and +compile these tools manually in order to use them. diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/linux/ffmpeg.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/linux/ffmpeg.sha1 new file mode 100644 index 0000000000..4e0ba67fd1 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/linux/ffmpeg.sha1 @@ -0,0 +1 @@ +333a4e5ef11602e30efbc3081291aa7760b2bc23
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/mac/ffmpeg.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/mac/ffmpeg.sha1 new file mode 100644 index 0000000000..b8c380f35c --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/mac/ffmpeg.sha1 @@ -0,0 +1 @@ +58783bcefa695b050c23ae29fd044e1d1974d2bc
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygEMF-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygEMF-1.dll.sha1 new file mode 100644 index 0000000000..13ada8aa19 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygEMF-1.dll.sha1 @@ -0,0 +1 @@ +4dffd9a099f6e541a0d2aff162aebb4438aa8bf7
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygICE-6.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygICE-6.dll.sha1 new file mode 100644 index 0000000000..3bbd9051c7 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygICE-6.dll.sha1 @@ -0,0 +1 @@ +5381656ace3270e8ef8f707c9df288fde75cef72
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygMagick++-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygMagick++-1.dll.sha1 new file mode 100644 index 0000000000..ff934c7040 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygMagick++-1.dll.sha1 @@ -0,0 +1 @@ +120b4f60511d1e8cdd14fbade0c814feefb0c7b2
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygMagick++-5.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygMagick++-5.dll.sha1 new file mode 100644 index 0000000000..2534411105 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygMagick++-5.dll.sha1 @@ -0,0 +1 @@ +5ecd18f12adb080e1bb82cf88d33f6b450222888
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygMagickCore-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygMagickCore-1.dll.sha1 new file mode 100644 index 0000000000..34b4d738bc --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygMagickCore-1.dll.sha1 @@ -0,0 +1 @@ +c289cd5eb1130bc86bf290b2e5a636f762a1008c
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygMagickCore-5.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygMagickCore-5.dll.sha1 new file mode 100644 index 0000000000..2661f9d71e --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygMagickCore-5.dll.sha1 @@ -0,0 +1 @@ +520657343b39e45c455979cbe1871e931ec6df2d
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygMagickWand-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygMagickWand-1.dll.sha1 new file mode 100644 index 0000000000..802a7dbd2a --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygMagickWand-1.dll.sha1 @@ -0,0 +1 @@ +5b5a74b3e82b5868210ab2544f6978bf700116c1
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygMagickWand-5.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygMagickWand-5.dll.sha1 new file mode 100644 index 0000000000..bb199cd1a0 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygMagickWand-5.dll.sha1 @@ -0,0 +1 @@ +8a4a4756b6331376b0ad154d5f9e4f3260520a38
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygSM-6.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygSM-6.dll.sha1 new file mode 100644 index 0000000000..1bfe2fcae6 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygSM-6.dll.sha1 @@ -0,0 +1 @@ +b797e00e7611e52414a016edd47ee4c9e8d900e4
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygX11-6.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygX11-6.dll.sha1 new file mode 100644 index 0000000000..2f7a878ff9 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygX11-6.dll.sha1 @@ -0,0 +1 @@ +9c9f218d3cd887c9e2c9c59bbafe1261ffaf7e84
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXau-6.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXau-6.dll.sha1 new file mode 100644 index 0000000000..9948b7fbb5 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXau-6.dll.sha1 @@ -0,0 +1 @@ +2c8229b012e529bbe818a8edfd15c03e90eadf5d
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXaw-7.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXaw-7.dll.sha1 new file mode 100644 index 0000000000..3d5bb84ca2 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXaw-7.dll.sha1 @@ -0,0 +1 @@ +6fe7631a96ab1a806e1c73ce646d2ddc560be7f9
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXdmcp-6.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXdmcp-6.dll.sha1 new file mode 100644 index 0000000000..3b45026179 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXdmcp-6.dll.sha1 @@ -0,0 +1 @@ +0e6982fa3e0e9a83c7112d096ed276fab086a955
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXext-6.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXext-6.dll.sha1 new file mode 100644 index 0000000000..485d0fa6fa --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXext-6.dll.sha1 @@ -0,0 +1 @@ +e6a68be83898d69b15c66b735d6eaaa4ea6aedf0
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXft-2.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXft-2.dll.sha1 new file mode 100644 index 0000000000..b0b18c76a2 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXft-2.dll.sha1 @@ -0,0 +1 @@ +bc682da3888716c0f982db7e3bf3d46d41c736e5
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXmu-6.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXmu-6.dll.sha1 new file mode 100644 index 0000000000..8b414fedbb --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXmu-6.dll.sha1 @@ -0,0 +1 @@ +48063eb7f607c8a7c0c540faccf388966cb92f42
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXpm-4.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXpm-4.dll.sha1 new file mode 100644 index 0000000000..0e3c4099d0 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXpm-4.dll.sha1 @@ -0,0 +1 @@ +fc6213ed13d99ec14d22bfc5a9c533b67094a792
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXrender-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXrender-1.dll.sha1 new file mode 100644 index 0000000000..9b08e78109 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXrender-1.dll.sha1 @@ -0,0 +1 @@ +c6940d61bd14db07a707824edd2b67e6a1b6e1e2
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXt-6.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXt-6.dll.sha1 new file mode 100644 index 0000000000..843827f239 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygXt-6.dll.sha1 @@ -0,0 +1 @@ +a5a528a2f4398b2097af60c2070745ce229f4005
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygattr-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygattr-1.dll.sha1 new file mode 100644 index 0000000000..a04198abce --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygattr-1.dll.sha1 @@ -0,0 +1 @@ +0e84a068d46a4691d55603b99c6341ae0b049243
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygautotrace-3.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygautotrace-3.dll.sha1 new file mode 100644 index 0000000000..8c05563776 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygautotrace-3.dll.sha1 @@ -0,0 +1 @@ +6e2bb1cc1fce79ba221cfbf0ef21634327fc580a
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygbz2-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygbz2-1.dll.sha1 new file mode 100644 index 0000000000..133ba08126 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygbz2-1.dll.sha1 @@ -0,0 +1 @@ +990f8dcd0934b73cd0bd87b1482f7b532cfed071
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcairo-2.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcairo-2.dll.sha1 new file mode 100644 index 0000000000..e2f350519b --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcairo-2.dll.sha1 @@ -0,0 +1 @@ +bea314b000f02bb42d7dec160ab4b4749b6f82c5
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcairo-gobject-2.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcairo-gobject-2.dll.sha1 new file mode 100644 index 0000000000..f2b49a149a --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcairo-gobject-2.dll.sha1 @@ -0,0 +1 @@ +13cb1e859801f0008f8dca05e2edb2389db2b716
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcairo-script-interpreter-2.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcairo-script-interpreter-2.dll.sha1 new file mode 100644 index 0000000000..5e207d6483 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcairo-script-interpreter-2.dll.sha1 @@ -0,0 +1 @@ +3a743c3eb16932b770252624fa440a3d66a21cd0
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcharset-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcharset-1.dll.sha1 new file mode 100644 index 0000000000..a8c7e75ca4 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcharset-1.dll.sha1 @@ -0,0 +1 @@ +790a74405e115d386ae95e00ded534bebcd2a0fa
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcroco-0.6-3.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcroco-0.6-3.dll.sha1 new file mode 100644 index 0000000000..284863cb1d --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcroco-0.6-3.dll.sha1 @@ -0,0 +1 @@ +b6efb1d84d7296db090b0ce6a16dd74d8d05c479
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcrypt-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcrypt-0.dll.sha1 new file mode 100644 index 0000000000..d520d25caa --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcrypt-0.dll.sha1 @@ -0,0 +1 @@ +8b8d9c937ee18ebd2ea709b334e3600cea0dfbdf
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcrypto-0.9.8.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcrypto-0.9.8.dll.sha1 new file mode 100644 index 0000000000..ee7080972d --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcrypto-0.9.8.dll.sha1 @@ -0,0 +1 @@ +1abeec13689e403a4b59802aa74bedc10147d161
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcrypto-1.0.0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcrypto-1.0.0.dll.sha1 new file mode 100644 index 0000000000..947b23af64 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygcrypto-1.0.0.dll.sha1 @@ -0,0 +1 @@ +b8d5f99e6f5c87003ca1046511a1b11a6482e8da
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygdatrie-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygdatrie-1.dll.sha1 new file mode 100644 index 0000000000..e29954d6a1 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygdatrie-1.dll.sha1 @@ -0,0 +1 @@ +971fca4d171b8587ddbd036fc03a0957f864d219
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygdb-4.5.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygdb-4.5.dll.sha1 new file mode 100644 index 0000000000..b61e245e9b --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygdb-4.5.dll.sha1 @@ -0,0 +1 @@ +6e6cfc54f0e599b487c6121b4085b445666b281c
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygdb_cxx-4.5.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygdb_cxx-4.5.dll.sha1 new file mode 100644 index 0000000000..64dc86968e --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygdb_cxx-4.5.dll.sha1 @@ -0,0 +1 @@ +ec7661eb54a3b67d4bb5b3d2e97a985e86956b9c
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygdbus-1-3.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygdbus-1-3.dll.sha1 new file mode 100644 index 0000000000..831e3b0a55 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygdbus-1-3.dll.sha1 @@ -0,0 +1 @@ +3ff66e122ec05ee5920a22168e0a13cc1f1f5c94
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygexpat-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygexpat-1.dll.sha1 new file mode 100644 index 0000000000..4fb05ec2d7 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygexpat-1.dll.sha1 @@ -0,0 +1 @@ +e44623ba2a32438437335c45fb4fec9e71f1c40b
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfam-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfam-0.dll.sha1 new file mode 100644 index 0000000000..b29548f9a2 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfam-0.dll.sha1 @@ -0,0 +1 @@ +d78883226a4caf68f077bd85e86e2910c64b0a7e
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygffi-4.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygffi-4.dll.sha1 new file mode 100644 index 0000000000..ff5414d135 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygffi-4.dll.sha1 @@ -0,0 +1 @@ +82f4ad8a1ddc9bb624f98e8a3c4b366b72a62de8
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfftw3-3.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfftw3-3.dll.sha1 new file mode 100644 index 0000000000..af8972dfc7 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfftw3-3.dll.sha1 @@ -0,0 +1 @@ +2a03cf1072fe7dca391d7c60e97c1abeb114f1d3
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfftw3_threads-3.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfftw3_threads-3.dll.sha1 new file mode 100644 index 0000000000..1cf15c05e7 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfftw3_threads-3.dll.sha1 @@ -0,0 +1 @@ +0abae96305f254a75d5d501720891a3cf94aa90e
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfftw3f-3.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfftw3f-3.dll.sha1 new file mode 100644 index 0000000000..905ecdc7bc --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfftw3f-3.dll.sha1 @@ -0,0 +1 @@ +2421e456d3f3d5696f4a0e05508bdaf4845e22d1
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfftw3f_threads-3.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfftw3f_threads-3.dll.sha1 new file mode 100644 index 0000000000..9ab935b620 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfftw3f_threads-3.dll.sha1 @@ -0,0 +1 @@ +c5f59c5596e528550e18d0f49287a1009188a987
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfontconfig-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfontconfig-1.dll.sha1 new file mode 100644 index 0000000000..0b9d285bb7 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfontconfig-1.dll.sha1 @@ -0,0 +1 @@ +b57ece16d8fbcffc3093077cfbc59e27837d0a22
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygform-10.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygform-10.dll.sha1 new file mode 100644 index 0000000000..4e8a9ec8bb --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygform-10.dll.sha1 @@ -0,0 +1 @@ +ad015f1a5519f4273bf363f672517ef3f6979d1e
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygformw-10.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygformw-10.dll.sha1 new file mode 100644 index 0000000000..3672c59bca --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygformw-10.dll.sha1 @@ -0,0 +1 @@ +f916406b3c3089fdcf30704a2cbc950c8777e8ee
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfpx-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfpx-1.dll.sha1 new file mode 100644 index 0000000000..0e2460f801 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfpx-1.dll.sha1 @@ -0,0 +1 @@ +5bd5c3dc8cf76f5b51924ac791732cab9c1bfbc9
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfreetype-6.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfreetype-6.dll.sha1 new file mode 100644 index 0000000000..dbcc05e9f6 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygfreetype-6.dll.sha1 @@ -0,0 +1 @@ +c1dab7fcaf8e432da82bb54db0fe5443dcc6108e
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggcc_s-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggcc_s-1.dll.sha1 new file mode 100644 index 0000000000..c816925e0b --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggcc_s-1.dll.sha1 @@ -0,0 +1 @@ +9e1bf06cb56add8605dbc30fca7f80456ae277f7
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggd-2.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggd-2.dll.sha1 new file mode 100644 index 0000000000..5676c292ed --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggd-2.dll.sha1 @@ -0,0 +1 @@ +fb6196220213b528fde5240644092cb46f280241
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggdbm-4.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggdbm-4.dll.sha1 new file mode 100644 index 0000000000..b9993822d5 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggdbm-4.dll.sha1 @@ -0,0 +1 @@ +313d5bcff3cec41fd0ad900f9b85ad0496faa321
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggdbm_compat-4.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggdbm_compat-4.dll.sha1 new file mode 100644 index 0000000000..50daa35ddf --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggdbm_compat-4.dll.sha1 @@ -0,0 +1 @@ +25aa87318875cc0dc629bba4ef026a1ff317a3af
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggdk_pixbuf-2.0-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggdk_pixbuf-2.0-0.dll.sha1 new file mode 100644 index 0000000000..06050c62c8 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggdk_pixbuf-2.0-0.dll.sha1 @@ -0,0 +1 @@ +5219082f9f75966922b87a7af74c00b646c4727f
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggif-4.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggif-4.dll.sha1 new file mode 100644 index 0000000000..d3ea4026a2 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggif-4.dll.sha1 @@ -0,0 +1 @@ +cc7e49fc71a1fe9a0ab71abc0abd7f20e0343869
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggio-2.0-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggio-2.0-0.dll.sha1 new file mode 100644 index 0000000000..41244e160e --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggio-2.0-0.dll.sha1 @@ -0,0 +1 @@ +440585cf42662e7914b6e56e2382b02fdbba6b46
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygglib-2.0-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygglib-2.0-0.dll.sha1 new file mode 100644 index 0000000000..ce1f4c9d09 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygglib-2.0-0.dll.sha1 @@ -0,0 +1 @@ +343f7949dc69522a2e113d6d38de44d978d2faef
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggmodule-2.0-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggmodule-2.0-0.dll.sha1 new file mode 100644 index 0000000000..15de0ac2be --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggmodule-2.0-0.dll.sha1 @@ -0,0 +1 @@ +37ef0fbb18621551f330cc62fbd47b3e8d9c8f5a
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggmp-3.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggmp-3.dll.sha1 new file mode 100644 index 0000000000..e5f0b57670 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggmp-3.dll.sha1 @@ -0,0 +1 @@ +565d5ff727282f6d816bf92172922d14182ec94c
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggobject-2.0-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggobject-2.0-0.dll.sha1 new file mode 100644 index 0000000000..1d8ee64f5d --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggobject-2.0-0.dll.sha1 @@ -0,0 +1 @@ +700bc0a7f33b277879f29ddcfabc2b402610cb77
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggomp-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggomp-1.dll.sha1 new file mode 100644 index 0000000000..d37a7e14b0 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggomp-1.dll.sha1 @@ -0,0 +1 @@ +73f38ddbd12332c016df34cfcca2ba921e65dc65
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggraphite2-3.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggraphite2-3.dll.sha1 new file mode 100644 index 0000000000..e1a89dcb3f --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggraphite2-3.dll.sha1 @@ -0,0 +1 @@ +9efd795275d1f96517e5a0f244d8a7475db5cb1d
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggs-9.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggs-9.dll.sha1 new file mode 100644 index 0000000000..7b57c306ef --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggs-9.dll.sha1 @@ -0,0 +1 @@ +b625a1ae0239575c71ccda73ce45d0968104fc25
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggthread-2.0-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggthread-2.0-0.dll.sha1 new file mode 100644 index 0000000000..abe679e784 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyggthread-2.0-0.dll.sha1 @@ -0,0 +1 @@ +e92450092f0bc993ecbcd7ffded19c1c5065f576
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygharfbuzz-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygharfbuzz-0.dll.sha1 new file mode 100644 index 0000000000..1be289848c --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygharfbuzz-0.dll.sha1 @@ -0,0 +1 @@ +b85e98d97e0a5d4433944f9a61702b8c72de01a6
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyghistory7.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyghistory7.dll.sha1 new file mode 100644 index 0000000000..8152312728 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyghistory7.dll.sha1 @@ -0,0 +1 @@ +7e956b29c3dcc794b0f1c3fce846672c52dbd4e1
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicons-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicons-0.dll.sha1 new file mode 100644 index 0000000000..52d6853124 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicons-0.dll.sha1 @@ -0,0 +1 @@ +898ff4657f30ca444730bdc3bb001514c1716046
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygiconv-2.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygiconv-2.dll.sha1 new file mode 100644 index 0000000000..696104bd62 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygiconv-2.dll.sha1 @@ -0,0 +1 @@ +c120461cdd2fc30c69706cc63b27ae49bdea675c
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicudata.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicudata.dll.sha1 new file mode 100644 index 0000000000..3e9f951a92 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicudata.dll.sha1 @@ -0,0 +1 @@ +58c1f82ef478adefd0033395cda1485b9412c7b3
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicudata48.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicudata48.dll.sha1 new file mode 100644 index 0000000000..28a598c1b3 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicudata48.dll.sha1 @@ -0,0 +1 @@ +92ca111c2be38397991c44ccacc3a67cb5c6a5b8
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicui18n.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicui18n.dll.sha1 new file mode 100644 index 0000000000..b5f51a122a --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicui18n.dll.sha1 @@ -0,0 +1 @@ +2a015f94928e5b7d52958c7be9775a896e51398a
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicui18n48.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicui18n48.dll.sha1 new file mode 100644 index 0000000000..42096dfcda --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicui18n48.dll.sha1 @@ -0,0 +1 @@ +66426b6620d0fed4177c295dad2a63756f6cc593
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicuio.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicuio.dll.sha1 new file mode 100644 index 0000000000..11f5ba9d4f --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicuio.dll.sha1 @@ -0,0 +1 @@ +a3c8274ea09f70f9c01117236e123089c5163934
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicuio48.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicuio48.dll.sha1 new file mode 100644 index 0000000000..661d07bf1b --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicuio48.dll.sha1 @@ -0,0 +1 @@ +b17017f069e79f25a1aed785b5a434f225ee8dbf
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicule.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicule.dll.sha1 new file mode 100644 index 0000000000..298950f22f --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicule.dll.sha1 @@ -0,0 +1 @@ +78198238fdd2d0f0273634f5f1bf45df2d540070
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicule48.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicule48.dll.sha1 new file mode 100644 index 0000000000..31e9bde2ef --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicule48.dll.sha1 @@ -0,0 +1 @@ +9b58dc534be46a869da670d33e264f8e24f757a1
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygiculx.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygiculx.dll.sha1 new file mode 100644 index 0000000000..591068ad5e --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygiculx.dll.sha1 @@ -0,0 +1 @@ +6067e76fa0a2a0deee325780f2fb024e21d4c38f
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygiculx48.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygiculx48.dll.sha1 new file mode 100644 index 0000000000..faa04f68cd --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygiculx48.dll.sha1 @@ -0,0 +1 @@ +795e3529a036d9f91eab404d9b12f58ba8df687e
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicutest.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicutest.dll.sha1 new file mode 100644 index 0000000000..342df7e9bc --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicutest.dll.sha1 @@ -0,0 +1 @@ +220a64eaea188d884f7090137290469b4cebb8d8
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicutest48.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicutest48.dll.sha1 new file mode 100644 index 0000000000..29d49ffda0 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicutest48.dll.sha1 @@ -0,0 +1 @@ +ed1837353a5cf3f0b8df58566fb6fbadd13b32ef
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicuuc.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicuuc.dll.sha1 new file mode 100644 index 0000000000..2e0c7166cb --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicuuc.dll.sha1 @@ -0,0 +1 @@ +60c0353aec5d23b01575ae827c9c52f14bf444df
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicuuc48.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicuuc48.dll.sha1 new file mode 100644 index 0000000000..f41e477786 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygicuuc48.dll.sha1 @@ -0,0 +1 @@ +989854added78030b05bec29fb9607f3e61ca261
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygidn-11.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygidn-11.dll.sha1 new file mode 100644 index 0000000000..833c24ffde --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygidn-11.dll.sha1 @@ -0,0 +1 @@ +6809296c66c4d25f6866ea72600137a7e95a9941
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygintl-3.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygintl-3.dll.sha1 new file mode 100644 index 0000000000..99aa5042ba --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygintl-3.dll.sha1 @@ -0,0 +1 @@ +3a355271c58342a39f5c221ed28f6238cbf2f5c6
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygintl-8.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygintl-8.dll.sha1 new file mode 100644 index 0000000000..6f71b3071b --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygintl-8.dll.sha1 @@ -0,0 +1 @@ +72c33b4c94cc8a73bf374cd8a942d6c590f20be8
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygjasper-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygjasper-1.dll.sha1 new file mode 100644 index 0000000000..d643922e59 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygjasper-1.dll.sha1 @@ -0,0 +1 @@ +9a38b383ab963d5a84b23297d61dccfc42228b8f
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygjbig-2.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygjbig-2.dll.sha1 new file mode 100644 index 0000000000..6453edf68a --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygjbig-2.dll.sha1 @@ -0,0 +1 @@ +cb6362af2e9ab44f25f5cd337dbfc513b3399840
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygjpeg-7.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygjpeg-7.dll.sha1 new file mode 100644 index 0000000000..86cb1f535e --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygjpeg-7.dll.sha1 @@ -0,0 +1 @@ +ba24c6a42c9c959a72a6edbf2c86f35b4faa9684
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygjpeg-8.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygjpeg-8.dll.sha1 new file mode 100644 index 0000000000..15d38d6ed7 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygjpeg-8.dll.sha1 @@ -0,0 +1 @@ +551cc8498767757803d222160613d202905d759d
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyglcms-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyglcms-1.dll.sha1 new file mode 100644 index 0000000000..1dfea8a53a --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyglcms-1.dll.sha1 @@ -0,0 +1 @@ +5e80e346bc2e538f71216db1c4f52d7e1dbe6e66
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyglcms2-2.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyglcms2-2.dll.sha1 new file mode 100644 index 0000000000..59fd4dbf1d --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyglcms2-2.dll.sha1 @@ -0,0 +1 @@ +c8cab07b75d2cc1d303f76e1876bc1768068f042
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyglsa.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyglsa.dll.sha1 new file mode 100644 index 0000000000..a3926a3cdf --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyglsa.dll.sha1 @@ -0,0 +1 @@ +2ffabc758eaf6a2a8566294618633e0dd7322477
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyglsa64.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyglsa64.dll.sha1 new file mode 100644 index 0000000000..21e1ea72a9 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyglsa64.dll.sha1 @@ -0,0 +1 @@ +b14940b39351282ebb9f23c600c9f8d9c337833e
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygltdl-7.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygltdl-7.dll.sha1 new file mode 100644 index 0000000000..d3b3849ce8 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygltdl-7.dll.sha1 @@ -0,0 +1 @@ +f58d53e0fcfc4fc98851cce9f0aacf7d6ed1388c
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyglzma-5.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyglzma-5.dll.sha1 new file mode 100644 index 0000000000..ddf8309c56 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyglzma-5.dll.sha1 @@ -0,0 +1 @@ +65a0e994a1b3d5578e39c10fa284f4fb13bfeb88
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygmagic-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygmagic-1.dll.sha1 new file mode 100644 index 0000000000..adbd1d890a --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygmagic-1.dll.sha1 @@ -0,0 +1 @@ +c0b04a2444445fbc382467439c51cdbc1e90f009
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygmenu-10.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygmenu-10.dll.sha1 new file mode 100644 index 0000000000..499eda8c2f --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygmenu-10.dll.sha1 @@ -0,0 +1 @@ +c7a6a30626107ad1150cc1074d62a476728edfec
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygmenuw-10.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygmenuw-10.dll.sha1 new file mode 100644 index 0000000000..08f2e291ca --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygmenuw-10.dll.sha1 @@ -0,0 +1 @@ +9e792b1c8fd0fdf992a1f06c361123ad1833f59d
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygming-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygming-1.dll.sha1 new file mode 100644 index 0000000000..2d3c80a9a2 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygming-1.dll.sha1 @@ -0,0 +1 @@ +3f20881c17b0d87a150e148b432c889ffa596b74
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygmp-3.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygmp-3.dll.sha1 new file mode 100644 index 0000000000..00572bff6e --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygmp-3.dll.sha1 @@ -0,0 +1 @@ +94ea4af89faadd221223b25fe472356bbc8c4064
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygncurses++-10.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygncurses++-10.dll.sha1 new file mode 100644 index 0000000000..687cac40ab --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygncurses++-10.dll.sha1 @@ -0,0 +1 @@ +21a8fcef8c6d66769ea8c019294e27c959135df0
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygncurses++w-10.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygncurses++w-10.dll.sha1 new file mode 100644 index 0000000000..51b5fac510 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygncurses++w-10.dll.sha1 @@ -0,0 +1 @@ +fb5fea774f9e78f7ddc29a516e58f7afc45a113e
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygncurses-10.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygncurses-10.dll.sha1 new file mode 100644 index 0000000000..b5eba51d7d --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygncurses-10.dll.sha1 @@ -0,0 +1 @@ +5bc703d14e864e9befd0fd5f44272592d050475a
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygncursesw-10.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygncursesw-10.dll.sha1 new file mode 100644 index 0000000000..57087fd324 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygncursesw-10.dll.sha1 @@ -0,0 +1 @@ +78b1afe0c9041cebdd4ed1e56d2bf4c8fbecd921
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpanel-10.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpanel-10.dll.sha1 new file mode 100644 index 0000000000..c14fe85b93 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpanel-10.dll.sha1 @@ -0,0 +1 @@ +8fb3eeff239da3e9e43dd76775dc0afee518724f
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpanelw-10.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpanelw-10.dll.sha1 new file mode 100644 index 0000000000..47452ceec8 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpanelw-10.dll.sha1 @@ -0,0 +1 @@ +9ba395e919eb4c8ad559164ad3b71cd3841df8fa
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpango-1.0-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpango-1.0-0.dll.sha1 new file mode 100644 index 0000000000..ecb841237b --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpango-1.0-0.dll.sha1 @@ -0,0 +1 @@ +4d05a789b8b9e6c05a0e1d315bc7afd7ec015301
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpangocairo-1.0-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpangocairo-1.0-0.dll.sha1 new file mode 100644 index 0000000000..b5adeeb269 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpangocairo-1.0-0.dll.sha1 @@ -0,0 +1 @@ +b6753663b06ca5751c9299dab1309860dbbc83ee
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpangoft2-1.0-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpangoft2-1.0-0.dll.sha1 new file mode 100644 index 0000000000..0e0480703e --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpangoft2-1.0-0.dll.sha1 @@ -0,0 +1 @@ +4ca6f24c58a45bdbc832e590a16b8aa860b165d7
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpangoxft-1.0-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpangoxft-1.0-0.dll.sha1 new file mode 100644 index 0000000000..38c5482d5a --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpangoxft-1.0-0.dll.sha1 @@ -0,0 +1 @@ +41ceee24e37c1e608e4eddac1b7ebc35c6e0697e
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpaper-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpaper-1.dll.sha1 new file mode 100644 index 0000000000..be53175bc4 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpaper-1.dll.sha1 @@ -0,0 +1 @@ +edebc8b3e83ea9c855e6c8a735e2e0ea0f3467e9
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpcre-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpcre-0.dll.sha1 new file mode 100644 index 0000000000..b25d1880df --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpcre-0.dll.sha1 @@ -0,0 +1 @@ +19de1650d85398b71046329d30d47934a9074f03
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpcre-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpcre-1.dll.sha1 new file mode 100644 index 0000000000..2412050ac5 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpcre-1.dll.sha1 @@ -0,0 +1 @@ +28e70bf479bfd46136ab90b108c8da025989ec8d
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygperl5_14.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygperl5_14.dll.sha1 new file mode 100644 index 0000000000..454fd58b7b --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygperl5_14.dll.sha1 @@ -0,0 +1 @@ +443430fba095f486d8a5aaa143b16f874581449d
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpixman-1-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpixman-1-0.dll.sha1 new file mode 100644 index 0000000000..846e6dc191 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpixman-1-0.dll.sha1 @@ -0,0 +1 @@ +0460c239ee07cd48ada93d5b255abd4d2450227d
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygplotter-2.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygplotter-2.dll.sha1 new file mode 100644 index 0000000000..a727a66441 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygplotter-2.dll.sha1 @@ -0,0 +1 @@ +633f96712f6f3a1f63ffbb30dc6d54927ecc9625
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpng12.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpng12.dll.sha1 new file mode 100644 index 0000000000..f18d7ecb3c --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpng12.dll.sha1 @@ -0,0 +1 @@ +b2885ee84dd9a19ef2ae94e81edc004463faf98c
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpng14-14.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpng14-14.dll.sha1 new file mode 100644 index 0000000000..e8a573f9f4 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpng14-14.dll.sha1 @@ -0,0 +1 @@ +9f916788fb5702dd71724b21a28f65f484acc1b5
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpng15-15.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpng15-15.dll.sha1 new file mode 100644 index 0000000000..6d16da6b71 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpng15-15.dll.sha1 @@ -0,0 +1 @@ +f270fc7cc289432d8986525e4086cd593e69f5df
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpopt-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpopt-0.dll.sha1 new file mode 100644 index 0000000000..28ba62bc14 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpopt-0.dll.sha1 @@ -0,0 +1 @@ +f0dd911bd052ac86b77ac416423a91de881374a9
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpstoedit-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpstoedit-0.dll.sha1 new file mode 100644 index 0000000000..9ccc62ac12 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygpstoedit-0.dll.sha1 @@ -0,0 +1 @@ +8769e9cade95b51bf031921dbf538af7b7ed9616
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygreadline7.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygreadline7.dll.sha1 new file mode 100644 index 0000000000..af803366b4 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygreadline7.dll.sha1 @@ -0,0 +1 @@ +16ba39c158758803af249e2ba1f82ac41fc4c8e8
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygrsvg-2-2.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygrsvg-2-2.dll.sha1 new file mode 100644 index 0000000000..075efdee01 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygrsvg-2-2.dll.sha1 @@ -0,0 +1 @@ +f31b442d60a6254df74fb1c50eecc7764da70497
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygsigsegv-2.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygsigsegv-2.dll.sha1 new file mode 100644 index 0000000000..2b3132218b --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygsigsegv-2.dll.sha1 @@ -0,0 +1 @@ +de1a8754327d24d9691c0da1f85ab97e305bdf39
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygsqlite3-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygsqlite3-0.dll.sha1 new file mode 100644 index 0000000000..4f1d359659 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygsqlite3-0.dll.sha1 @@ -0,0 +1 @@ +ee3488768cc77446bc3314547e1a7f5b442c5df4
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygssl-0.9.8.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygssl-0.9.8.dll.sha1 new file mode 100644 index 0000000000..d1c0bd29ae --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygssl-0.9.8.dll.sha1 @@ -0,0 +1 @@ +33d0635a7480df65371ab93e5cf281570b50fae5
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygssl-1.0.0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygssl-1.0.0.dll.sha1 new file mode 100644 index 0000000000..119faf16e5 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygssl-1.0.0.dll.sha1 @@ -0,0 +1 @@ +24abaccaa15508bb26a87e8480d8cc9bb3fdaece
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygssp-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygssp-0.dll.sha1 new file mode 100644 index 0000000000..71213ae3cf --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygssp-0.dll.sha1 @@ -0,0 +1 @@ +f093531a544f830d9577220bb3a1bf5bd8df2ec2
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygstdc++-6.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygstdc++-6.dll.sha1 new file mode 100644 index 0000000000..24307b447c --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygstdc++-6.dll.sha1 @@ -0,0 +1 @@ +e6b44b19242a011df419b09b64c46597d4378b90
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygthai-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygthai-0.dll.sha1 new file mode 100644 index 0000000000..5b49fd1938 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygthai-0.dll.sha1 @@ -0,0 +1 @@ +f11ff45f289b8d36d624d4d53b51859986627675
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygtic-10.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygtic-10.dll.sha1 new file mode 100644 index 0000000000..89792cb7d4 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygtic-10.dll.sha1 @@ -0,0 +1 @@ +7b16da6042a554d4fdd7909b6093d192ab9a2aa3
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygticw-10.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygticw-10.dll.sha1 new file mode 100644 index 0000000000..94115f245e --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygticw-10.dll.sha1 @@ -0,0 +1 @@ +61017a0d8a09fec100c631b2b40bd6d935954769
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygtiff-5.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygtiff-5.dll.sha1 new file mode 100644 index 0000000000..f718575d13 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygtiff-5.dll.sha1 @@ -0,0 +1 @@ +7b7dcf0a48a1f2da4baa5fbef95160bde900bd53
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygtiffxx-5.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygtiffxx-5.dll.sha1 new file mode 100644 index 0000000000..ceace63d3a --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygtiffxx-5.dll.sha1 @@ -0,0 +1 @@ +7587f424284e82e775d99ea05fe0394bd7fca030
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyguuid-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyguuid-1.dll.sha1 new file mode 100644 index 0000000000..763a5799ef --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cyguuid-1.dll.sha1 @@ -0,0 +1 @@ +b418145c0635c4d9c3d46d8a0334fed16fa41b07
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygwin1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygwin1.dll.sha1 new file mode 100644 index 0000000000..85adb7c200 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygwin1.dll.sha1 @@ -0,0 +1 @@ +2124fabeb6adc54fe7d24f6055847a5d7a87b369
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygxcb-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygxcb-1.dll.sha1 new file mode 100644 index 0000000000..070811748c --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygxcb-1.dll.sha1 @@ -0,0 +1 @@ +17d2046d7d7720f9938c1de6ed805aaff1a6481c
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygxcb-render-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygxcb-render-0.dll.sha1 new file mode 100644 index 0000000000..590881f7c4 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygxcb-render-0.dll.sha1 @@ -0,0 +1 @@ +b482081ef69175e97807af097caa2fb13f68d87b
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygxcb-shm-0.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygxcb-shm-0.dll.sha1 new file mode 100644 index 0000000000..72ca105219 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygxcb-shm-0.dll.sha1 @@ -0,0 +1 @@ +acdb0328ec4b29efef9da66acb2b78259acdf4a7
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygxml2-2.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygxml2-2.dll.sha1 new file mode 100644 index 0000000000..8a852d8af4 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygxml2-2.dll.sha1 @@ -0,0 +1 @@ +d5b32c9640c210b211c1247aa1d0c8e09f638c2f
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygz.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygz.dll.sha1 new file mode 100644 index 0000000000..8adc962ab3 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/cygz.dll.sha1 @@ -0,0 +1 @@ +c9fb7d74bc0c53085bae99a5f39310984bb0fd06
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/ffmpeg.exe.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/ffmpeg.exe.sha1 new file mode 100644 index 0000000000..872aaee618 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/ffmpeg.exe.sha1 @@ -0,0 +1 @@ +e525fa4ed35e292567e756d8455310d7a2db2389
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/libgomp-1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/libgomp-1.dll.sha1 new file mode 100644 index 0000000000..af0dc4172c --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/libgomp-1.dll.sha1 @@ -0,0 +1 @@ +5be80ff707f97c54780c1aec1a853f61ffb3fe8c
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/pthreadgc2.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/pthreadgc2.dll.sha1 new file mode 100644 index 0000000000..ea6e3da47b --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/pthreadgc2.dll.sha1 @@ -0,0 +1 @@ +02c3ddd6022bbfb335e28776fcb3f45f6b9535db
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/zlib1.dll.sha1 b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/zlib1.dll.sha1 new file mode 100644 index 0000000000..b1f7bd9e4b --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/video_quality_toolchain/win/zlib1.dll.sha1 @@ -0,0 +1 @@ +e590ab0d2c81e98f5ec2baf671afa350352aa5b1
\ No newline at end of file diff --git a/third_party/libwebrtc/tools_webrtc/vim/webrtc.ycm_extra_conf.py b/third_party/libwebrtc/tools_webrtc/vim/webrtc.ycm_extra_conf.py new file mode 100644 index 0000000000..d11f79ae24 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/vim/webrtc.ycm_extra_conf.py @@ -0,0 +1,356 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +# Autocompletion config for YouCompleteMe in WebRTC. This is just copied from +# tools/vim in chromium with very minor modifications. +# +# USAGE: +# +# 1. Install YCM [https://github.com/Valloric/YouCompleteMe] +# (Googlers should check out [go/ycm]) +# +# 2. Create a symbolic link to this file called .ycm_extra_conf.py in the +# directory above your WebRTC checkout (i.e. next to your .gclient file). +# +# cd src +# ln -rs tools_webrtc/vim/webrtc.ycm_extra_conf.py \ +# ../.ycm_extra_conf.py +# +# 3. (optional) Whitelist the .ycm_extra_conf.py from step #2 by adding the +# following to your .vimrc: +# +# let g:ycm_extra_conf_globlist=['<path to .ycm_extra_conf.py>'] +# +# You can also add other .ycm_extra_conf.py files you want to use to this +# list to prevent excessive prompting each time you visit a directory +# covered by a config file. +# +# 4. Profit +# +# +# Usage notes: +# +# * You must use ninja & clang to build WebRTC. +# +# * You must have run "gn gen" and built WebRTC recently. +# +# +# Hacking notes: +# +# * The purpose of this script is to construct an accurate enough command line +# for YCM to pass to clang so it can build and extract the symbols. +# +# * Right now, we only pull the -I and -D flags. That seems to be sufficient +# for everything I've used it for. +# +# * That whole ninja & clang thing? We could support other configs if someone +# were willing to write the correct commands and a parser. +# +# * This has only been tested on gPrecise. + +import os +import os.path +import shlex +import subprocess +import sys + +# Flags from YCM's default config. +_DEFAULT_FLAGS = [ + '-DUSE_CLANG_COMPLETER', + '-std=c++11', + '-x', + 'c++', +] + +_HEADER_ALTERNATES = ('.cc', '.cpp', '.c', '.mm', '.m') + +_EXTENSION_FLAGS = { + '.m': ['-x', 'objective-c'], + '.mm': ['-x', 'objective-c++'], +} + + +def FindWebrtcSrcFromFilename(filename): + """Searches for the root of the WebRTC checkout. + + Simply checks parent directories until it finds .gclient and src/. + + Args: + filename: (String) Path to source file being edited. + + Returns: + (String) Path of 'src/', or None if unable to find. + """ + curdir = os.path.normpath(os.path.dirname(filename)) + while not (os.path.basename(curdir) == 'src' + and os.path.exists(os.path.join(curdir, 'DEPS')) and + (os.path.exists(os.path.join(curdir, '..', '.gclient')) + or os.path.exists(os.path.join(curdir, '.git')))): + nextdir = os.path.normpath(os.path.join(curdir, '..')) + if nextdir == curdir: + return None + curdir = nextdir + return curdir + + +def GetDefaultSourceFile(webrtc_root, filename): + """Returns the default source file to use as an alternative to `filename`. + + Compile flags used to build the default source file is assumed to be a + close-enough approximation for building `filename`. + + Args: + webrtc_root: (String) Absolute path to the root of WebRTC checkout. + filename: (String) Absolute path to the source file. + + Returns: + (String) Absolute path to substitute source file. + """ + if 'test.' in filename: + return os.path.join(webrtc_root, 'base', 'logging_unittest.cc') + return os.path.join(webrtc_root, 'base', 'logging.cc') + + +def GetNinjaBuildOutputsForSourceFile(out_dir, filename): + """Returns a list of build outputs for filename. + + The list is generated by invoking 'ninja -t query' tool to retrieve a list of + inputs and outputs of `filename`. This list is then filtered to only include + .o and .obj outputs. + + Args: + out_dir: (String) Absolute path to ninja build output directory. + filename: (String) Absolute path to source file. + + Returns: + (List of Strings) List of target names. Will return [] if `filename` doesn't + yield any .o or .obj outputs. + """ + # Ninja needs the path to the source file relative to the output build + # directory. + rel_filename = os.path.relpath(filename, out_dir) + + p = subprocess.Popen(['ninja', '-C', out_dir, '-t', 'query', rel_filename], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) + stdout, _ = p.communicate() + if p.returncode != 0: + return [] + + # The output looks like: + # ../../relative/path/to/source.cc: + # outputs: + # obj/reative/path/to/target.source.o + # obj/some/other/target2.source.o + # another/target.txt + # + outputs_text = stdout.partition('\n outputs:\n')[2] + output_lines = [line.strip() for line in outputs_text.split('\n')] + return [ + target for target in output_lines + if target and (target.endswith('.o') or target.endswith('.obj')) + ] + + +def GetClangCommandLineForNinjaOutput(out_dir, build_target): + """Returns the Clang command line for building `build_target` + + Asks ninja for the list of commands used to build `filename` and returns the + final Clang invocation. + + Args: + out_dir: (String) Absolute path to ninja build output directory. + build_target: (String) A build target understood by ninja + + Returns: + (String or None) Clang command line or None if a Clang command line couldn't + be determined. + """ + p = subprocess.Popen( + ['ninja', '-v', '-C', out_dir, '-t', 'commands', build_target], + stdout=subprocess.PIPE, + universal_newlines=True) + stdout, _ = p.communicate() + if p.returncode != 0: + return None + + # Ninja will return multiple build steps for all dependencies up to + # `build_target`. The build step we want is the last Clang invocation, which + # is expected to be the one that outputs `build_target`. + for line in reversed(stdout.split('\n')): + if 'clang' in line: + return line + return None + + +def GetClangCommandLineFromNinjaForSource(out_dir, filename): + """Returns a Clang command line used to build `filename`. + + The same source file could be built multiple times using different tool + chains. In such cases, this command returns the first Clang invocation. We + currently don't prefer one toolchain over another. Hopefully the tool chain + corresponding to the Clang command line is compatible with the Clang build + used by YCM. + + Args: + out_dir: (String) Absolute path to WebRTC checkout. + filename: (String) Absolute path to source file. + + Returns: + (String or None): Command line for Clang invocation using `filename` as a + source. Returns None if no such command line could be found. + """ + build_targets = GetNinjaBuildOutputsForSourceFile(out_dir, filename) + for build_target in build_targets: + command_line = GetClangCommandLineForNinjaOutput(out_dir, build_target) + if command_line: + return command_line + return None + + +def GetClangOptionsFromCommandLine(clang_commandline, out_dir, + additional_flags): + """Extracts relevant command line options from `clang_commandline` + + Args: + clang_commandline: (String) Full Clang invocation. + out_dir: (String) Absolute path to ninja build directory. Relative paths in + the command line are relative to `out_dir`. + additional_flags: (List of String) Additional flags to return. + + Returns: + (List of Strings) The list of command line flags for this source file. Can + be empty. + """ + clang_flags = [] + additional_flags + + # Parse flags that are important for YCM's purposes. + clang_tokens = shlex.split(clang_commandline) + for flag_index, flag in enumerate(clang_tokens): + if flag.startswith('-I'): + # Relative paths need to be resolved, because they're relative to + # the output dir, not the source. + if flag[2] == '/': + clang_flags.append(flag) + else: + abs_path = os.path.normpath(os.path.join(out_dir, flag[2:])) + clang_flags.append('-I' + abs_path) + elif flag.startswith('-std'): + clang_flags.append(flag) + elif flag.startswith('-') and flag[1] in 'DWFfmO': + if flag in ['-Wno-deprecated-register', '-Wno-header-guard']: + # These flags causes libclang (3.3) to crash. Remove it until + # things are fixed. + continue + clang_flags.append(flag) + elif flag == '-isysroot': + # On Mac -isysroot <path> is used to find the system headers. + # Copy over both flags. + if flag_index + 1 < len(clang_tokens): + clang_flags.append(flag) + clang_flags.append(clang_tokens[flag_index + 1]) + elif flag.startswith('--sysroot='): + # On Linux we use a sysroot image. + sysroot_path = flag.lstrip('--sysroot=') + if sysroot_path.startswith('/'): + clang_flags.append(flag) + else: + abs_path = os.path.normpath(os.path.join(out_dir, sysroot_path)) + clang_flags.append('--sysroot=' + abs_path) + return clang_flags + + +def GetClangOptionsFromNinjaForFilename(webrtc_root, filename): + """Returns the Clang command line options needed for building `filename`. + + Command line options are based on the command used by ninja for building + `filename`. If `filename` is a .h file, uses its companion .cc or .cpp file. + If a suitable companion file can't be located or if ninja doesn't know about + `filename`, then uses default source files in WebRTC for determining the + commandline. + + Args: + webrtc_root: (String) Path to src/. + filename: (String) Absolute path to source file being edited. + + Returns: + (List of Strings) The list of command line flags for this source file. Can + be empty. + """ + if not webrtc_root: + return [] + + # Generally, everyone benefits from including WebRTC's src/, because all of + # WebRTC's includes are relative to that. + additional_flags = ['-I' + os.path.join(webrtc_root)] + + # Version of Clang used to compile WebRTC can be newer then version of + # libclang that YCM uses for completion. So it's possible that YCM's + # libclang doesn't know about some used warning options, which causes + # compilation warnings (and errors, because of '-Werror'); + additional_flags.append('-Wno-unknown-warning-option') + + sys.path.append(os.path.join(webrtc_root, 'tools', 'vim')) + from ninja_output import GetNinjaOutputDirectory + out_dir = GetNinjaOutputDirectory(webrtc_root) + + basename, extension = os.path.splitext(filename) + if extension == '.h': + candidates = [basename + ext for ext in _HEADER_ALTERNATES] + else: + candidates = [filename] + + clang_line = None + buildable_extension = extension + for candidate in candidates: + clang_line = GetClangCommandLineFromNinjaForSource(out_dir, candidate) + if clang_line: + buildable_extension = os.path.splitext(candidate)[1] + break + + additional_flags += _EXTENSION_FLAGS.get(buildable_extension, []) + + if not clang_line: + # If ninja didn't know about filename or it's companion files, then try + # a default build target. It is possible that the file is new, or + # build.ninja is stale. + clang_line = GetClangCommandLineFromNinjaForSource( + out_dir, GetDefaultSourceFile(webrtc_root, filename)) + + if not clang_line: + return additional_flags + + return GetClangOptionsFromCommandLine(clang_line, out_dir, additional_flags) + + +def FlagsForFile(filename): + """This is the main entry point for YCM. Its interface is fixed. + + Args: + filename: (String) Path to source file being edited. + + Returns: + (Dictionary) + 'flags': (List of Strings) Command line flags. + 'do_cache': (Boolean) True if the result should be cached. + """ + abs_filename = os.path.abspath(filename) + webrtc_root = FindWebrtcSrcFromFilename(abs_filename) + clang_flags = GetClangOptionsFromNinjaForFilename(webrtc_root, abs_filename) + + # If clang_flags could not be determined, then assume that was due to a + # transient failure. Preventing YCM from caching the flags allows us to + # try to determine the flags again. + should_cache_flags_for_file = bool(clang_flags) + + final_flags = _DEFAULT_FLAGS + clang_flags + + return {'flags': final_flags, 'do_cache': should_cache_flags_for_file} diff --git a/third_party/libwebrtc/tools_webrtc/whitespace.txt b/third_party/libwebrtc/tools_webrtc/whitespace.txt new file mode 100644 index 0000000000..e930e82ad2 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/whitespace.txt @@ -0,0 +1,19 @@ +You can modify this file to create no-op changelists. + +Try to write something funny. And please don't add trailing whitespace. +Whitespaces are cool, except when they're trailing. Then they're kind of +80s. +Here's a funny story: What do you call a root beer in a square glass? Beer! +Holidays are coming, beware! Bought any gifts yet? + +Lemur was here :) +Kjellander as well :-o +mbonadei as well. +jansson is here now :) + +Foo Bar Baz Bur + +Alios ego vidi ventos; alias prospexi animo procellas +I have seen other winds; I looked out of my mind at other storms + - Cicero +Lahiru modifiying the line numbber 17Lahiru modifiying the line numbber 17 |