diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
| commit | 26a029d407be480d791972afb5975cf62c9360a6 (patch) | |
| tree | f435a8308119effd964b339f76abb83a57c29483 /dom/media/webrtc/third_party_build | |
| parent | Initial commit. (diff) | |
| download | firefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz firefox-26a029d407be480d791972afb5975cf62c9360a6.zip | |
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'dom/media/webrtc/third_party_build')
32 files changed, 3970 insertions, 0 deletions
diff --git a/dom/media/webrtc/third_party_build/README.md b/dom/media/webrtc/third_party_build/README.md new file mode 100644 index 0000000000..9151445c87 --- /dev/null +++ b/dom/media/webrtc/third_party_build/README.md @@ -0,0 +1,17 @@ +# Vendoring libwebrtc and the fast-forward process + +Most of the important information about this process is contained on the fast-forward +automation wiki page +[here](https://wiki.mozilla.org/Media/WebRTC/libwebrtc_Update_Process/automation_plan). + +To skip the history and details and go directly to starting the libwebrtc fast-foward +process, go to the +[Operation Checklist](https://wiki.mozilla.org/Media/WebRTC/libwebrtc_Update_Process/automation_plan#Operation_Checklist). + +# Fixing errors reported in scripts + +In most cases, the scripts report errors including suggestions on how to resolve the +issue. If you're seeing an error message referring you to this README.md file, the +likely issue is that you're missing environment variables that should be set in a +config_env file in .moz-fast-forward. The default for that file can be found at +dom/media/webrtc/third_party_build/default_config_env. diff --git a/dom/media/webrtc/third_party_build/build_no_op_commits.sh b/dom/media/webrtc/third_party_build/build_no_op_commits.sh new file mode 100644 index 0000000000..c7fb6d95e3 --- /dev/null +++ b/dom/media/webrtc/third_party_build/build_no_op_commits.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +function show_error_msg() +{ + echo "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +source dom/media/webrtc/third_party_build/use_config_env.sh + +echo "MOZ_LIBWEBRTC_SRC: $MOZ_LIBWEBRTC_SRC" + +# After this point: +# * eE: All commands should succeed. +# * u: All variables should be defined before use. +# * o pipefail: All stages of all pipes should succeed. +set -eEuo pipefail + +CURRENT_DIR=`pwd` +cd $MOZ_LIBWEBRTC_SRC + +MANUAL_INTERVENTION_COMMIT_FILE="$TMP_DIR/manual_commits.txt" +rm -f $MANUAL_INTERVENTION_COMMIT_FILE + +# Find the common commit between our previous work branch and trunk +CURRENT_RELEASE_BASE=`git merge-base branch-heads/$MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM master` + +# Write no-op files for the cherry-picked release branch commits. For more +# details on what this is doing, see make_upstream_revert_noop.sh. +COMMITS=`git log -r $CURRENT_RELEASE_BASE..branch-heads/$MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM --format='%h'` +for commit in $COMMITS; do + + echo "Processing release branch commit $commit for no-op handling" + + # Don't process the commit if the commit message is missing the customary + # line that shows which upstream commit is being cherry-picked. + CNT=`git show $commit | grep "cherry picked from commit" | wc -l | tr -d " " || true` + if [ $CNT != 1 ]; then + # record the commit to list at the end of this script as + # 'needing intervention' + echo " no cherry-pick info found, skipping commit $commit" + echo "$commit" >> $MANUAL_INTERVENTION_COMMIT_FILE + continue + fi + + CHERRY_PICK_COMMIT=`git show $commit | grep "cherry picked from commit" | tr -d "()" | awk '{ print $5; }'` + SHORT_SHA=`git show --name-only $CHERRY_PICK_COMMIT --format='%h' | head -1` + echo " commit $commit cherry-picks $SHORT_SHA" + + echo "We already cherry-picked this when we vendored $commit." \ + > $STATE_DIR/$SHORT_SHA.no-op-cherry-pick-msg + +done + +# This section checks for commits that may have been cherry-picked in more +# than one release branch. +TARGET_RELEASE_BASE=`git merge-base $MOZ_TARGET_UPSTREAM_BRANCH_HEAD master` +NEW_COMMITS=`git log -r $TARGET_RELEASE_BASE..$MOZ_TARGET_UPSTREAM_BRANCH_HEAD --format='%h'` + +# Convert the files that we've already generated for no-op detection into +# something that we can use as a regular expression for searching. +KNOWN_NO_OP_COMMITS=`cd $STATE_DIR ; \ + ls *.no-op-cherry-pick-msg \ + | sed 's/\.no-op-cherry-pick-msg//' \ + | paste -sd '|' /dev/stdin` + +for commit in $NEW_COMMITS; do + + echo "Processing next release branch commit $commit for no-op handling" + + # Don't process the commit if the commit message is missing the customary + # line that shows which upstream commit is being cherry-picked. + CNT=`git show $commit | grep "cherry picked from commit" | wc -l | tr -d " " || true` + if [ $CNT != 1 ]; then + # record the commit to list at the end of this script as + # 'needing intervention' + echo " no cherry-pick info found, skipping commit $commit" + echo "$commit" >> $MANUAL_INTERVENTION_COMMIT_FILE + continue + fi + + CHERRY_PICK_COMMIT=`git show $commit | grep "cherry picked from commit" | tr -d "()" | awk '{ print $5; }'` + SHORT_SHA=`git show --name-only $CHERRY_PICK_COMMIT --format='%h' | head -1` + + # The trick here is that we only want to include no-op processing for the + # commits that appear both here _and_ in the previous release's cherry-pick + # commits. We check the known list of no-op commits to see if it was + # cherry picked in the previous release branch and then create another + # file for the new release branch commit that will ultimately be a no-op. + if [[ "$SHORT_SHA" =~ ^($KNOWN_NO_OP_COMMITS)$ ]]; then + echo " commit $commit cherry-picks $SHORT_SHA" + cp $STATE_DIR/$SHORT_SHA.no-op-cherry-pick-msg $STATE_DIR/$commit.no-op-cherry-pick-msg + fi + +done + +if [ ! -f $MANUAL_INTERVENTION_COMMIT_FILE ]; then + echo "No commits require manual intervention" + exit +fi + +echo $" +Each of the following commits requires manual intervention to +verify the source of the cherry-pick or there may be errors +reported during the fast-forward processing. Without this +intervention, the common symptom is that the vendored commit +file count (0) will not match the upstream commit file count. +" + +for commit in `cat $MANUAL_INTERVENTION_COMMIT_FILE`; do + SUMMARY=`git show --oneline --name-only $commit | head -1` + echo " '$SUMMARY'" +done + +echo $" +To manually create the no-op tracking files needed, +run the following command (in bash) for each commit in question: + ( export FUTURE_UPSTREAM_COMMIT=\"{short-sha-of-upstream-commit}\" ; \\ + export ALREADY_USED_COMMIT=\"{short-sha-of-already-used-commit}\" ; \\ + echo \"We already cherry-picked this when we vendored \$ALREADY_USED_COMMIT.\" \\ + > $STATE_DIR/\$FUTURE_UPSTREAM_COMMIT.no-op-cherry-pick-msg ) +" diff --git a/dom/media/webrtc/third_party_build/cherry_pick_commit.py b/dom/media/webrtc/third_party_build/cherry_pick_commit.py new file mode 100644 index 0000000000..5aa6c9e6a6 --- /dev/null +++ b/dom/media/webrtc/third_party_build/cherry_pick_commit.py @@ -0,0 +1,420 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import atexit +import os +import re +import shutil +import sys + +from filter_git_changes import filter_git_changes +from restore_patch_stack import restore_patch_stack +from run_operations import ( + get_last_line, + run_git, + run_hg, + run_shell, + update_resume_state, +) +from vendor_and_commit import vendor_and_commit + +# This script cherry-picks an upstream commit with the appropriate +# commit message, and adds the no-op commit tracking file for the when +# we vendor the upstream commit later. + +error_help = None +script_name = os.path.basename(__file__) + + +def early_exit_handler(): + print(f"*** ERROR *** {script_name} did not complete successfully") + if error_help is not None: + print(error_help) + + +def write_commit_message_file( + commit_message_filename, + github_path, + github_sha, + bug_number, + reviewers, +): + print(f"commit_message_filename: {commit_message_filename}") + print(f"github_path: {github_path}") + print(f"github_sha: {github_sha}") + print(f"bug_number: {bug_number}") + + cmd = "git show --format=%H --no-patch {}".format(github_sha) + stdout_lines = run_git(cmd, github_path) + github_long_sha = stdout_lines[0] + print(f"github_long_sha: {github_long_sha}") + + cmd = "git show --format=%s%n%n%b --no-patch {}".format(github_sha) + github_commit_msg_lines = run_git(cmd, github_path) + + with open(commit_message_filename, "w") as ofile: + ofile.write( + "Bug {} - Cherry-pick upstream libwebrtc commit {} r?{}".format( + bug_number, github_sha, reviewers + ) + ) + ofile.write("\n") + ofile.write("\n") + ofile.write( + "Upstream commit: https://webrtc.googlesource.com/src/+/{}".format( + github_long_sha + ) + ) + ofile.write("\n") + for line in github_commit_msg_lines: + ofile.write(" {}".format(line)) + ofile.write("\n") + + +def cherry_pick_commit( + commit_message_filename, + github_path, + github_sha, +): + print(f"commit_message_filename: {commit_message_filename}") + print(f"github_path: {github_path}") + print(f"github_sha: {github_sha}") + + cmd = "git cherry-pick --no-commit {}".format(github_sha) + run_git(cmd, github_path) + + cmd = "git commit --file {}".format(os.path.abspath(commit_message_filename)) + run_git(cmd, github_path) + + +def write_noop_tracking_file( + github_sha, + bug_number, +): + noop_basename = "{}.no-op-cherry-pick-msg".format(github_sha) + noop_filename = os.path.join(args.state_path, noop_basename) + print(f"noop_filename: {noop_filename}") + with open(noop_filename, "w") as ofile: + ofile.write("We cherry-picked this in bug {}".format(bug_number)) + ofile.write("\n") + shutil.copy(noop_filename, args.patch_path) + cmd = "hg add {}".format(os.path.join(args.patch_path, noop_basename)) + run_hg(cmd) + cmd = "hg amend {}".format(os.path.join(args.patch_path, noop_basename)) + run_hg(cmd) + + +if __name__ == "__main__": + default_target_dir = "third_party/libwebrtc" + default_state_dir = ".moz-fast-forward" + default_log_dir = ".moz-fast-forward/logs" + default_tmp_dir = ".moz-fast-forward/tmp" + default_script_dir = "dom/media/webrtc/third_party_build" + default_patch_dir = "third_party/libwebrtc/moz-patch-stack" + default_repo_dir = ".moz-fast-forward/moz-libwebrtc" + default_tar_name = "moz-libwebrtc.tar.gz" + + parser = argparse.ArgumentParser( + description="Cherry-pick upstream libwebrtc commit" + ) + parser.add_argument( + "--target-path", + default=default_target_dir, + help="target path for vendoring (defaults to {})".format(default_target_dir), + ) + parser.add_argument( + "--state-path", + default=default_state_dir, + help="path to state directory (defaults to {})".format(default_state_dir), + ) + parser.add_argument( + "--log-path", + default=default_log_dir, + help="path to log directory (defaults to {})".format(default_log_dir), + ) + parser.add_argument( + "--tmp-path", + default=default_tmp_dir, + help="path to tmp directory (defaults to {})".format(default_tmp_dir), + ) + parser.add_argument( + "--script-path", + default=default_script_dir, + help="path to script directory (defaults to {})".format(default_script_dir), + ) + parser.add_argument( + "--repo-path", + default=default_repo_dir, + help="path to moz-libwebrtc repo (defaults to {})".format(default_repo_dir), + ) + parser.add_argument( + "--tar-name", + default=default_tar_name, + help="name of tar file (defaults to {})".format(default_tar_name), + ) + parser.add_argument( + "--commit-sha", + required=True, + help="sha of commit to examine", + ) + parser.add_argument( + "--branch", + default="mozpatches", + help="moz-libwebrtc branch (defaults to mozpatches)", + ) + parser.add_argument( + "--commit-bug-number", + type=int, + required=True, + help="integer Bugzilla number (example: 1800920)", + ) + parser.add_argument( + "--patch-path", + default=default_patch_dir, + help="path to save patches (defaults to {})".format(default_patch_dir), + ) + parser.add_argument( + "--reviewers", + required=True, + help='reviewers for cherry-picked patch (like "ng,mjf")', + ) + parser.add_argument( + "--abort", + action="store_true", + default=False, + help="abort an interrupted cherry-pick", + ) + parser.add_argument( + "--continue", + dest="cont", # because args.continue causes syntax errors + action="store_true", + default=False, + help="continue an interrupted cherry-pick", + ) + args = parser.parse_args() + + # register the exit handler after the arg parser completes so '--help' doesn't exit with + # an error. + atexit.register(early_exit_handler) + + # other scripts assume the short-sha is used for various comparisons, so + # make sure the provided sha is in the short form. + cmd = f"git show --format=%h --no-patch {args.commit_sha}" + args.commit_sha = run_git(cmd, args.repo_path)[0] + + commit_message_filename = os.path.join(args.tmp_path, "cherry-pick-commit_msg.txt") + + resume_state_filename = os.path.join(args.state_path, "cherry_pick_commit.resume") + resume_state = "" + if os.path.exists(resume_state_filename): + resume_state = get_last_line(resume_state_filename).strip() + print(f"resume_state: '{resume_state}'") + + # don't allow abort/continue flags if not in resume state + error_help = "--abort or --continue flags are not allowed when not in resume state" + if len(resume_state) == 0 and (args.abort or args.cont): + sys.exit(1) + error_help = None + + # detect missing abort/continue flags if in resume state + error_help = "cherry-pick in progress, use --abort or --continue" + if len(resume_state) != 0 and not args.abort and not args.cont: + sys.exit(1) + error_help = None + + # handle aborting cherry-pick + if args.abort: + # If the resume_state is not resume2 or resume3 that means we may + # have committed something to mercurial. First we need to check + # for our cherry-pick commit message, and if found, remove + # that commit. + if not (resume_state == "resume2" or resume_state == "resume3"): + run_hg("hg revert --all") + run_hg("hg purge {}".format(args.target_path)) + # check for committed mercurial patch and backout + stdout_lines = run_hg("hg log --template {desc|firstline}\n -r .") + # check for "Cherry-pick upstream libwebrtc commit" + print("stdout_lines before filter: {}".format(stdout_lines)) + stdout_lines = [ + line + for line in stdout_lines + if re.findall("Cherry-pick upstream libwebrtc commit", line) + ] + print("looking for commit: {}".format(stdout_lines)) + if len(stdout_lines) > 0: + cmd = "hg prune ." + print("calling '{}'".format(cmd)) + run_hg(cmd) + print("restoring patch stack") + restore_patch_stack( + args.repo_path, + args.branch, + os.path.abspath(args.patch_path), + args.state_path, + args.tar_name, + "https", # unused in this case + ) + # reset the resume file + print("reset resume file") + update_resume_state("", resume_state_filename) + print("after resetting resume file") + atexit.unregister(early_exit_handler) + sys.exit(0) + + # make sure the mercurial repo is clean before beginning + error_help = ( + "There are modified or untracked files in the mercurial repo.\n" + f"Please start with a clean repo before running {script_name}" + ) + stdout_lines = run_hg("hg status") + if len(stdout_lines) != 0: + sys.exit(1) + + # make sure the github repo exists + error_help = ( + f"No moz-libwebrtc github repo found at {args.repo_path}\n" + f"Please run restore_patch_stack.py before running {script_name}" + ) + if not os.path.exists(args.repo_path): + sys.exit(1) + error_help = None + + if len(resume_state) == 0: + update_resume_state("resume2", resume_state_filename) + print("-------") + print(f"------- write commit message file {commit_message_filename}") + print("-------") + write_commit_message_file( + commit_message_filename, + args.repo_path, + args.commit_sha, + args.commit_bug_number, + args.reviewers, + ) + + if len(resume_state) == 0 or resume_state == "resume2": + resume_state = "" + update_resume_state("resume3", resume_state_filename) + print("-------") + print(f"------- cherry-pick {args.commit_sha} into {args.repo_path}") + print("-------") + full_commit_message_filename = os.path.abspath(commit_message_filename) + error_help = ( + f"The cherry-pick operation of {args.commit_sha} has failed.\n" + "To fix this issue, you will need to jump to the github\n" + f"repo at {args.repo_path} .\n" + "Please resolve all the cherry-pick conflicts, and commit the changes\n" + "using:\n" + f" git commit --file {full_commit_message_filename}\n" + "\n" + "When the github cherry-pick is complete, resume running this\n" + f"script ({script_name})" + ) + cherry_pick_commit( + commit_message_filename, + args.repo_path, + args.commit_sha, + ) + error_help = None + + if len(resume_state) == 0 or resume_state == "resume3": + resume_state = "" + update_resume_state("resume4", resume_state_filename) + print("-------") + print(f"------- vendor from {args.repo_path}") + print("-------") + error_help = ( + f"Vendoring the newly cherry-picked git commit ({args.commit_sha}) has failed.\n" + "The mercurial repo is in an unknown state. This failure is\n" + "rare and thus makes it difficult to provide definitive guidance.\n" + "In essence, the current failing command is:\n" + f"./mach python {args.script_path}/vendor_and_commit.py \\\n" + f" --script-path {args.script_path} \\\n" + f" --repo-path {args.repo_path} \\\n" + f" --branch {args.branch} \\\n" + f" --commit-sha {args.commit_sha} \\\n" + f" --target-path {args.target_path} \\\n" + f" --state-path {args.state_path} \\\n" + f" --log-path {args.log_path} \\\n" + f" --commit-msg-path {commit_message_filename}\n" + "\n" + "Additional guidance may be in the terminal output above. Resolve\n" + "issues encountered by vendor_and_commit.py followed by re-running\n" + "vendor_and_commit.py to resume/complete its processing. After\n" + "vendor_and_commit.py completes successfully, resume running\n" + f"this script ({script_name})" + ) + vendor_and_commit( + args.script_path, + args.repo_path, + args.branch, + args.commit_sha, + args.target_path, # os.path.abspath(args.target_path), + args.state_path, + args.log_path, + commit_message_filename, + ) + error_help = None + + if len(resume_state) == 0 or resume_state == "resume4": + resume_state = "" + update_resume_state("resume5", resume_state_filename) + error_help = ( + "Reverting change to 'third_party/libwebrtc/README.mozilla'\n" + "has failed. The cherry-pick commit should not modify\n" + "'third_party/libwebrtc/README.mozilla'. If necessary\n" + "manually revert changes to 'third_party/libwebrtc/README.mozilla'\n" + f"and re-run {script_name}\n" + "to complete the cherry-pick processing." + ) + # The normal vendoring process updates README.mozilla with info + # on what commit was vendored and the command line used to do + # the vendoring. Since we're only reusing the vendoring script + # here, we don't want to update the README.mozilla file. + cmd = "hg revert -r tip^ third_party/libwebrtc/README.mozilla" + run_hg(cmd) + cmd = "hg amend" + run_hg(cmd) + error_help = None + + if len(resume_state) == 0 or resume_state == "resume5": + resume_state = "" + update_resume_state("resume6", resume_state_filename) + # get the files changed from the newly vendored cherry-pick + # commit in mercurial + cmd = "hg status --change tip --exclude '**/README.*'" + stdout_lines = run_shell(cmd) # run_shell to allow file wildcard + print("Mercurial changes:\n{}".format(stdout_lines)) + hg_file_change_cnt = len(stdout_lines) + + # get the files changed from the original cherry-picked patch in + # our github repo (moz-libwebrtc) + git_paths_changed = filter_git_changes(args.repo_path, args.commit_sha, None) + print("github changes:\n{}".format(git_paths_changed)) + git_file_change_cnt = len(git_paths_changed) + + error_help = ( + f"Vendoring the cherry-pick of commit {args.commit_sha} has failed due to mismatched\n" + f"changed file counts between mercurial ({hg_file_change_cnt}) and git ({git_file_change_cnt}).\n" + "This may be because the mozilla patch-stack was not verified after\n" + "running restore_patch_stack.py. After reconciling the changes in\n" + f"the newly committed mercurial patch, please re-run {script_name} to complete\n" + "the cherry-pick processing." + ) + if hg_file_change_cnt != git_file_change_cnt: + sys.exit(1) + error_help = None + + if len(resume_state) == 0 or resume_state == "resume6": + resume_state = "" + update_resume_state("", resume_state_filename) + print("-------") + print("------- write the noop tracking file") + print("-------") + write_noop_tracking_file(args.commit_sha, args.commit_bug_number) + + # unregister the exit handler so the normal exit doesn't falsely + # report as an error. + atexit.unregister(early_exit_handler) diff --git a/dom/media/webrtc/third_party_build/commit-build-file-changes.sh b/dom/media/webrtc/third_party_build/commit-build-file-changes.sh new file mode 100644 index 0000000000..b6bf1cd4e1 --- /dev/null +++ b/dom/media/webrtc/third_party_build/commit-build-file-changes.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +function show_error_msg() +{ + echo "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +# All commands should be printed as they are executed +# set -x + +# After this point: +# * eE: All commands should succede. +# * u: All variables should be defined before use. +# * o pipefail: All stages of all pipes should succede. +set -eEuo pipefail + +MOZ_BUILD_CHANGE_CNT=`hg status --include="**moz.build" third_party/libwebrtc | wc -l | tr -d " "` +echo "MOZ_BUILD_CHANGE_CNT: $MOZ_BUILD_CHANGE_CNT" +if [ "x$MOZ_BUILD_CHANGE_CNT" != "x0" ]; then + CURRENT_COMMIT_SHA=`hg id -i | sed 's/+//'` + COMMIT_DESC=`hg --config alias.log=log log -T '{desc|firstline}' -r $CURRENT_COMMIT_SHA` + + # since we have build file changes, touch the CLOBBER file + cat CLOBBER | egrep "^#|^$" > CLOBBER.new + mv CLOBBER.new CLOBBER + echo "Modified build files in third_party/libwebrtc - $COMMIT_DESC" >> CLOBBER + + ADD_CNT=`hg status --include="**moz.build" -nu third_party/libwebrtc | wc -l | tr -d " "` + DEL_CNT=`hg status --include="**moz.build" -nd third_party/libwebrtc | wc -l | tr -d " "` + if [ "x$ADD_CNT" != "x0" ]; then + hg status --include="**moz.build" -nu third_party/libwebrtc | xargs hg add + fi + if [ "x$DEL_CNT" != "x0" ]; then + hg status --include="**moz.build" -nd third_party/libwebrtc | xargs hg rm + fi + + hg commit -m \ + "$COMMIT_DESC - moz.build file updates" \ + --include="**moz.build" --include="CLOBBER" third_party/libwebrtc CLOBBER +fi + +echo "Done in $0" diff --git a/dom/media/webrtc/third_party_build/default_config_env b/dom/media/webrtc/third_party_build/default_config_env new file mode 100644 index 0000000000..af8b77bba6 --- /dev/null +++ b/dom/media/webrtc/third_party_build/default_config_env @@ -0,0 +1,45 @@ +#!/bin/bash + +# Edit {path-to} to match the location of your copy of Mozilla's +# fork of libwebrtc (at https://github.com/mozilla/libwebrtc). +export MOZ_LIBWEBRTC_SRC=$STATE_DIR/moz-libwebrtc + +# The previous fast-forward bug number is used for some error messaging. +export MOZ_PRIOR_FASTFORWARD_BUG="1867099" + +# Fast-forwarding each Chromium version of libwebrtc should be done +# under a separate bugzilla bug. This bug number is used when crafting +# the commit summary as each upstream commit is vendored into the +# mercurial repository. The bug used for the v106 fast-forward was +# 1800920. +export MOZ_FASTFORWARD_BUG="1871981" + +# MOZ_NEXT_LIBWEBRTC_MILESTONE and MOZ_NEXT_FIREFOX_REL_TARGET are +# not used during fast-forward processing, but facilitate generating this +# default config. To generate an default config for the next update, run +# bash dom/media/webrtc/third_party_build/update_default_config_env.sh +export MOZ_NEXT_LIBWEBRTC_MILESTONE=120 +export MOZ_NEXT_FIREFOX_REL_TARGET=124 + +# For Chromium release branches, see: +# https://chromiumdash.appspot.com/branches + +# Chromium's v119 release branch was 6045. This is used to pre-stack +# the previous release branch's commits onto the appropriate base commit +# (the first common commit between trunk and the release branch). +export MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM="6045" + +# New target release branch for v120 is branch-heads/6099. This is used +# to calculate the next upstream commit. +export MOZ_TARGET_UPSTREAM_BRANCH_HEAD="branch-heads/6099" + +# For local development 'mozpatches' is fine for a branch name, but when +# pushing the patch stack to github, it should be named something like +# 'moz-mods-chr120-for-rel124'. +export MOZ_LIBWEBRTC_BRANCH="mozpatches" + +# After elm has been merged to mozilla-central, the patch stack in +# moz-libwebrtc should be pushed to github. The script +# push_official_branch.sh uses this branch name when pushing to the +# public repo. +export MOZ_LIBWEBRTC_OFFICIAL_BRANCH="moz-mods-chr120-for-rel124" diff --git a/dom/media/webrtc/third_party_build/default_mozconfig b/dom/media/webrtc/third_party_build/default_mozconfig new file mode 100644 index 0000000000..195d4bebd3 --- /dev/null +++ b/dom/media/webrtc/third_party_build/default_mozconfig @@ -0,0 +1,7 @@ +mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/../obj-fast-forward-build +mk_add_options AUTOCLOBBER=1 +ac_add_options --with-ccache + +ac_add_options --enable-debug +ac_add_options --disable-optimize + diff --git a/dom/media/webrtc/third_party_build/detect_upstream_revert.sh b/dom/media/webrtc/third_party_build/detect_upstream_revert.sh new file mode 100644 index 0000000000..cbfe65af81 --- /dev/null +++ b/dom/media/webrtc/third_party_build/detect_upstream_revert.sh @@ -0,0 +1,113 @@ +#!/bin/bash + +function show_error_msg() +{ + echo "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +source dom/media/webrtc/third_party_build/use_config_env.sh + +# If DEBUG_GEN is set all commands should be printed as they are executed +if [ ! "x$DEBUG_GEN" = "x" ]; then + set -x +fi + +if [ "x$MOZ_LIBWEBRTC_SRC" = "x" ]; then + echo "MOZ_LIBWEBRTC_SRC is not defined, see README.md" + exit +fi + +if [ -d $MOZ_LIBWEBRTC_SRC ]; then + echo "MOZ_LIBWEBRTC_SRC is $MOZ_LIBWEBRTC_SRC" +else + echo "Path $MOZ_LIBWEBRTC_SRC is not found, see README.md" + exit +fi + +if [ "x$MOZ_LIBWEBRTC_BRANCH" = "x" ]; then + echo "MOZ_LIBWEBRTC_BRANCH is not defined, see README.md" + exit +fi + +if [ "x$AUTO_FIX_REVERT_AS_NOOP" = "x" ]; then + AUTO_FIX_REVERT_AS_NOOP="0" +fi + +find_base_commit +find_next_commit + +MOZ_LIBWEBRTC_COMMIT_MSG=`cd $MOZ_LIBWEBRTC_SRC ; \ +git show --name-only --oneline $MOZ_LIBWEBRTC_NEXT_BASE \ + | head -1 | sed 's/[^ ]* //'` + +echo "MOZ_LIBWEBRTC_BASE: $MOZ_LIBWEBRTC_BASE" +echo "MOZ_LIBWEBRTC_NEXT_BASE: $MOZ_LIBWEBRTC_NEXT_BASE" +echo "MOZ_LIBWEBRTC_COMMIT_MSG: $MOZ_LIBWEBRTC_COMMIT_MSG" +export MOZ_LIBWEBRTC_REVERT_SHA=`cd $MOZ_LIBWEBRTC_SRC ; \ +git log --oneline -r $MOZ_LIBWEBRTC_BASE..$MOZ_TARGET_UPSTREAM_BRANCH_HEAD \ + | grep -F "Revert \"$MOZ_LIBWEBRTC_COMMIT_MSG" \ + | tail -1 | awk '{print $1;}' || true` + +if [ "x$MOZ_LIBWEBRTC_REVERT_SHA" == "x" ]; then + echo "no revert commit summary detected in newer commits matching $MOZ_LIBWEBRTC_COMMIT_MSG" + exit +fi + +echo "found potential MOZ_LIBWEBRTC_REVERT_SHA: $MOZ_LIBWEBRTC_REVERT_SHA" +echo "checking commit message for 'This reverts commit' to confirm" +CONFIRM_SHA=`cd $MOZ_LIBWEBRTC_SRC ; \ +git show $MOZ_LIBWEBRTC_REVERT_SHA \ + | awk '/This reverts commit/ { print $NF; }' \ + | tr -d '[:punct:]'` + +if [ "x$CONFIRM_SHA" == "x" ]; then + echo "no revert commit listed in commit message for $MOZ_LIBWEBRTC_REVERT_SHA" + exit +fi + +# convert to short sha +CONFIRM_SHA=`cd $MOZ_LIBWEBRTC_SRC ; \ +git show --format='%h' --no-patch $CONFIRM_SHA` + +if [ $MOZ_LIBWEBRTC_NEXT_BASE != $CONFIRM_SHA ]; then + echo "revert sha ($MOZ_LIBWEBRTC_NEXT_BASE) does not match commit listed in commit summary ($CONFIRM_SHA)" + exit +fi + + +if [ "x$AUTO_FIX_REVERT_AS_NOOP" = "x1" ]; then + echo "AUTO_FIX_REVERT_AS_NOOP detected, fixing land/revert pair automatically" + bash $SCRIPT_DIR/make_upstream_revert_noop.sh + exit +fi + +echo $" +The next upstream commit has a corresponding future \"Revert\" commit. + +There are 2 common ways forward in this situation: +1. If you're relatively certain there will not be rebase conflicts in the + github repo ($MOZ_LIBWEBRTC_SRC), simply run: + SKIP_NEXT_REVERT_CHK=1 bash $SCRIPT_DIR/loop-ff.sh + +2. The surer method for no rebase conflicts is to cherry-pick both the + next commit, and the commit that reverts the next commit onto the + bottom of our patch stack in github. This pushes the likely rebase + conflict into the future when the upstream fix is relanded, but + ensures we only have to deal with the conflict once. The following + commands will add the necessary commits to the bottom of our patch + stack in github, and leave indicator files in the home directory that + help loop-ff know when to invoke special no-op commit handling: + + MOZ_LIBWEBRTC_BASE=$MOZ_LIBWEBRTC_BASE \\ + MOZ_LIBWEBRTC_NEXT_BASE=$MOZ_LIBWEBRTC_NEXT_BASE \\ + MOZ_LIBWEBRTC_REVERT_SHA=$MOZ_LIBWEBRTC_REVERT_SHA \\ + bash $SCRIPT_DIR/make_upstream_revert_noop.sh + + SKIP_NEXT_REVERT_CHK=1 bash $SCRIPT_DIR/loop-ff.sh +" +exit 1 diff --git a/dom/media/webrtc/third_party_build/elm_arcconfig.patch b/dom/media/webrtc/third_party_build/elm_arcconfig.patch new file mode 100644 index 0000000000..46adb9c6f1 --- /dev/null +++ b/dom/media/webrtc/third_party_build/elm_arcconfig.patch @@ -0,0 +1,10 @@ +diff --git a/.arcconfig b/.arcconfig +--- a/.arcconfig ++++ b/.arcconfig +@@ -1,5 +1,5 @@ + { + "phabricator.uri" : "https://phabricator.services.mozilla.com/", +- "repository.callsign": "MOZILLACENTRAL", ++ "repository.callsign": "ELM", + "history.immutable": false + } diff --git a/dom/media/webrtc/third_party_build/elm_rebase.sh b/dom/media/webrtc/third_party_build/elm_rebase.sh new file mode 100644 index 0000000000..ba0028b7a4 --- /dev/null +++ b/dom/media/webrtc/third_party_build/elm_rebase.sh @@ -0,0 +1,271 @@ +#!/bin/bash + +# This script exists to help with the rebase process on elm. It rebases +# each patch individually to make it easier to fix rebase conflicts +# without jeopardizing earlier, sucessfully rebased commits. In order to +# limit rebase conflicts around generated moz.build files, it regenerates +# moz.build file commits. It also ensures any commits with 'FLOAT' in the +# commit summary are pushed to the top of the fast-forward stack to help +# the sheriffs more easily merge our commit stack from elm to moz-central. +# +# Occasionally, there will be upstream vendored commits that break the +# build file generation with follow on commits that fix that error. In +# order to allow the rebase process to work more smoothly, it is possible +# to annotate a commit with the string '(skip-generation)' and normal +# build file generation (detected with changes to BUILD.gn files) is +# disabled for that commit. The script outputs instructions for handling +# this situation. +# +# Note: the very first rebase operation may require some manual +# intervention. The user will need to provide, at minimum, the first +# commit of the fast-forward stack if the script is unable to determine +# it automatically. Example: +# MOZ_BOTTOM_FF=30f0afb7e4c5 \ +# bash dom/media/webrtc/third_party_build/elm_rebase.sh +# +# Assumes the top of the fast-forward stack to rebase is the current revision, +# ".". + +function show_error_msg() +{ + echo "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +source dom/media/webrtc/third_party_build/use_config_env.sh + +GENERATION_ERROR=$" +Generating build files has failed. The most common reason for this +failure is that the current commit has an upcoming '(fix-xxxxxx)' commit +that will then allow the build file generation to complete. If the +current situation seems to fit that pattern, adding a line with +'(skip-generation)' to the commit message will ensure that future rebase +operations do not attempt to generate build files for this commit. It may +be as simple as running the following commands: + HGPLAIN=1 hg log -T '{desc}' -r tip > $TMP_DIR/commit_message.txt + ed -s $TMP_DIR/commit_message.txt <<< $'3i\n(skip-generation)\n\n.\nw\nq' + hg commit --amend -l $TMP_DIR/commit_message.txt + bash $0 +" +COMMIT_LIST_FILE=$TMP_DIR/rebase-commit-list.txt +export HGPLAIN=1 + +# After this point: +# * eE: All commands should succeed. +# * o pipefail: All stages of all pipes should succeed. +set -eEo pipefail + +if [ -f $STATE_DIR/rebase_resume_state ]; then + source $STATE_DIR/rebase_resume_state +else + + if [ "x" == "x$MOZ_TOP_FF" ]; then + MOZ_TOP_FF=`hg log -r . -T"{node|short}"` + + ERROR_HELP=$" +The topmost commit to be rebased is not in the public phase. Should it be +pushed to elm first? If this is intentional, please rerun the command and pass +it in explicitly: + MOZ_TOP_FF=$MOZ_TOP_FF bash $0 +" + if [[ $(hg phase -r .) != *public ]]; then + echo "$ERROR_HELP" + exit 1 + fi + ERROR_HELP="" + + ERROR_HELP=$" +The topmost commit to be rebased is public but has descendants. If those +descendants should not be rebased, please rerun the command and pass the commit +in explicitly: + MOZ_TOP_FF=$MOZ_TOP_FF bash $0 +" + if [ "x" != "x$(hg log -r 'descendants(.) and !.' -T'{node|short}')" ]; then + echo "$ERROR_HELP" + exit 1 + fi + ERROR_HELP="" + fi + + hg pull central + + ERROR_HELP=$" +Automatically determining the bottom (earliest) commit of the fast-forward +stack has failed. Please provide the bottom commit of the fast-forward +stack. The bottom commit means the commit following the most recent +mozilla-central commit. This could be the sha of the .arcconfig commit +if it is the bottom commit. +That command looks like: + MOZ_BOTTOM_FF={base-sha} bash $0 +" + if [ "x" == "x$MOZ_BOTTOM_FF" ]; then + # Finds the common ancestor between our top fast-forward commit and + # mozilla-central using: + # ancestor($MOZ_TOP_FF, central) + MOZ_OLD_CENTRAL=`hg id --id --rev "ancestor($MOZ_TOP_FF, central)"` + # Using that ancestor and $MOZ_TOP_FF as a range, find the commit _after_ + # the the common commit using limit(range, 1, 1) which gives the first + # commit of the range, offset by one commit. + MOZ_BOTTOM_FF=`hg id --id --rev "limit($MOZ_OLD_CENTRAL::$MOZ_TOP_FF, 1, 1)"` + fi + if [ "x" == "x$MOZ_BOTTOM_FF" ]; then + echo "No value found for the bottom commit of the fast-forward commit stack." + echo "$ERROR_HELP" + exit 1 + fi + ERROR_HELP="" + + # After this point: + # * eE: All commands should succeed. + # * u: All variables should be defined before use. + # * o pipefail: All stages of all pipes should succeed. + set -eEuo pipefail + + MOZ_NEW_CENTRAL=`hg log -r central -T"{node|short}"` + + echo "bottom of fast-foward tree is $MOZ_BOTTOM_FF" + echo "top of fast-forward tree (webrtc-fast-forward) is $MOZ_TOP_FF" + echo "new target for elm rebase $MOZ_NEW_CENTRAL (tip of moz-central)" + + hg log -T '{rev}:{node|short} {desc|firstline}\n' \ + -r $MOZ_BOTTOM_FF::$MOZ_TOP_FF > $COMMIT_LIST_FILE + + # move all FLOAT lines to end of file, and delete the "empty" tilde line + # line at the beginning + ed -s $COMMIT_LIST_FILE <<< $'g/- FLOAT -/m$\ng/^~$/d\nw\nq' + + MOZ_BOOKMARK=`date "+webrtc-fast-forward-%Y-%m-%d--%H-%M"` + hg bookmark -r elm $MOZ_BOOKMARK + + hg update $MOZ_NEW_CENTRAL + + # pre-work is complete, let's write out a temporary config file that allows + # us to resume + echo $"export MOZ_BOTTOM_FF=$MOZ_BOTTOM_FF +export MOZ_TOP_FF=$MOZ_TOP_FF +export MOZ_OLD_CENTRAL=$MOZ_OLD_CENTRAL +export MOZ_NEW_CENTRAL=$MOZ_NEW_CENTRAL +export MOZ_BOOKMARK=$MOZ_BOOKMARK +" > $STATE_DIR/rebase_resume_state +fi # if [ -f $STATE_DIR/rebase_resume_state ]; then ; else + +# grab all commits +COMMITS=`cat $COMMIT_LIST_FILE | awk '{print $1;}'` + +echo -n "Commits: " +for commit in $COMMITS; do +echo -n "$commit " +done +echo "" + +for commit in $COMMITS; do + echo "Processing $commit" + FULL_COMMIT_LINE=`head -1 $COMMIT_LIST_FILE` + + function remove_commit () { + echo "Removing from list '$FULL_COMMIT_LINE'" + ed -s $COMMIT_LIST_FILE <<< $'1d\nw\nq' + } + + IS_BUILD_COMMIT=`hg log -T '{desc|firstline}' -r $commit \ + | grep "file updates" | wc -l | tr -d " " || true` + echo "IS_BUILD_COMMIT: $IS_BUILD_COMMIT" + if [ "x$IS_BUILD_COMMIT" != "x0" ]; then + echo "Skipping $commit:" + hg log -T '{desc|firstline}' -r $commit + remove_commit + continue + fi + + IS_SKIP_GEN_COMMIT=`hg log --verbose \ + -r $commit \ + | grep "skip-generation" | wc -l | tr -d " " || true` + echo "IS_SKIP_GEN_COMMIT: $IS_SKIP_GEN_COMMIT" + + echo "Generate patch for: $commit" + hg export -r $commit > $TMP_DIR/rebase.patch + + echo "Import patch for $commit" + hg import $TMP_DIR/rebase.patch || \ + ( hg log -T '{desc}' -r $commit > $TMP_DIR/rebase_commit_message.txt ; \ + remove_commit ; \ + echo "Error importing: '$FULL_COMMIT_LINE'" ; \ + echo "Please fix import errors, then:" ; \ + echo " hg commit -l $TMP_DIR/rebase_commit_message.txt" ; \ + echo " bash $0" ; \ + exit 1 ) + + remove_commit + + if [ "x$IS_SKIP_GEN_COMMIT" != "x0" ]; then + echo "Skipping build generation for $commit" + continue + fi + + MODIFIED_BUILD_RELATED_FILE_CNT=`hg diff -c tip --stat \ + --include 'third_party/libwebrtc/**BUILD.gn' \ + --include 'third_party/libwebrtc/webrtc.gni' \ + --include 'dom/media/webrtc/third_party_build/gn-configs/webrtc.json' \ + | wc -l | tr -d " "` + echo "MODIFIED_BUILD_RELATED_FILE_CNT: $MODIFIED_BUILD_RELATED_FILE_CNT" + if [ "x$MODIFIED_BUILD_RELATED_FILE_CNT" != "x0" ]; then + echo "Regenerate build files" + ./mach python python/mozbuild/mozbuild/gn_processor.py \ + dom/media/webrtc/third_party_build/gn-configs/webrtc.json || \ + ( echo "$GENERATION_ERROR" ; exit 1 ) + + MOZ_BUILD_CHANGE_CNT=`hg status third_party/libwebrtc \ + --include 'third_party/libwebrtc/**moz.build' | wc -l | tr -d " "` + if [ "x$MOZ_BUILD_CHANGE_CNT" != "x0" ]; then + bash dom/media/webrtc/third_party_build/commit-build-file-changes.sh + NEWEST_COMMIT=`hg log -T '{desc|firstline}' -r tip` + echo "NEWEST_COMMIT: $NEWEST_COMMIT" + echo "NEWEST_COMMIT: $NEWEST_COMMIT" >> $LOG_DIR/rebase-build-changes-commits.log + fi + echo "Done generating build files" + fi + + echo "Done processing $commit" +done + +rm $STATE_DIR/rebase_resume_state + +# This is blank in case no changes have been made in third_party/libwebrtc +# since the previous rebase (or original elm reset). +PATCH_STACK_FIXUP="" + +echo "Checking for new mercurial changes in third_party/libwebrtc" +FIXUP_INSTRUCTIONS=$" +Mercurial changes in third_party/libwebrtc since the last rebase have been +detected (using the verify_vendoring.sh script). Running the following +commands should remedy the situation: + + ./mach python $SCRIPT_DIR/extract-for-git.py $MOZ_OLD_CENTRAL::$MOZ_NEW_CENTRAL + mv mailbox.patch $MOZ_LIBWEBRTC_SRC + (cd $MOZ_LIBWEBRTC_SRC && \\ + git am mailbox.patch) + bash $SCRIPT_DIR/verify_vendoring.sh + +When verify_vendoring.sh is successful, run the following in bash: + (source $SCRIPT_DIR/use_config_env.sh ; + ./mach python $SCRIPT_DIR/save_patch_stack.py \\ + --repo-path $MOZ_LIBWEBRTC_SRC \\ + --target-branch-head $MOZ_TARGET_UPSTREAM_BRANCH_HEAD \\ + --separate-commit-bug-number $MOZ_FASTFORWARD_BUG ) +" +bash $SCRIPT_DIR/verify_vendoring.sh &> $LOG_DIR/log-verify.txt || PATCH_STACK_FIXUP="$FIXUP_INSTRUCTIONS" +echo "Done checking for new mercurial changes in third_party/libwebrtc" + +REMAINING_STEPS=$" +The rebase process is complete. The following steps must be completed manually: +$PATCH_STACK_FIXUP + ./mach bootstrap --application=browser --no-system-changes && \\ + ./mach build && \\ + hg push -r tip --force && \\ + hg push -B $MOZ_BOOKMARK +" +echo "$REMAINING_STEPS" diff --git a/dom/media/webrtc/third_party_build/extract-for-git.py b/dom/media/webrtc/third_party_build/extract-for-git.py new file mode 100644 index 0000000000..2b926647fb --- /dev/null +++ b/dom/media/webrtc/third_party_build/extract-for-git.py @@ -0,0 +1,146 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import argparse +import os +import re +import subprocess + +# This script extracts commits that touch third party webrtc code so they can +# be imported into Git. It filters out commits that are not part of upstream +# code and rewrites the paths to match upstream. Finally, the commits are +# combined into a mailbox file that can be applied with `git am`. +LIBWEBRTC_DIR = "third_party/libwebrtc" + + +def build_commit_list(revset, env): + """Build commit list from the specified revset. + + The revset can be a single revision, e.g. 52bb9bb94661, or a range, + e.g. 8c08a5bb8a99::52bb9bb94661, or any other valid revset + (check hg help revset). Only commits that touch libwebrtc are included. + """ + res = subprocess.run( + ["hg", "log", "-r", revset, "-M", "--template", "{node}\n", LIBWEBRTC_DIR], + capture_output=True, + text=True, + env=env, + ) + return [line.strip() for line in res.stdout.strip().split("\n")] + + +def extract_author_date(sha1, env): + res = subprocess.run( + ["hg", "log", "-r", sha1, "--template", "{author}|{date|isodate}"], + capture_output=True, + text=True, + env=env, + ) + return res.stdout.split("|") + + +def extract_description(sha1, env): + res = subprocess.run( + ["hg", "log", "-r", sha1, "--template", "{desc}"], + capture_output=True, + text=True, + env=env, + ) + return res.stdout + + +def extract_commit(sha1, env): + res = subprocess.run( + ["hg", "log", "-r", sha1, "-pg", "--template", "\n"], + capture_output=True, + text=True, + env=env, + ) + return res.stdout + + +def filter_nonwebrtc(commit): + filtered = [] + skipping = False + for line in commit.split("\n"): + # Extract only patches affecting libwebrtc, but avoid commits that + # touch build, which is tracked by a separate repo, or that affect + # moz.build files which are code generated. + if ( + line.startswith("diff --git a/" + LIBWEBRTC_DIR) + and not line.startswith("diff --git a/" + LIBWEBRTC_DIR + "/build/") + and not line.startswith("diff --git a/" + LIBWEBRTC_DIR + "/third_party/") + and not line.startswith("diff --git a/" + LIBWEBRTC_DIR + "/README.moz") + and not line.startswith( + "diff --git a/" + LIBWEBRTC_DIR + "/moz-patch-stack/" + ) + and not line.endswith("moz.build") + ): + skipping = False + elif line.startswith("diff --git"): + skipping = True + + if not skipping: + filtered.append(line) + return "\n".join(filtered) + + +def fixup_paths(commit): + # make sure we only rewrite paths in the diff-related or rename lines + commit = re.sub( + f"^rename (from|to) {LIBWEBRTC_DIR}/", "rename \\1 ", commit, flags=re.MULTILINE + ) + return re.sub(f"( [ab])/{LIBWEBRTC_DIR}/", "\\1/", commit) + + +def write_as_mbox(sha1, author, date, description, commit, ofile): + # Use same magic date as git format-patch + ofile.write("From {} Mon Sep 17 00:00:00 2001\n".format(sha1)) + ofile.write("From: {}\n".format(author)) + ofile.write("Date: {}\n".format(date)) + description = description.split("\n") + ofile.write("Subject: {}\n".format(description[0])) + ofile.write("\n".join(description[1:])) + ofile.write( + "\nMercurial Revision: https://hg.mozilla.org/mozilla-central/rev/{}\n".format( + sha1 + ) + ) + ofile.write(commit) + ofile.write("\n") + ofile.write("\n") + + +if __name__ == "__main__": + commits = [] + parser = argparse.ArgumentParser( + description="Format commits for upstream libwebrtc" + ) + parser.add_argument( + "revsets", metavar="revset", type=str, nargs="+", help="A revset to process" + ) + parser.add_argument( + "--target", choices=("libwebrtc", "build", "third_party"), default="libwebrtc" + ) + args = parser.parse_args() + + if args.target != "libwebrtc": + LIBWEBRTC_DIR = os.path.join(LIBWEBRTC_DIR, args.target) + + # must run 'hg' with HGPLAIN=1 to ensure aliases don't interfere with + # command output. + env = os.environ.copy() + env["HGPLAIN"] = "1" + + for revset in args.revsets: + commits.extend(build_commit_list(revset, env)) + + with open("mailbox.patch", "w") as ofile: + for sha1 in commits: + author, date = extract_author_date(sha1, env) + description = extract_description(sha1, env) + filtered_commit = filter_nonwebrtc(extract_commit(sha1, env)) + if len(filtered_commit) == 0: + continue + fixedup_commit = fixup_paths(filtered_commit) + write_as_mbox(sha1, author, date, description, fixedup_commit, ofile) diff --git a/dom/media/webrtc/third_party_build/fast-forward-libwebrtc.sh b/dom/media/webrtc/third_party_build/fast-forward-libwebrtc.sh new file mode 100644 index 0000000000..c5d490405b --- /dev/null +++ b/dom/media/webrtc/third_party_build/fast-forward-libwebrtc.sh @@ -0,0 +1,159 @@ +#!/bin/bash + +function show_error_msg() +{ + echo "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +source dom/media/webrtc/third_party_build/use_config_env.sh + +# If DEBUG_GEN is set all commands should be printed as they are executed +if [ ! "x$DEBUG_GEN" = "x" ]; then + set -x +fi + +if [ "x$MOZ_LIBWEBRTC_SRC" = "x" ]; then + echo "MOZ_LIBWEBRTC_SRC is not defined, see README.md" + exit +fi + +if [ -d $MOZ_LIBWEBRTC_SRC ]; then + echo "MOZ_LIBWEBRTC_SRC is $MOZ_LIBWEBRTC_SRC" +else + echo "Path $MOZ_LIBWEBRTC_SRC is not found, see README.md" + exit +fi + +if [ "x$MOZ_LIBWEBRTC_BRANCH" = "x" ]; then + echo "MOZ_LIBWEBRTC_BRANCH is not defined, see README.md" + exit +fi + +RESUME_FILE=$STATE_DIR/fast_forward.resume +RESUME="" +if [ -f $RESUME_FILE ]; then + RESUME=`tail -1 $RESUME_FILE` +fi + +GIT_IS_REBASING=`cd $MOZ_LIBWEBRTC_SRC && git status | grep "interactive rebase in progress" | wc -l | tr -d " " || true` +if [ "x$GIT_IS_REBASING" != "x0" ]; then + echo "There is currently a git rebase operation in progress at $MOZ_LIBWEBRTC_SRC." + echo "Please resolve the rebase before attempting to continue the fast-forward" + echo "operation." + exit 1 +fi + +if [ "x$RESUME" = "x" ]; then + SKIP_TO="run" + # Check for modified files and abort if present. + MODIFIED_FILES=`hg status --exclude "third_party/libwebrtc/**.orig" third_party/libwebrtc` + if [ "x$MODIFIED_FILES" = "x" ]; then + # Completely clean the mercurial checkout before proceeding + hg update -C -r . + hg purge + else + echo "There are modified files in the checkout. Cowardly aborting!" + echo "$MODIFIED_FILES" + exit 1 + fi +else + SKIP_TO=$RESUME + hg revert -C third_party/libwebrtc/README.moz-ff-commit &> /dev/null +fi + +find_base_commit +find_next_commit + +# After this point: +# * eE: All commands should succeed. +# * u: All variables should be defined before use. +# * o pipefail: All stages of all pipes should succeed. +set -eEuo pipefail + +echo " MOZ_LIBWEBRTC_BASE: $MOZ_LIBWEBRTC_BASE" +echo "MOZ_LIBWEBRTC_NEXT_BASE: $MOZ_LIBWEBRTC_NEXT_BASE" +echo " RESUME: $RESUME" +echo "SKIP_TO: $SKIP_TO" + +echo "-------" +echo "------- Write cmd-line to third_party/libwebrtc/README.moz-ff-commit" +echo "-------" +echo "# MOZ_LIBWEBRTC_SRC=$MOZ_LIBWEBRTC_SRC MOZ_LIBWEBRTC_BRANCH=$MOZ_LIBWEBRTC_BRANCH bash $0" \ + >> third_party/libwebrtc/README.moz-ff-commit + +echo "-------" +echo "------- Write new-base to last line of third_party/libwebrtc/README.moz-ff-commit" +echo "-------" +echo "# base of lastest vendoring" >> third_party/libwebrtc/README.moz-ff-commit +echo "$MOZ_LIBWEBRTC_NEXT_BASE" >> third_party/libwebrtc/README.moz-ff-commit + +REBASE_HELP=$" +The rebase operation onto $MOZ_LIBWEBRTC_NEXT_BASE has failed. Please +resolve all the rebase conflicts. To fix this issue, you will need to +jump to the github repo at $MOZ_LIBWEBRTC_SRC . +When the github rebase is complete, re-run the script to resume the +fast-forward process. +" +function rebase_mozlibwebrtc_stack { + echo "-------" + echo "------- Rebase $MOZ_LIBWEBRTC_BRANCH to $MOZ_LIBWEBRTC_NEXT_BASE" + echo "-------" + ERROR_HELP=$REBASE_HELP + ( cd $MOZ_LIBWEBRTC_SRC && \ + git checkout -q $MOZ_LIBWEBRTC_BRANCH && \ + git rebase $MOZ_LIBWEBRTC_NEXT_BASE \ + &> $LOG_DIR/log-rebase-moz-libwebrtc.txt \ + ) + ERROR_HELP="" +} + +function write_commit_message_file { + echo "-------" + echo "------- Write commit message file ($TMP_DIR/commit_msg.txt)" + echo "-------" + UPSTREAM_LONG_SHA=`cd $MOZ_LIBWEBRTC_SRC && \ + git show --format='%H' --no-patch $MOZ_LIBWEBRTC_NEXT_BASE` + echo "Bug $MOZ_FASTFORWARD_BUG - Vendor libwebrtc from $MOZ_LIBWEBRTC_NEXT_BASE" \ + > $TMP_DIR/commit_msg.txt + echo "" >> $TMP_DIR/commit_msg.txt + if [ -f $STATE_DIR/$MOZ_LIBWEBRTC_NEXT_BASE.no-op-cherry-pick-msg ]; then + cat $STATE_DIR/$MOZ_LIBWEBRTC_NEXT_BASE.no-op-cherry-pick-msg >> $TMP_DIR/commit_msg.txt + echo "" >> $TMP_DIR/commit_msg.txt + fi + echo "Upstream commit: https://webrtc.googlesource.com/src/+/$UPSTREAM_LONG_SHA" >> $TMP_DIR/commit_msg.txt + (cd $MOZ_LIBWEBRTC_SRC && \ + git show --name-only $MOZ_LIBWEBRTC_NEXT_BASE | grep "^ ") >> $TMP_DIR/commit_msg.txt +} + +if [ $SKIP_TO = "run" ]; then + echo "resume2" > $RESUME_FILE + rebase_mozlibwebrtc_stack; +fi + +if [ $SKIP_TO = "resume2" ]; then SKIP_TO="run"; fi +if [ $SKIP_TO = "run" ]; then + echo "resume3" > $RESUME_FILE + write_commit_message_file; +fi + +if [ $SKIP_TO = "resume3" ]; then SKIP_TO="run"; fi +if [ $SKIP_TO = "run" ]; then + ./mach python dom/media/webrtc/third_party_build/vendor_and_commit.py \ + --repo-path $MOZ_LIBWEBRTC_SRC \ + --script-path $SCRIPT_DIR \ + --commit-msg-path $TMP_DIR/commit_msg.txt \ + --commit-sha $MOZ_LIBWEBRTC_NEXT_BASE +fi + +echo "" > $RESUME_FILE + +# now that we've committed the vendored code, we can delete the +# no-op commit tracking file if it exists. +if [ -f $STATE_DIR/$MOZ_LIBWEBRTC_NEXT_BASE.no-op-cherry-pick-msg ]; then + rm $STATE_DIR/$MOZ_LIBWEBRTC_NEXT_BASE.no-op-cherry-pick-msg +fi diff --git a/dom/media/webrtc/third_party_build/fetch_github_repo.py b/dom/media/webrtc/third_party_build/fetch_github_repo.py new file mode 100644 index 0000000000..b9d10e0b6c --- /dev/null +++ b/dom/media/webrtc/third_party_build/fetch_github_repo.py @@ -0,0 +1,146 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import argparse +import os +import re +import shutil +import sys + +from run_operations import run_git, run_shell + +# This script fetches the moz-libwebrtc github repro with the expected +# upstream remote and branch-heads setup. This is used by both the +# prep_repo.sh script as well as the restore_patch_stack.py script. +# +# For speed and conservation of network resources, after fetching all +# the data, a tar of the repo is made and used if available. + + +def fetch_repo(github_path, clone_protocol, force_fetch, tar_path): + capture_output = False + + # check for pre-existing repo - make sure we force the removal + if force_fetch and os.path.exists(github_path): + print("Removing existing repo: {}".format(github_path)) + shutil.rmtree(github_path) + + # To test with ssh (and not give away your default public key): + # ssh-keygen -t rsa -f ~/.ssh/id_moz_github -q -N "" + # git -c core.sshCommand="ssh -i ~/.ssh/id_moz_github -o IdentitiesOnly=yes" clone git@github.com:mozilla/libwebrtc.git moz-libwebrtc + # (cd moz-libwebrtc && git config core.sshCommand "ssh -i ~/.ssh/id_moz_github -o IdentitiesOnly=yes") + + # clone https://github.com/mozilla/libwebrtc + if not os.path.exists(github_path): + # check for pre-existing tar, use it if we have it + if os.path.exists(tar_path): + print("Using tar file to reconstitute repo") + cmd = "cd {} ; tar --extract --gunzip --file={}".format( + os.path.dirname(github_path), os.path.basename(tar_path) + ) + run_shell(cmd, capture_output) + else: + print("Cloning github repo") + # sure would be nice to have python 3.10's match + if clone_protocol == "ssh": + url_prefix = "git@github.com:" + elif clone_protocol == "https": + url_prefix = "https://github.com/" + else: + print("clone protocol should be either https or ssh") + sys.exit(1) + + run_shell( + "git clone {}mozilla/libwebrtc {}".format(url_prefix, github_path), + capture_output, + ) + + # setup upstream (https://webrtc.googlesource.com/src) + stdout_lines = run_git("git config --local --list", github_path) + stdout_lines = [ + path for path in stdout_lines if re.findall("^remote.upstream.url.*", path) + ] + if len(stdout_lines) == 0: + print("Fetching upstream") + run_git("git checkout master", github_path) + run_git( + "git remote add upstream https://webrtc.googlesource.com/src", github_path + ) + run_git("git fetch upstream", github_path) + run_git("git merge upstream/master", github_path) + else: + print( + "Upstream remote (https://webrtc.googlesource.com/src) already configured" + ) + + # setup upstream branch-heads + stdout_lines = run_git( + "git config --local --get-all remote.upstream.fetch", github_path + ) + if len(stdout_lines) == 1: + print("Fetching upstream branch-heads") + run_git( + "git config --local --add remote.upstream.fetch +refs/branch-heads/*:refs/remotes/branch-heads/*", + github_path, + ) + run_git("git fetch upstream", github_path) + else: + print("Upstream remote branch-heads already configured") + + # do a sanity fetch in case this was not a freshly cloned copy of the + # repo, meaning it may not have all the mozilla branches present. + run_git("git fetch --all", github_path) + + # create tar to avoid time refetching + if not os.path.exists(tar_path): + print("Creating tar file for quicker restore") + cmd = "cd {} ; tar --create --gzip --file={} {}".format( + os.path.dirname(github_path), + os.path.basename(tar_path), + os.path.basename(github_path), + ) + run_shell(cmd, capture_output) + + +if __name__ == "__main__": + default_state_dir = ".moz-fast-forward" + default_tar_name = "moz-libwebrtc.tar.gz" + + parser = argparse.ArgumentParser( + description="Restore moz-libwebrtc github patch stack" + ) + parser.add_argument( + "--repo-path", + required=True, + help="path to libwebrtc repo", + ) + parser.add_argument( + "--force-fetch", + action="store_true", + default=False, + help="force rebuild an existing repo directory", + ) + parser.add_argument( + "--clone-protocol", + choices=["https", "ssh"], + required=True, + help="Use either https or ssh to clone the git repo", + ) + parser.add_argument( + "--tar-name", + default=default_tar_name, + help="name of tar file (defaults to {})".format(default_tar_name), + ) + parser.add_argument( + "--state-path", + default=default_state_dir, + help="path to state directory (defaults to {})".format(default_state_dir), + ) + args = parser.parse_args() + + fetch_repo( + args.repo_path, + args.clone_protocol, + args.force_fetch, + os.path.join(args.state_path, args.tar_name), + ) diff --git a/dom/media/webrtc/third_party_build/filter_git_changes.py b/dom/media/webrtc/third_party_build/filter_git_changes.py new file mode 100644 index 0000000000..ee6cdbcbd9 --- /dev/null +++ b/dom/media/webrtc/third_party_build/filter_git_changes.py @@ -0,0 +1,87 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import argparse +import importlib +import re +import sys + +sys.path.insert(0, "./dom/media/webrtc/third_party_build") +vendor_libwebrtc = importlib.import_module("vendor-libwebrtc") + +from run_operations import run_git + + +def filter_git_changes(github_path, commit_sha, diff_filter): + command = [ + "git", + "show", + "--oneline", + "--name-status", + "--pretty=format:", + None if not diff_filter else "--diff-filter={}".format(diff_filter), + commit_sha, + ] + # strip possible empty elements from command list + command = " ".join([x for x in command if x is not None]) + + # Get the list of changes in the upstream commit. + stdout_lines = run_git(command, github_path) + + changed_files = [line.strip() for line in stdout_lines] + changed_files = [line for line in changed_files if line != ""] + + # Fetch the list of excludes and includes used in the vendoring script. + exclude_file_list = vendor_libwebrtc.get_excluded_files() + exclude_dir_list = vendor_libwebrtc.get_excluded_dirs() + include_list = vendor_libwebrtc.get_included_path_overrides() + + # First, search for changes in files that are specifically included. + # Do this first, because some of these files might be filtered out + # by the exclude list. + regex_includes = "|".join(["^.\t{}$".format(i) for i in include_list]) + included_files = [ + path for path in changed_files if re.findall(regex_includes, path) + ] + + # Convert the directory exclude list to a regex string and filter + # out the excluded directory paths (note the lack of trailing '$' + # in the regex). + regex_excludes = "|".join( + ["^(M|A|D|R\d\d\d)\t{}".format(i) for i in exclude_dir_list] + ) + files_not_excluded = [ + path for path in changed_files if not re.findall(regex_excludes, path) + ] + + # Convert the file exclude list to a regex string and filter out the + # excluded file paths. The trailing '$' in the regex ensures that + # we can exclude, for example, '.vpython' and not '.vpython3'. + regex_excludes = "|".join(["^.\t{}$".format(i) for i in exclude_file_list]) + files_not_excluded = [ + path for path in files_not_excluded if not re.findall(regex_excludes, path) + ] + + return included_files + files_not_excluded + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Get relevant change count from an upstream git commit" + ) + parser.add_argument( + "--repo-path", + required=True, + help="path to libwebrtc repo", + ) + parser.add_argument("--commit-sha", required=True, help="sha of commit to examine") + parser.add_argument( + "--diff-filter", + choices=("A", "D", "R"), + help="filter for adds (A), deletes (D), or renames (R)", + ) + args = parser.parse_args() + + paths = filter_git_changes(args.repo_path, args.commit_sha, args.diff_filter) + for path in paths: + print(path) diff --git a/dom/media/webrtc/third_party_build/gn-configs/README.md b/dom/media/webrtc/third_party_build/gn-configs/README.md new file mode 100644 index 0000000000..111d2d022e --- /dev/null +++ b/dom/media/webrtc/third_party_build/gn-configs/README.md @@ -0,0 +1,16 @@ +# Generate new gn json files and moz.build files for building libwebrtc in our tree + +/!\ This is only supported on Linux and macOS. If you are on Windows, you can run +the script under [WSL](https://docs.microsoft.com/en-us/windows/wsl/install). + +1. The script should be run from the top directory of our firefox tree. + + ``` + ./mach python python/mozbuild/mozbuild/gn_processor.py dom/media/webrtc/third_party_build/gn-configs/webrtc.json + ``` + +2. Checkin all the generated/modified files and try your build! + +# Adding new configurations to the build + +Edit the `main` function in the `python/mozbuild/mozbuild/gn_processor.py` file. diff --git a/dom/media/webrtc/third_party_build/gn-configs/webrtc.json b/dom/media/webrtc/third_party_build/gn-configs/webrtc.json new file mode 100644 index 0000000000..d1194046af --- /dev/null +++ b/dom/media/webrtc/third_party_build/gn-configs/webrtc.json @@ -0,0 +1,91 @@ +{ + "target_dir": "third_party/libwebrtc", + "gn_target": "//:webrtc", + "gn_sandbox_variables": { + "COMPILE_FLAGS": { + "WARNINGS_AS_ERRORS": [] + }, + "FINAL_LIBRARY": "webrtc" + }, + "mozilla_flags": ["-fobjc-arc", "-mavx2", "-mfma", "-mfpu=neon", "-msse2"], + "write_mozbuild_variables": { + "INCLUDE_TK_CFLAGS_DIRS": [ + "third_party/libwebrtc/modules/desktop_capture/desktop_capture_gn", + "third_party/libwebrtc/modules/portal/portal_gn", + "third_party/libwebrtc/modules/video_capture/video_capture_internal_impl_gn" + ], + "INCLUDE_SYSTEM_LIBVPX_HANDLING": [ + "third_party/libwebrtc/modules/video_coding/webrtc_libvpx_interface_gn", + "third_party/libwebrtc/modules/video_coding/webrtc_vp8_gn", + "third_party/libwebrtc/modules/video_coding/webrtc_vp9_gn", + "third_party/libwebrtc/third_party/libvpx/libvpx_gn" + ] + }, + "non_unified_sources": [ + "third_party/libwebrtc/api/audio_codecs/opus/audio_encoder_opus_config.cc", + "third_party/libwebrtc/api/video/i210_buffer.cc", + "third_party/libwebrtc/api/video/i410_buffer.cc", + "third_party/libwebrtc/api/video/i422_buffer.cc", + "third_party/libwebrtc/api/video/i444_buffer.cc", + "third_party/libwebrtc/audio/channel_send_frame_transformer_delegate.cc", + "third_party/libwebrtc/common_audio/vad/vad_core.c", + "third_party/libwebrtc/common_audio/vad/webrtc_vad.c", + "third_party/libwebrtc/common_audio/signal_processing/resample_by_2_mips.c", + "third_party/libwebrtc/modules/audio_coding/codecs/isac/fix/source/decode_plc.c", + "third_party/libwebrtc/modules/audio_coding/codecs/isac/fix/source/lpc_masking_model.c", + "third_party/libwebrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter.c", + "third_party/libwebrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c", + "third_party/libwebrtc/modules/audio_coding/neteq/audio_vector.cc", + "third_party/libwebrtc/modules/audio_coding/neteq/underrun_optimizer.cc", + "third_party/libwebrtc/modules/audio_device/android/audio_manager.cc", + "third_party/libwebrtc/modules/audio_device/android/audio_record_jni.cc", + "third_party/libwebrtc/modules/audio_device/android/audio_track_jni.cc", + "third_party/libwebrtc/modules/audio_device/android/opensles_player.cc", + "third_party/libwebrtc/modules/audio_device/linux/audio_device_pulse_linux.cc", + "third_party/libwebrtc/modules/audio_device/linux/audio_mixer_manager_pulse_linux.cc", + "third_party/libwebrtc/modules/audio_device/win/audio_device_core_win.cc", + "third_party/libwebrtc/modules/audio_processing/aecm/aecm_core.cc", + "third_party/libwebrtc/modules/audio_processing/aecm/aecm_core_c.cc", + "third_party/libwebrtc/modules/audio_processing/aecm/aecm_core_mips.cc", + "third_party/libwebrtc/modules/audio_processing/aecm/aecm_core_neon.cc", + "third_party/libwebrtc/modules/audio_processing/aecm/echo_control_mobile.cc", + "third_party/libwebrtc/modules/audio_processing/echo_control_mobile_impl.cc", + "third_party/libwebrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator.cc", + "third_party/libwebrtc/modules/audio_processing/gain_control_impl.cc", + "third_party/libwebrtc/modules/audio_processing/rms_level.cc", + "third_party/libwebrtc/modules/congestion_controller/goog_cc/loss_based_bandwidth_estimation.cc", + "third_party/libwebrtc/modules/congestion_controller/goog_cc/send_side_bandwidth_estimation.cc", + "third_party/libwebrtc/modules/desktop_capture/fallback_desktop_capturer_wrapper.cc", + "third_party/libwebrtc/modules/desktop_capture/linux/wayland/base_capturer_pipewire.cc", + "third_party/libwebrtc/modules/desktop_capture/linux/wayland/moz_base_capturer_pipewire.cc", + "third_party/libwebrtc/modules/desktop_capture/mouse_cursor_monitor_linux.cc", + "third_party/libwebrtc/modules/desktop_capture/win/screen_capturer_win_gdi.cc", + "third_party/libwebrtc/modules/pacing/prioritized_packet_queue.cc", + "third_party/libwebrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator_single_stream.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/flexfec_03_header_reader_writer.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/flexfec_receiver.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/rtcp_packet/tmmbn.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/rtcp_packet/tmmbr.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/rtcp_sender.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_vp8.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/rtp_rtcp_impl2.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_egress.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_video.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/rtp_util.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/ulpfec_generator.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/video_rtp_depacketizer_vp9.cc", + "third_party/libwebrtc/modules/third_party/g722/g722_encode.c", + "third_party/libwebrtc/modules/video_capture/windows/device_info_ds.cc", + "third_party/libwebrtc/modules/video_capture/windows/help_functions_ds.cc", + "third_party/libwebrtc/modules/video_capture/windows/sink_filter_ds.cc", + "third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc", + "third_party/libwebrtc/modules/video_coding/svc/scalability_structure_key_svc.cc", + "third_party/libwebrtc/modules/video_coding/svc/scalability_structure_simulcast.cc", + "third_party/libwebrtc/rtc_base/win/hstring.cc", + "third_party/libwebrtc/third_party/abseil-cpp/absl/strings/numbers.cc", + "third_party/libwebrtc/third_party/abseil-cpp/absl/synchronization/blocking_counter.cc", + "third_party/libwebrtc/third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_posix.cc", + "third_party/libwebrtc/third_party/abseil-cpp/absl/time/time.cc", + "third_party/libwebrtc/video/rtp_video_stream_receiver2.cc" + ] +} diff --git a/dom/media/webrtc/third_party_build/lookup_branch_head.py b/dom/media/webrtc/third_party_build/lookup_branch_head.py new file mode 100644 index 0000000000..afd0e6c791 --- /dev/null +++ b/dom/media/webrtc/third_party_build/lookup_branch_head.py @@ -0,0 +1,98 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import argparse +import json +import os +import pathlib +import sys +import urllib.request + +# default cache file location in STATE_DIR location +default_cache_path = ".moz-fast-forward/milestone.cache" + + +def fetch_branch_head_dict(): + milestone_url = ( + "https://chromiumdash.appspot.com/fetch_milestones?only_branched=true" + ) + uf = urllib.request.urlopen(milestone_url) + html = uf.read() + milestone_dict = json.loads(html) + + # There is more information in the json dictionary, but we only care + # about the milestone (version) to branch "name" (webrtc_branch) + # info. For example: + # v106 -> 5249 (which translates to branch-heads/5249) + # v107 -> 5304 (which translates to branch-heads/5304) + # + # As returned from web query, milestones are integers and branch + # "names" are strings. + new_dict = {} + for row in milestone_dict: + new_dict[row["milestone"]] = row["webrtc_branch"] + + return new_dict + + +def read_dict_from_cache(cache_path): + if cache_path is not None and os.path.exists(cache_path): + with open(cache_path, "r") as ifile: + return json.loads(ifile.read(), object_hook=jsonKeys2int) + return {} + + +def write_dict_to_cache(cache_path, milestones): + with open(cache_path, "w") as ofile: + ofile.write(json.dumps(milestones)) + + +def get_branch_head(milestone, cache_path=default_cache_path): + milestones = read_dict_from_cache(cache_path) + + # if the cache didn't exist or is stale, try to fetch using a web query + if milestone not in milestones: + try: + milestones = fetch_branch_head_dict() + write_dict_to_cache(cache_path, milestones) + except Exception: + pass + + if milestone in milestones: + return milestones[milestone] + return None + + +# From https://stackoverflow.com/questions/1450957/pythons-json-module-converts-int-dictionary-keys-to-strings +def jsonKeys2int(x): + if isinstance(x, dict): + return {int(k): v for k, v in x.items()} + return x + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Get libwebrtc branch-head for given chromium milestone" + ) + parser.add_argument( + "milestone", type=int, help="integer chromium milestone (example: 106)" + ) + parser.add_argument("-v", "--verbose", action="store_true") + parser.add_argument("-c", "--cache", type=pathlib.Path, help="path to cache file") + args = parser.parse_args() + + # if the user provided a cache path use it, otherwise use the default + local_cache_path = args.cache or default_cache_path + + branch_head = get_branch_head(args.milestone, local_cache_path) + if branch_head is None: + sys.exit("error: chromium milestone '{}' is not found.".format(args.milestone)) + + if args.verbose: + print( + "chromium milestone {} uses branch-heads/{}".format( + args.milestone, branch_head + ) + ) + else: + print(branch_head) diff --git a/dom/media/webrtc/third_party_build/loop-ff.sh b/dom/media/webrtc/third_party_build/loop-ff.sh new file mode 100644 index 0000000000..73ad22822c --- /dev/null +++ b/dom/media/webrtc/third_party_build/loop-ff.sh @@ -0,0 +1,284 @@ +#!/bin/bash + +source dom/media/webrtc/third_party_build/use_config_env.sh + +# file for logging loop script output +LOOP_OUTPUT_LOG=$LOG_DIR/log-loop-ff.txt + +function echo_log() +{ + echo "===loop-ff=== $@" 2>&1| tee -a $LOOP_OUTPUT_LOG +} + +function show_error_msg() +{ + echo_log "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo_log "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +# If DEBUG_LOOP_FF is set all commands should be printed as they are executed +if [ ! "x$DEBUG_LOOP_FF" = "x" ]; then + set -x +fi + +if [ "x$MOZ_LIBWEBRTC_SRC" = "x" ]; then + echo "MOZ_LIBWEBRTC_SRC is not defined, see README.md" + exit +fi + +if [ ! -d $MOZ_LIBWEBRTC_SRC ]; then + echo "Path $MOZ_LIBWEBRTC_SRC is not found, see README.md" + exit +fi + +if [ "x$MOZ_LIBWEBRTC_BRANCH" = "x" ]; then + echo "MOZ_LIBWEBRTC_BRANCH is not defined, see README.md" + exit +fi + +if [ "x$MOZ_STOP_AFTER_COMMIT" = "x" ]; then + MOZ_STOP_AFTER_COMMIT=`cd $MOZ_LIBWEBRTC_SRC ; git show $MOZ_TARGET_UPSTREAM_BRANCH_HEAD --format='%h' --name-only | head -1` + echo "No MOZ_STOP_AFTER_COMMIT variable defined - stopping at $MOZ_TARGET_UPSTREAM_BRANCH_HEAD" +fi + +if [ "x$MOZ_ADVANCE_ONE_COMMIT" = "x" ]; then + MOZ_ADVANCE_ONE_COMMIT="" +fi + +MOZ_CHANGED=0 +GIT_CHANGED=0 +HANDLE_NOOP_COMMIT="" + +# After this point: +# * eE: All commands should succeed. +# * u: All variables should be defined before use. +# * o pipefail: All stages of all pipes should succeed. +set -eEuo pipefail + +# start a new log with every run of this script +rm -f $LOOP_OUTPUT_LOG +# make sure third_party/libwebrtc/README.moz-ff-commit is the committed version +# so we properly determine MOZ_LIBWEBRTC_BASE and MOZ_LIBWEBRTC_NEXT_BASE +# in the loop below +hg revert -C third_party/libwebrtc/README.moz-ff-commit &> /dev/null + +# check for a resume situation from fast-forward-libwebrtc.sh +RESUME_FILE=$STATE_DIR/fast_forward.resume +RESUME="" +if [ -f $RESUME_FILE ]; then + RESUME=`tail -1 $RESUME_FILE` +fi + +# check for the situation where we've encountered an error when running +# detect_upstream_revert.sh and should skip running it a second time. +SKIP_NEXT_REVERT_CHK="" +if [ -f $STATE_DIR/loop.skip-revert-detect ]; then + SKIP_NEXT_REVERT_CHK=`tail -1 $STATE_DIR/loop.skip-revert-detect` +fi +echo "SKIP_NEXT_REVERT_CHK: '$SKIP_NEXT_REVERT_CHK'" 2>&1| tee -a $LOOP_OUTPUT_LOG + +ERROR_HELP=$" +It appears that verification of initial vendoring from our local copy +of the moz-libwebrtc git repo containing our patch-stack has failed. +- If you have never previously run the fast-forward (loop-ff.sh) script, + you may need to prepare the github repository by running prep_repo.sh. +- If you have previously run loop-ff.sh successfully, there may be a new + change to third_party/libwebrtc that should be extracted from mercurial + and added to the patch stack in github. It may be as easy as running: + ./mach python $SCRIPT_DIR/extract-for-git.py tip::tip + mv mailbox.patch $MOZ_LIBWEBRTC_SRC + (cd $MOZ_LIBWEBRTC_SRC && \\ + git am mailbox.patch) + +To verify vendoring, run: + bash $SCRIPT_DIR/verify_vendoring.sh + +When verify_vendoring.sh is successful, please run the following command +in bash: + (source $SCRIPT_DIR/use_config_env.sh ; + ./mach python $SCRIPT_DIR/save_patch_stack.py \ + --repo-path $MOZ_LIBWEBRTC_SRC \ + --target-branch-head $MOZ_TARGET_UPSTREAM_BRANCH_HEAD ) + +You may resume running this script with the following command: + bash $SCRIPT_DIR/loop-ff.sh +" +# if we're not in the resume situation from fast-forward-libwebrtc.sh +if [ "x$RESUME" = "x" ]; then + # start off by verifying the vendoring process to make sure no changes have + # been added to elm to fix bugs. + echo_log "Verifying vendoring..." + # The script outputs its own error message when verifying fails, so + # capture that output of verify_vendoring.sh quietly. + bash $SCRIPT_DIR/verify_vendoring.sh &> $LOG_DIR/log-verify.txt + echo_log "Done verifying vendoring." +fi +ERROR_HELP="" + +for (( ; ; )); do + +find_base_commit +find_next_commit + +if [ $MOZ_LIBWEBRTC_BASE == $MOZ_LIBWEBRTC_NEXT_BASE ]; then + echo_log "Processing complete, already at upstream $MOZ_LIBWEBRTC_BASE" + exit +fi + +echo_log "===================" + +COMMITS_REMAINING=`cd $MOZ_LIBWEBRTC_SRC ; \ + git log --oneline $MOZ_LIBWEBRTC_BASE..$MOZ_TARGET_UPSTREAM_BRANCH_HEAD \ + | wc -l | tr -d " "` +echo_log "Commits remaining: $COMMITS_REMAINING" + +echo "Before revert detection, SKIP_NEXT_REVERT_CHK: '$SKIP_NEXT_REVERT_CHK'" 2>&1| tee -a $LOOP_OUTPUT_LOG +echo "Before revert detection, RESUME: '$RESUME'" 2>&1| tee -a $LOOP_OUTPUT_LOG +ERROR_HELP=$"Some portion of the detection and/or fixing of upstream revert commits +has failed. Please fix the state of the git hub repo at: $MOZ_LIBWEBRTC_SRC. +When fixed, please resume this script with the following command: + bash $SCRIPT_DIR/loop-ff.sh +" +if [ "x$SKIP_NEXT_REVERT_CHK" == "x" ] && [ "x$RESUME" == "x" ]; then + echo_log "Check for upcoming revert commit" + echo "true" > $STATE_DIR/loop.skip-revert-detect + AUTO_FIX_REVERT_AS_NOOP=1 bash $SCRIPT_DIR/detect_upstream_revert.sh \ + 2>&1| tee -a $LOOP_OUTPUT_LOG +fi +echo "" > $STATE_DIR/loop.skip-revert-detect +ERROR_HELP="" + +echo_log "Looking for $STATE_DIR/$MOZ_LIBWEBRTC_NEXT_BASE.no-op-cherry-pick-msg" +if [ -f $STATE_DIR/$MOZ_LIBWEBRTC_NEXT_BASE.no-op-cherry-pick-msg ]; then + echo_log "Detected special commit msg, setting HANDLE_NOOP_COMMIT=1" + HANDLE_NOOP_COMMIT="1" +fi + +echo_log "Moving from moz-libwebrtc commit $MOZ_LIBWEBRTC_BASE to $MOZ_LIBWEBRTC_NEXT_BASE" +bash $SCRIPT_DIR/fast-forward-libwebrtc.sh 2>&1| tee -a $LOOP_OUTPUT_LOG + +MOZ_CHANGED=`hg diff -c tip --stat \ + | egrep -ve "README.moz-ff-commit|README.mozilla|files changed," \ + | wc -l | tr -d " " || true` +GIT_CHANGED=`./mach python $SCRIPT_DIR/filter_git_changes.py \ + --repo-path $MOZ_LIBWEBRTC_SRC --commit-sha $MOZ_LIBWEBRTC_NEXT_BASE \ + | wc -l | tr -d " "` +FILE_CNT_MISMATCH_MSG=$" +The number of files changed in the upstream commit ($GIT_CHANGED) does +not match the number of files changed in the local Mozilla repo +commit ($MOZ_CHANGED). This may indicate a mismatch between the vendoring +script and this script, or it could be a true error in the import +processing. Once the issue has been resolved, the following steps +remain for this commit: + # generate moz.build files (may not be necessary) + ./mach python python/mozbuild/mozbuild/gn_processor.py \\ + $SCRIPT_DIR/gn-configs/webrtc.json + # commit the updated moz.build files with the appropriate commit msg + bash $SCRIPT_DIR/commit-build-file-changes.sh + # do a (hopefully) quick test build + ./mach build +" +echo_log "Verify number of files changed MOZ($MOZ_CHANGED) GIT($GIT_CHANGED)" +if [ "x$HANDLE_NOOP_COMMIT" == "x1" ]; then + echo_log "NO-OP commit detected, we expect file changed counts to differ" +elif [ $MOZ_CHANGED -ne $GIT_CHANGED ]; then + echo_log "MOZ_CHANGED $MOZ_CHANGED should equal GIT_CHANGED $GIT_CHANGED" + echo "$FILE_CNT_MISMATCH_MSG" 2>&1| tee -a $LOOP_OUTPUT_LOG + exit 1 +fi +HANDLE_NOOP_COMMIT="" + +# save the current patch stack in case we need to reconstitute it later +echo_log "Save patch-stack" +./mach python $SCRIPT_DIR/save_patch_stack.py \ + --skip-startup-sanity \ + --repo-path $MOZ_LIBWEBRTC_SRC \ + --branch $MOZ_LIBWEBRTC_BRANCH \ + --patch-path "third_party/libwebrtc/moz-patch-stack" \ + --state-path $STATE_DIR \ + --target-branch-head $MOZ_TARGET_UPSTREAM_BRANCH_HEAD \ + 2>&1| tee -a $LOOP_OUTPUT_LOG + +MODIFIED_BUILD_RELATED_FILE_CNT=`hg diff -c tip --stat \ + --include 'third_party/libwebrtc/**BUILD.gn' \ + --include 'third_party/libwebrtc/webrtc.gni' \ + | grep -v "files changed" \ + | wc -l | tr -d " " || true` +ERROR_HELP=$" +Generating build files has failed. This likely means changes to one or more +BUILD.gn files are required. Commit those changes following the instructions +in https://wiki.mozilla.org/Media/WebRTC/libwebrtc_Update_Process#Operational_notes +Then complete these steps: + # generate moz.build files (may not be necessary) + ./mach python python/mozbuild/mozbuild/gn_processor.py \\ + $SCRIPT_DIR/gn-configs/webrtc.json + # commit the updated moz.build files with the appropriate commit msg + bash $SCRIPT_DIR/commit-build-file-changes.sh + # do a (hopefully) quick test build + ./mach build +After a successful build, you may resume this script. +" +echo_log "Modified BUILD.gn (or webrtc.gni) files: $MODIFIED_BUILD_RELATED_FILE_CNT" +MOZ_BUILD_CHANGE_CNT=0 +if [ "x$MODIFIED_BUILD_RELATED_FILE_CNT" != "x0" ]; then + echo_log "Regenerate build files" + ./mach python python/mozbuild/mozbuild/gn_processor.py \ + $SCRIPT_DIR/gn-configs/webrtc.json 2>&1| tee -a $LOOP_OUTPUT_LOG + + MOZ_BUILD_CHANGE_CNT=`hg status third_party/libwebrtc \ + --include 'third_party/libwebrtc/**moz.build' | wc -l | tr -d " "` + if [ "x$MOZ_BUILD_CHANGE_CNT" != "x0" ]; then + echo_log "Detected modified moz.build files, commiting" + bash $SCRIPT_DIR/commit-build-file-changes.sh 2>&1| tee -a $LOOP_OUTPUT_LOG + fi +fi +ERROR_HELP="" + +ERROR_HELP=$" +The test build has failed. Most likely this is due to an upstream api +change that must be reflected in Mozilla code outside of the +third_party/libwebrtc directory. After fixing the build, you may resume +running this script with the following command: + bash $SCRIPT_DIR/loop-ff.sh +" +echo_log "Test build" +./mach build 2>&1| tee -a $LOOP_OUTPUT_LOG +ERROR_HELP="" + +# If we've committed moz.build changes, spin up try builds. +if [ "x$MOZ_BUILD_CHANGE_CNT" != "x0" ]; then + TRY_FUZZY_QUERY_STRING="^build-" + CURRENT_TIME=`date` + echo_log "Detected modified moz.build files, starting try builds with" + echo_log "'$TRY_FUZZY_QUERY_STRING' at $CURRENT_TIME" + echo_log "This try push is started to help earlier detection of build issues" + echo_log "across different platforms supported by Mozilla." + echo_log "Note - this step can take a long time (occasionally in the 10min range)" + echo_log " with little or no feedback." + # Show the time used for this command, and don't let it fail if the + # command times out so the script continues running. This command + # can take quite long, occasionally 10min. + (time ./mach try fuzzy --full -q $TRY_FUZZY_QUERY_STRING) 2>&1| tee -a $LOOP_OUTPUT_LOG || true +fi + +if [ ! "x$MOZ_STOP_AFTER_COMMIT" = "x" ]; then +if [ $MOZ_LIBWEBRTC_NEXT_BASE = $MOZ_STOP_AFTER_COMMIT ]; then + break +fi +fi + +if [ ! "x$MOZ_ADVANCE_ONE_COMMIT" = "x" ]; then + echo_log "Done advancing one commit." + exit +fi + +# successfully completed one iteration through the loop, so we can reset RESUME +RESUME="" + +done + +echo_log "Completed fast-foward to $MOZ_STOP_AFTER_COMMIT" diff --git a/dom/media/webrtc/third_party_build/make_upstream_revert_noop.sh b/dom/media/webrtc/third_party_build/make_upstream_revert_noop.sh new file mode 100755 index 0000000000..7562c8f264 --- /dev/null +++ b/dom/media/webrtc/third_party_build/make_upstream_revert_noop.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +# This script takes the current base sha, the next base sha, and the sha +# of the commit that reverts the next base as determined by +# detect_upstream_revert.sh and "inserts" two commits at the bottom of the +# moz_libwebrtc GitHub patch stack. The two commits are exact copies of +# the upcoming commit and its corresponding revert commit, with commit +# messages indicating they are temporary commits. Additionally, 2 files +# are written that act as markers for the fast-forward script and contain +# supplemental commit message text. +# +# When the fast-forward script runs, it will rebase onto the next base +# sha. Since we have a corresponding, identical temp commit at the bottom +# of our patch stack, the temp commit will be absorbed as unnecessary. +# Since the patch stack now has the temp revert commit at the bottom, this +# results in a “no-op” commit. The marker file indicates that specially +# handling should occur in the fast-forward-libwebrtc.sh and loop-ff.sh +# scripts. This special handling includes adding the supplemental commit +# text that explains why the commit is a no-op (or empty) commit and +# skipping the verification of the number of files changed between our +# mercurial repository and the GitHub repository. + +function show_error_msg() +{ + echo "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +source dom/media/webrtc/third_party_build/use_config_env.sh + +# If DEBUG_GEN is set all commands should be printed as they are executed +if [ ! "x$DEBUG_GEN" = "x" ]; then + set -x +fi + +if [ "x$MOZ_LIBWEBRTC_SRC" = "x" ]; then + echo "MOZ_LIBWEBRTC_SRC is not defined, see README.md" + exit +fi + +if [ -d $MOZ_LIBWEBRTC_SRC ]; then + echo "MOZ_LIBWEBRTC_SRC is $MOZ_LIBWEBRTC_SRC" +else + echo "Path $MOZ_LIBWEBRTC_SRC is not found, see README.md" + exit +fi + +if [ "x$MOZ_LIBWEBRTC_BRANCH" = "x" ]; then + echo "MOZ_LIBWEBRTC_BRANCH is not defined, see README.md" + exit +fi + +# After this point: +# * eE: All commands should succeed. +# * u: All variables should be defined before use. +# * o pipefail: All stages of all pipes should succeed. +set -eEuo pipefail + +find_base_commit +find_next_commit +echo "MOZ_LIBWEBRTC_BASE: $MOZ_LIBWEBRTC_BASE" +echo "MOZ_LIBWEBRTC_NEXT_BASE: $MOZ_LIBWEBRTC_NEXT_BASE" +echo "MOZ_LIBWEBRTC_REVERT_SHA: $MOZ_LIBWEBRTC_REVERT_SHA" + +# These files serve dual purposes: +# 1) They serve as marker/indicator files to loop-ff.sh to +# know to process the commit differently, accounting for +# the no-op nature of the commit and it's corresponding +# revert commit. +# 2) The contain supplemental commit message text to explain +# why the commits are essentially empty. +# They are written first on the off chance that the rebase +# operation below fails and requires manual intervention, +# thus avoiding the operator of these scripts to remember to +# generate these two files. +echo $"Essentially a no-op since we're going to see this change +reverted when we vendor in $MOZ_LIBWEBRTC_REVERT_SHA." \ +> $STATE_DIR/$MOZ_LIBWEBRTC_NEXT_BASE.no-op-cherry-pick-msg + +echo "We already cherry-picked this when we vendored $MOZ_LIBWEBRTC_NEXT_BASE." \ +> $STATE_DIR/$MOZ_LIBWEBRTC_REVERT_SHA.no-op-cherry-pick-msg + +cd $MOZ_LIBWEBRTC_SRC +git checkout -b moz-cherry-pick $MOZ_LIBWEBRTC_BASE + +COMMIT_MSG_FILE=$TMP_DIR/commit.msg + +# build commit message with annotated summary +git show --format='%s%n%n%b' --no-patch $MOZ_LIBWEBRTC_NEXT_BASE > $COMMIT_MSG_FILE +ed -s $COMMIT_MSG_FILE <<EOF +1,s/^\(.*\)$/(tmp-cherry-pick) & ($MOZ_LIBWEBRTC_NEXT_BASE)/ +w +q +EOF +git cherry-pick --no-commit $MOZ_LIBWEBRTC_NEXT_BASE +git commit --file $COMMIT_MSG_FILE + +# build commit message with annotated summary +git show --format='%s%n%n%b' --no-patch $MOZ_LIBWEBRTC_REVERT_SHA > $COMMIT_MSG_FILE +ed -s $COMMIT_MSG_FILE <<EOF +1,s/^\(.*\)$/(tmp-cherry-pick) & ($MOZ_LIBWEBRTC_REVERT_SHA)/ +w +q +EOF +git cherry-pick --no-commit $MOZ_LIBWEBRTC_REVERT_SHA +git commit --file $COMMIT_MSG_FILE + +git checkout $MOZ_LIBWEBRTC_BRANCH +git rebase moz-cherry-pick +git branch -d moz-cherry-pick + diff --git a/dom/media/webrtc/third_party_build/pre-warmed-milestone.cache b/dom/media/webrtc/third_party_build/pre-warmed-milestone.cache new file mode 100644 index 0000000000..826268db47 --- /dev/null +++ b/dom/media/webrtc/third_party_build/pre-warmed-milestone.cache @@ -0,0 +1 @@ +{"109": "5414", "108": "5359", "107": "5304", "106": "5249", "105": "5195"} diff --git a/dom/media/webrtc/third_party_build/prep_repo.sh b/dom/media/webrtc/third_party_build/prep_repo.sh new file mode 100644 index 0000000000..b601ebc808 --- /dev/null +++ b/dom/media/webrtc/third_party_build/prep_repo.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +function show_error_msg() +{ + echo "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +source dom/media/webrtc/third_party_build/use_config_env.sh +export HGPLAIN=1 + +echo "MOZ_LIBWEBRTC_SRC: $MOZ_LIBWEBRTC_SRC" +echo "MOZ_LIBWEBRTC_BRANCH: $MOZ_LIBWEBRTC_BRANCH" +echo "MOZ_FASTFORWARD_BUG: $MOZ_FASTFORWARD_BUG" + +# After this point: +# * eE: All commands should succeed. +# * u: All variables should be defined before use. +# * o pipefail: All stages of all pipes should succeed. +set -eEuo pipefail + +# set CLONE_PROTOCOL="https", ignored if the repo already exists. +CLONE_PROTOCOL="https" + +# don't prompt for clone protocol if repo already exists +if [ ! -d $MOZ_LIBWEBRTC_SRC ]; then + echo "Clone moz-libwebrtc git repo with http or ssh?" + select yn in "https" "ssh"; do + case $yn in + https ) echo "Clone w/ https"; CLONE_PROTOCOL="https"; break;; + ssh ) echo "Clone w/ ssh"; CLONE_PROTOCOL="ssh"; break;; + esac + done +fi + +# wipe resume_state for new run +rm -f $STATE_DIR/resume_state + +# If there is no cache file for the branch-head lookups done in +# update_default_config.sh, go ahead and copy our small pre-warmed +# version. +if [ ! -f $STATE_DIR/milestone.cache ]; then + cp $SCRIPT_DIR/pre-warmed-milestone.cache $STATE_DIR/milestone.cache +fi + +# If there is no .mozconfig file, copy a basic one from default_mozconfig +if [ ! -f .mozconfig ]; then + cp $SCRIPT_DIR/default_mozconfig .mozconfig +fi + +# fetch the github repro +./mach python $SCRIPT_DIR/fetch_github_repo.py \ + --repo-path $MOZ_LIBWEBRTC_SRC \ + --clone-protocol $CLONE_PROTOCOL \ + --state-path $STATE_DIR + +CURRENT_DIR=`pwd` +cd $MOZ_LIBWEBRTC_SRC + +# clear any possible previous patches +rm -f *.patch + +# create a new work branch and "export" a new patch stack to rebase +# find the common commit between our upstream release branch and trunk +CHERRY_PICK_BASE=`git merge-base branch-heads/$MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM master` +echo "common commit: $CHERRY_PICK_BASE" + +# create a new branch at the common commit and checkout the new branch +ERROR_HELP=$" +Unable to create branch '$MOZ_LIBWEBRTC_BRANCH'. This probably means +that prep_repo.sh is being called on a repo that already has a patch +stack in progress. If you're sure you want to do this, the following +commands will allow the process to continue: + ( cd $MOZ_LIBWEBRTC_SRC && \\ + git checkout $MOZ_LIBWEBRTC_BRANCH && \\ + git checkout -b $MOZ_LIBWEBRTC_BRANCH-old && \\ + git branch -D $MOZ_LIBWEBRTC_BRANCH ) && \\ + bash $0 +" +git branch $MOZ_LIBWEBRTC_BRANCH $CHERRY_PICK_BASE +ERROR_HELP="" +git checkout $MOZ_LIBWEBRTC_BRANCH + +# make sure we're starting with a clean tmp directory +rm -f $TMP_DIR/*.patch $TMP_DIR/*.patch.bak + +# grab the patches for all the commits in chrome's release branch for libwebrtc +git format-patch -o $TMP_DIR -k $CHERRY_PICK_BASE..branch-heads/$MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM +# tweak the release branch commit summaries to show they were cherry picked +sed -i.bak -e "/^Subject: / s/^Subject: /Subject: (cherry-pick-branch-heads\/$MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM) /" $TMP_DIR/*.patch +git am $TMP_DIR/*.patch # applies to branch mozpatches +rm $TMP_DIR/*.patch $TMP_DIR/*.patch.bak + +# we don't use restore_patch_stack.py here because it would overwrite the patches +# from the previous release branch we just added in the above step. + +# grab all the moz patches and apply +git am $CURRENT_DIR/third_party/libwebrtc/moz-patch-stack/*.patch + +cd $CURRENT_DIR + +# cp all the no-op files to STATE_DIR +NO_OP_FILE_COUNT=`ls third_party/libwebrtc/moz-patch-stack \ + | grep "no-op-cherry-pick-msg" | wc -l | tr -d " " || true` +if [ "x$NO_OP_FILE_COUNT" != "x0" ]; then + cp $CURRENT_DIR/third_party/libwebrtc/moz-patch-stack/*.no-op-cherry-pick-msg \ + $STATE_DIR +fi + +bash $SCRIPT_DIR/verify_vendoring.sh || exit 1 diff --git a/dom/media/webrtc/third_party_build/push_official_branch.sh b/dom/media/webrtc/third_party_build/push_official_branch.sh new file mode 100644 index 0000000000..e7ed17df2b --- /dev/null +++ b/dom/media/webrtc/third_party_build/push_official_branch.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# This script is a simple helper script that creates a branch name +# in moz-libwebrtc and pushes it to origin. You will need permissions +# to the mozilla fork of libwebrtc in order to complete this operation. +# The repo is: https://github.com/mozilla/libwebrtc/ +# +# Note: this should only be run after elm has been merged to mozilla-central + +function show_error_msg() +{ + echo "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +source dom/media/webrtc/third_party_build/use_config_env.sh + +# After this point: +# * eE: All commands should succeed. +# * o pipefail: All stages of all pipes should succeed. +set -eEo pipefail + +cd $MOZ_LIBWEBRTC_SRC + +git fetch + +if git show-ref --quiet refs/remotes/origin/$MOZ_LIBWEBRTC_OFFICIAL_BRANCH; then + echo "Branch '$MOZ_LIBWEBRTC_OFFICIAL_BRANCH' already exists remotely. Nothing to do." + + echo "Please ensure this branch information can be found on Bug $MOZ_FASTFORWARD_BUG" + echo "https://github.com/mozilla/libwebrtc/tree/$MOZ_LIBWEBRTC_OFFICIAL_BRANCH" + exit 0 +fi + +if git show-ref --quiet refs/heads/$MOZ_LIBWEBRTC_OFFICIAL_BRANCH; then + echo "Branch '$MOZ_LIBWEBRTC_OFFICIAL_BRANCH' already exists locally. No need to create it." +else + echo "Creating branch '$MOZ_LIBWEBRTC_OFFICIAL_BRANCH'" + git branch $MOZ_LIBWEBRTC_OFFICIAL_BRANCH +fi + +echo "Pushing the branch to https://github.com/mozilla/libwebrtc" +git push origin $MOZ_LIBWEBRTC_OFFICIAL_BRANCH + +echo "Please add this new branch information to Bug $MOZ_FASTFORWARD_BUG" +echo "https://github.com/mozilla/libwebrtc/tree/$MOZ_LIBWEBRTC_OFFICIAL_BRANCH" diff --git a/dom/media/webrtc/third_party_build/restore_elm_arcconfig.py b/dom/media/webrtc/third_party_build/restore_elm_arcconfig.py new file mode 100644 index 0000000000..a855f4d567 --- /dev/null +++ b/dom/media/webrtc/third_party_build/restore_elm_arcconfig.py @@ -0,0 +1,28 @@ +#!/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from subprocess import run + +# This script sets the Arcanist configuration for the elm repo. This script +# should be run after each repository reset. +# +# Usage: from the root of the repo `./mach python dom/media/webrtc/third_party_build/restore_elm_arcconfig.py` +# + +ret = run( + [ + "hg", + "import", + "-m", + "Bug 1729988 - FLOAT - REPO-elm - update .arcconfig repo callsign r=bgrins", + "dom/media/webrtc/third_party_build/elm_arcconfig.patch", + ] +).returncode +if ret != 0: + raise Exception(f"Failed to add FLOATing arcconfig patch for ELM: { ret }") +else: + print("ELM .arcconfig restored. Please push this change to ELM:") + print(" hg push -r tip") diff --git a/dom/media/webrtc/third_party_build/restore_patch_stack.py b/dom/media/webrtc/third_party_build/restore_patch_stack.py new file mode 100644 index 0000000000..cc25623881 --- /dev/null +++ b/dom/media/webrtc/third_party_build/restore_patch_stack.py @@ -0,0 +1,131 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import argparse +import os +import re +import shutil +import sys + +from fetch_github_repo import fetch_repo +from run_operations import get_last_line, run_git, run_hg, run_shell + +# This script restores the mozilla patch stack and no-op commit tracking +# files. In the case of repo corruption or a mistake made during +# various rebase conflict resolution operations, the patch-stack can be +# restored rather than requiring the user to restart the fast-forward +# process from the beginning. + + +def restore_patch_stack( + github_path, + github_branch, + patch_directory, + state_directory, + tar_name, + clone_protocol, +): + # make sure the repo is clean before beginning + stdout_lines = run_hg("hg status third_party/libwebrtc") + if len(stdout_lines) != 0: + print("There are modified or untracked files under third_party/libwebrtc") + print("Please cleanup the repo under third_party/libwebrtc before running") + print(os.path.basename(__file__)) + sys.exit(1) + + # first, refetch the repo (hopefully utilizing the tarfile for speed) so + # the patches apply cleanly + print("fetch repo") + fetch_repo( + github_path, clone_protocol, True, os.path.join(state_directory, tar_name) + ) + + # remove any stale no-op-cherry-pick-msg files in state_directory + print("clear no-op-cherry-pick-msg files") + run_shell("rm {}/*.no-op-cherry-pick-msg || true".format(state_directory)) + + # lookup latest vendored commit from third_party/libwebrtc/README.moz-ff-commit + print( + "lookup latest vendored commit from third_party/libwebrtc/README.moz-ff-commit" + ) + file = os.path.abspath("third_party/libwebrtc/README.moz-ff-commit") + last_vendored_commit = get_last_line(file) + + # checkout the previous vendored commit with proper branch name + print( + "checkout the previous vendored commit ({}) with proper branch name".format( + last_vendored_commit + ) + ) + cmd = "git checkout -b {} {}".format(github_branch, last_vendored_commit) + run_git(cmd, github_path) + + # restore the patches to moz-libwebrtc repo, use run_shell instead of + # run_hg to allow filepath wildcard + print("Restoring patch stack") + run_shell("cd {} && git am {}/*.patch".format(github_path, patch_directory)) + + # it is also helpful to restore the no-op-cherry-pick-msg files to + # the state directory so that if we're restoring a patch-stack we + # also restore the possibly consumed no-op tracking files. + no_op_files = [ + path + for path in os.listdir(patch_directory) + if re.findall(".*no-op-cherry-pick-msg$", path) + ] + for file in no_op_files: + shutil.copy(os.path.join(patch_directory, file), state_directory) + + print("Please run the following command to verify the state of the patch-stack:") + print(" bash dom/media/webrtc/third_party_build/verify_vendoring.sh") + + +if __name__ == "__main__": + default_patch_dir = "third_party/libwebrtc/moz-patch-stack" + default_state_dir = ".moz-fast-forward" + default_tar_name = "moz-libwebrtc.tar.gz" + + parser = argparse.ArgumentParser( + description="Restore moz-libwebrtc github patch stack" + ) + parser.add_argument( + "--repo-path", + required=True, + help="path to libwebrtc repo", + ) + parser.add_argument( + "--branch", + default="mozpatches", + help="moz-libwebrtc branch (defaults to mozpatches)", + ) + parser.add_argument( + "--patch-path", + default=default_patch_dir, + help="path to save patches (defaults to {})".format(default_patch_dir), + ) + parser.add_argument( + "--tar-name", + default=default_tar_name, + help="name of tar file (defaults to {})".format(default_tar_name), + ) + parser.add_argument( + "--state-path", + default=default_state_dir, + help="path to state directory (defaults to {})".format(default_state_dir), + ) + parser.add_argument( + "--clone-protocol", + choices=["https", "ssh"], + default="https", + help="Use either https or ssh to clone the git repo (ignored if tar file exists)", + ) + args = parser.parse_args() + + restore_patch_stack( + args.repo_path, + args.branch, + os.path.abspath(args.patch_path), + args.state_path, + args.tar_name, + args.clone_protocol, + ) diff --git a/dom/media/webrtc/third_party_build/run_operations.py b/dom/media/webrtc/third_party_build/run_operations.py new file mode 100644 index 0000000000..0e86710b5d --- /dev/null +++ b/dom/media/webrtc/third_party_build/run_operations.py @@ -0,0 +1,97 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import os +import subprocess +import sys + +# This is a collection of helper functions that use the subprocess.run +# command to execute commands. In the future, if we have cases where +# we find common python functionality in utilizing the data returned +# from these functions, that functionality can live here for easy reuse +# between other python scripts in dom/media/webrtc/third_party_build. + + +# must run 'hg' with HGPLAIN=1 to ensure aliases don't interfere with +# command output. +env = os.environ.copy() +env["HGPLAIN"] = "1" + + +def run_hg(cmd): + cmd_list = cmd.split(" ") + res = subprocess.run( + cmd_list, + capture_output=True, + text=True, + env=env, + ) + if res.returncode != 0: + print( + "Hit return code {} running '{}'. Aborting.".format(res.returncode, cmd), + file=sys.stderr, + ) + print(res.stderr) + sys.exit(1) + stdout = res.stdout.strip() + return [] if len(stdout) == 0 else stdout.split("\n") + + +def run_git(cmd, working_dir): + cmd_list = cmd.split(" ") + res = subprocess.run( + cmd_list, + capture_output=True, + text=True, + cwd=working_dir, + ) + if res.returncode != 0: + print( + "Hit return code {} running '{}'. Aborting.".format(res.returncode, cmd), + file=sys.stderr, + ) + print(res.stderr) + sys.exit(1) + stdout = res.stdout.strip() + return [] if len(stdout) == 0 else stdout.split("\n") + + +def run_shell(cmd, capture_output=True): + res = subprocess.run( + cmd, + shell=True, + capture_output=capture_output, + text=True, + ) + if res.returncode != 0: + print( + "Hit return code {} running '{}'. Aborting.".format(res.returncode, cmd), + file=sys.stderr, + ) + if capture_output: + print(res.stderr) + sys.exit(1) + output_lines = [] + if capture_output: + stdout = res.stdout.strip() + output_lines = [] if len(stdout) == 0 else stdout.split("\n") + + return output_lines + + +def get_last_line(file_path): + # technique from https://stackoverflow.com/questions/46258499/how-to-read-the-last-line-of-a-file-in-python + with open(file_path, "rb") as f: + try: # catch OSError in case of a one line file + f.seek(-2, os.SEEK_END) + while f.read(1) != b"\n": + f.seek(-2, os.SEEK_CUR) + except OSError: + f.seek(0) + return f.readline().decode().strip() + + +def update_resume_state(state, resume_state_filename): + with open(resume_state_filename, "w") as ofile: + ofile.write(state) + ofile.write("\n") diff --git a/dom/media/webrtc/third_party_build/save_patch_stack.py b/dom/media/webrtc/third_party_build/save_patch_stack.py new file mode 100644 index 0000000000..9b080ac0ff --- /dev/null +++ b/dom/media/webrtc/third_party_build/save_patch_stack.py @@ -0,0 +1,192 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import argparse +import atexit +import os +import re +import shutil +import sys + +from run_operations import run_git, run_hg, run_shell + +# This script saves the mozilla patch stack and no-op commit tracking +# files. This makes our fast-forward process much more resilient by +# saving the intermediate state after each upstream commit is processed. + +error_help = None +script_name = os.path.basename(__file__) + + +@atexit.register +def early_exit_handler(): + print("*** ERROR *** {} did not complete successfully".format(script_name)) + if error_help is not None: + print(error_help) + + +def save_patch_stack( + github_path, + github_branch, + patch_directory, + state_directory, + target_branch_head, + bug_number, +): + # remove the current patch files + files_to_remove = os.listdir(patch_directory) + for file in files_to_remove: + os.remove(os.path.join(patch_directory, file)) + + # find the base of the patch stack + cmd = "git merge-base {} {}".format(github_branch, target_branch_head) + stdout_lines = run_git(cmd, github_path) + merge_base = stdout_lines[0] + + # grab patch stack + cmd = "git format-patch --keep-subject --no-signature --output-directory {} {}..{}".format( + patch_directory, merge_base, github_branch + ) + run_git(cmd, github_path) + + # remove the commit summary from the file name + patches_to_rename = os.listdir(patch_directory) + for file in patches_to_rename: + shortened_name = re.sub("^(\d\d\d\d)-.*\.patch", "\\1.patch", file) + os.rename( + os.path.join(patch_directory, file), + os.path.join(patch_directory, shortened_name), + ) + + # remove the unhelpful first line of the patch files that only + # causes diff churn. For reasons why we can't skip creating backup + # files during the in-place editing, see: + # https://stackoverflow.com/questions/5694228/sed-in-place-flag-that-works-both-on-mac-bsd-and-linux + run_shell("sed -i'.bak' -e '1d' {}/*.patch".format(patch_directory)) + run_shell("rm {}/*.patch.bak".format(patch_directory)) + + # it is also helpful to save the no-op-cherry-pick-msg files from + # the state directory so that if we're restoring a patch-stack we + # also restore the possibly consumed no-op tracking files. + no_op_files = [ + path + for path in os.listdir(state_directory) + if re.findall(".*no-op-cherry-pick-msg$", path) + ] + for file in no_op_files: + shutil.copy(os.path.join(state_directory, file), patch_directory) + + # get missing files (that should be marked removed) + cmd = "hg status --no-status --deleted {}".format(patch_directory) + stdout_lines = run_hg(cmd) + if len(stdout_lines) != 0: + cmd = "hg rm {}".format(" ".join(stdout_lines)) + run_hg(cmd) + + # get unknown files (that should be marked added) + cmd = "hg status --no-status --unknown {}".format(patch_directory) + stdout_lines = run_hg(cmd) + if len(stdout_lines) != 0: + cmd = "hg add {}".format(" ".join(stdout_lines)) + run_hg(cmd) + + # if any files are marked for add/remove/modify, commit them + cmd = "hg status --added --removed --modified {}".format(patch_directory) + stdout_lines = run_hg(cmd) + if (len(stdout_lines)) != 0: + print("Updating {} files in {}".format(len(stdout_lines), patch_directory)) + if bug_number is None: + run_hg("hg amend") + else: + run_shell( + "hg commit --message 'Bug {} - updated libwebrtc patch stack'".format( + bug_number + ) + ) + + +if __name__ == "__main__": + default_patch_dir = "third_party/libwebrtc/moz-patch-stack" + default_script_dir = "dom/media/webrtc/third_party_build" + default_state_dir = ".moz-fast-forward" + + parser = argparse.ArgumentParser( + description="Save moz-libwebrtc github patch stack" + ) + parser.add_argument( + "--repo-path", + required=True, + help="path to libwebrtc repo", + ) + parser.add_argument( + "--branch", + default="mozpatches", + help="moz-libwebrtc branch (defaults to mozpatches)", + ) + parser.add_argument( + "--patch-path", + default=default_patch_dir, + help="path to save patches (defaults to {})".format(default_patch_dir), + ) + parser.add_argument( + "--state-path", + default=default_state_dir, + help="path to state directory (defaults to {})".format(default_state_dir), + ) + parser.add_argument( + "--target-branch-head", + required=True, + help="target branch head for fast-forward, should match MOZ_TARGET_UPSTREAM_BRANCH_HEAD in config_env", + ) + parser.add_argument( + "--script-path", + default=default_script_dir, + help="path to script directory (defaults to {})".format(default_script_dir), + ) + parser.add_argument( + "--separate-commit-bug-number", + type=int, + help="integer Bugzilla number (example: 1800920), if provided will write patch stack as separate commit", + ) + parser.add_argument( + "--skip-startup-sanity", + action="store_true", + default=False, + help="skip checking for clean repo and doing the initial verify vendoring", + ) + args = parser.parse_args() + + if not args.skip_startup_sanity: + # make sure the mercurial repo is clean before beginning + error_help = ( + "There are modified or untracked files in the mercurial repo.\n" + "Please start with a clean repo before running {}" + ).format(script_name) + stdout_lines = run_hg("hg status") + if len(stdout_lines) != 0: + sys.exit(1) + + # make sure the github repo exists + error_help = ( + "No moz-libwebrtc github repo found at {}\n" + "Please run restore_patch_stack.py before running {}" + ).format(args.repo_path, script_name) + if not os.path.exists(args.repo_path): + sys.exit(1) + error_help = None + + print("Verifying vendoring before saving patch-stack...") + run_shell("bash {}/verify_vendoring.sh".format(args.script_path), False) + + save_patch_stack( + args.repo_path, + args.branch, + os.path.abspath(args.patch_path), + args.state_path, + args.target_branch_head, + args.separate_commit_bug_number, + ) + + # unregister the exit handler so the normal exit doesn't falsely + # report as an error. + atexit.unregister(early_exit_handler) diff --git a/dom/media/webrtc/third_party_build/update_default_config.sh b/dom/media/webrtc/third_party_build/update_default_config.sh new file mode 100644 index 0000000000..f55ee7a5b1 --- /dev/null +++ b/dom/media/webrtc/third_party_build/update_default_config.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +function show_error_msg() +{ + echo "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +DEFAULT_CONFIG_PATH=dom/media/webrtc/third_party_build/default_config_env + +# use the previous default_config_env to make sure any locally overriden +# settings don't interfere with the update process. +MOZ_CONFIG_PATH=$DEFAULT_CONFIG_PATH source dom/media/webrtc/third_party_build/use_config_env.sh + +if [ "x" = "x$NEW_BUG_NUMBER" ]; then + NEXT_MILESTONE=$(($MOZ_NEXT_LIBWEBRTC_MILESTONE+1)) + echo "" + echo "NEW_BUG_NUMBER is not defined. Please use the bug number created for" + echo "updating the libwebrtc library to version $NEXT_MILESTONE." + echo "For more information about creating this bug, see the first step of the" + echo "prerequisites section of our wiki:" + echo "https://wiki.mozilla.org/Media/WebRTC/libwebrtc_Update_Process#Prerequisites" + echo "" + echo " NEW_BUG_NUMBER={v$NEXT_MILESTONE-bug-number} bash $0" + echo "" + exit +fi + +if [ "x$MOZ_NEXT_LIBWEBRTC_MILESTONE" = "x" ]; then + echo "MOZ_NEXT_LIBWEBRTC_MILESTONE is not defined, see README.md" + exit +fi + +if [ "x$MOZ_NEXT_FIREFOX_REL_TARGET" = "x" ]; then + echo "MOZ_NEXT_FIREFOX_REL_TARGET is not defined, see README.md" + exit +fi + + +# After this point: +# * eE: All commands should succeed. +# * u: All variables should be defined before use. +# * o pipefail: All stages of all pipes should succeed. +set -eEuo pipefail + +ERROR_HELP=$" +An error has occurred running $SCRIPT_DIR/write_default_config.py +" +MOZCONFIG=dom/media/webrtc/third_party_build/default_mozconfig \ + ./mach python $SCRIPT_DIR/write_default_config.py \ + --prior-bug-number $MOZ_FASTFORWARD_BUG \ + --bug-number $NEW_BUG_NUMBER \ + --milestone $MOZ_NEXT_LIBWEBRTC_MILESTONE \ + --release-target $MOZ_NEXT_FIREFOX_REL_TARGET \ + --output-path $SCRIPT_DIR/default_config_env + +# source our newly updated default_config_env so we can use the new settings +# to automatically commit the updated file. +source $DEFAULT_CONFIG_PATH + +hg commit -m \ + "Bug $MOZ_FASTFORWARD_BUG - updated default_config_env for v$MOZ_NEXT_LIBWEBRTC_MILESTONE" \ + $DEFAULT_CONFIG_PATH diff --git a/dom/media/webrtc/third_party_build/use_config_env.sh b/dom/media/webrtc/third_party_build/use_config_env.sh new file mode 100644 index 0000000000..bd78dc7e0e --- /dev/null +++ b/dom/media/webrtc/third_party_build/use_config_env.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +# Assume that if STATE_DIR is already defined, we do not need to +# execute this file again. +if [ "x$STATE_DIR" != "x" ]; then + # no need to run script since we've already run + return +fi + +export SCRIPT_DIR="dom/media/webrtc/third_party_build" +# first, make sure we're running from the top of moz-central repo +if [ ! -d $SCRIPT_DIR ]; then + echo "Error: unable to find directory $SCRIPT_DIR" + exit 1 +fi + +# Should we tie the location of the STATE_DIR to the path +# in MOZ_CONFIG_PATH? Probably. +export STATE_DIR=`pwd`/.moz-fast-forward +export LOG_DIR=$STATE_DIR/logs +export TMP_DIR=$STATE_DIR/tmp + +if [ ! -d $STATE_DIR ]; then + echo "Creating missing $STATE_DIR" + mkdir -p $STATE_DIR + if [ ! -d $STATE_DIR ]; then + echo "error: unable to find (or create) $STATE_DIR" + exit 1 + fi +fi +echo "Using STATE_DIR=$STATE_DIR" + +if [ ! -d $LOG_DIR ]; then + echo "Creating missing $LOG_DIR" + mkdir -p $LOG_DIR + if [ ! -d $LOG_DIR ]; then + echo "error: unable to find (or create) $LOG_DIR" + exit 1 + fi +fi +echo "Using LOG_DIR=$LOG_DIR" + +if [ ! -d $TMP_DIR ]; then + echo "Creating missing $TMP_DIR" + mkdir -p $TMP_DIR + if [ ! -d $TMP_DIR ]; then + echo "error: unable to find (or create) $TMP_DIR" + exit 1 + fi +fi +echo "Using TMP_DIR=$TMP_DIR" + +# Allow user to override default path to config_env +if [ "x$MOZ_CONFIG_PATH" = "x" ]; then + MOZ_CONFIG_PATH=$STATE_DIR/config_env + echo "Using default MOZ_CONFIG_PATH=$MOZ_CONFIG_PATH" +fi + +if [ ! -f $MOZ_CONFIG_PATH ]; then + echo "Creating default config file at $MOZ_CONFIG_PATH" + cp $SCRIPT_DIR/default_config_env $MOZ_CONFIG_PATH +fi +source $MOZ_CONFIG_PATH + + +function find_base_commit() +{ + # read the last line of README.moz-ff-commit to retrieve our current base + # commit in moz-libwebrtc + MOZ_LIBWEBRTC_BASE=`tail -1 third_party/libwebrtc/README.moz-ff-commit` + echo "prelim MOZ_LIBWEBRTC_BASE: $MOZ_LIBWEBRTC_BASE" + # if we've advanced into a chrome release branch, we need to adjust the + # MOZ_LIBWEBRTC_BASE to the last common commit so we can now advance up + # the trunk commits. + MOZ_LIBWEBRTC_BASE=`cd $MOZ_LIBWEBRTC_SRC ; git merge-base $MOZ_LIBWEBRTC_BASE $MOZ_TARGET_UPSTREAM_BRANCH_HEAD` + # now make it a short hash + MOZ_LIBWEBRTC_BASE=`cd $MOZ_LIBWEBRTC_SRC ; git rev-parse --short $MOZ_LIBWEBRTC_BASE` + echo "adjusted MOZ_LIBWEBRTC_BASE: $MOZ_LIBWEBRTC_BASE" +} +export -f find_base_commit + +function find_next_commit() +{ + # identify the next commit above our current base commit + MOZ_LIBWEBRTC_NEXT_BASE=`cd $MOZ_LIBWEBRTC_SRC ; \ + git log --oneline --ancestry-path $MOZ_LIBWEBRTC_BASE^..$MOZ_TARGET_UPSTREAM_BRANCH_HEAD \ + | tail -2 | head -1 | awk '{print $1;}'` +} +export -f find_next_commit diff --git a/dom/media/webrtc/third_party_build/vendor-libwebrtc.py b/dom/media/webrtc/third_party_build/vendor-libwebrtc.py new file mode 100644 index 0000000000..d820d8c006 --- /dev/null +++ b/dom/media/webrtc/third_party_build/vendor-libwebrtc.py @@ -0,0 +1,432 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import argparse +import datetime +import os +import shutil +import stat +import subprocess +import sys +import tarfile + +import requests + +THIRDPARTY_USED_IN_FIREFOX = [ + "abseil-cpp", + "pffft", + "rnnoise", +] + +LIBWEBRTC_DIR = os.path.normpath("third_party/libwebrtc") + + +# Files in this list are excluded. +def get_excluded_files(): + return [ + ".clang-format", + ".git-blame-ignore-revs", + ".gitignore", + ".vpython", + "CODE_OF_CONDUCT.md", + "ENG_REVIEW_OWNERS", + "PRESUBMIT.py", + "README.chromium", + "WATCHLISTS", + "codereview.settings", + "license_template.txt", + "native-api.md", + "presubmit_test.py", + "presubmit_test_mocks.py", + "pylintrc", + ] + + +# Directories in this list are excluded. Directories are handled +# separately from files so that script 'filter_git_changes.py' can use +# different regex handling for directory paths. +def get_excluded_dirs(): + return [ + # Only the camera code under sdk/android/api/org/webrtc is used, so + # we remove sdk/android and add back the specific files we want. + "sdk/android", + ] + + +# Paths in this list are included even if their parent directory is +# excluded in get_excluded_dirs() +def get_included_path_overrides(): + return [ + "sdk/android/src/java/org/webrtc/NativeLibrary.java", + "sdk/android/src/java/org/webrtc/FramerateBitrateAdjuster.java", + "sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java", + "sdk/android/src/java/org/webrtc/BitrateAdjuster.java", + "sdk/android/src/java/org/webrtc/MediaCodecWrapperFactory.java", + "sdk/android/src/java/org/webrtc/WebRtcClassLoader.java", + "sdk/android/src/java/org/webrtc/audio/WebRtcAudioRecord.java", + "sdk/android/src/java/org/webrtc/audio/WebRtcAudioTrack.java", + "sdk/android/src/java/org/webrtc/audio/WebRtcAudioManager.java", + "sdk/android/src/java/org/webrtc/audio/LowLatencyAudioBufferManager.java", + "sdk/android/src/java/org/webrtc/audio/WebRtcAudioUtils.java", + "sdk/android/src/java/org/webrtc/audio/WebRtcAudioEffects.java", + "sdk/android/src/java/org/webrtc/audio/VolumeLogger.java", + "sdk/android/src/java/org/webrtc/NativeCapturerObserver.java", + "sdk/android/src/java/org/webrtc/MediaCodecWrapper.java", + "sdk/android/src/java/org/webrtc/CalledByNative.java", + "sdk/android/src/java/org/webrtc/Histogram.java", + "sdk/android/src/java/org/webrtc/EglBase10Impl.java", + "sdk/android/src/java/org/webrtc/EglBase14Impl.java", + "sdk/android/src/java/org/webrtc/MediaCodecWrapperFactoryImpl.java", + "sdk/android/src/java/org/webrtc/AndroidVideoDecoder.java", + "sdk/android/src/java/org/webrtc/BaseBitrateAdjuster.java", + "sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java", + "sdk/android/src/java/org/webrtc/VideoCodecMimeType.java", + "sdk/android/src/java/org/webrtc/NativeAndroidVideoTrackSource.java", + "sdk/android/src/java/org/webrtc/VideoDecoderWrapper.java", + "sdk/android/src/java/org/webrtc/JNILogging.java", + "sdk/android/src/java/org/webrtc/CameraCapturer.java", + "sdk/android/src/java/org/webrtc/CameraSession.java", + "sdk/android/src/java/org/webrtc/H264Utils.java", + "sdk/android/src/java/org/webrtc/Empty.java", + "sdk/android/src/java/org/webrtc/DynamicBitrateAdjuster.java", + "sdk/android/src/java/org/webrtc/Camera1Session.java", + "sdk/android/src/java/org/webrtc/JniCommon.java", + "sdk/android/src/java/org/webrtc/NV12Buffer.java", + "sdk/android/src/java/org/webrtc/WrappedNativeI420Buffer.java", + "sdk/android/src/java/org/webrtc/GlGenericDrawer.java", + "sdk/android/src/java/org/webrtc/RefCountDelegate.java", + "sdk/android/src/java/org/webrtc/Camera2Session.java", + "sdk/android/src/java/org/webrtc/MediaCodecUtils.java", + "sdk/android/src/java/org/webrtc/CalledByNativeUnchecked.java", + "sdk/android/src/java/org/webrtc/VideoEncoderWrapper.java", + "sdk/android/src/java/org/webrtc/NV21Buffer.java", + "sdk/android/api/org/webrtc/RendererCommon.java", + "sdk/android/api/org/webrtc/RenderSynchronizer.java", + "sdk/android/api/org/webrtc/YuvHelper.java", + "sdk/android/api/org/webrtc/LibvpxVp9Encoder.java", + "sdk/android/api/org/webrtc/Metrics.java", + "sdk/android/api/org/webrtc/CryptoOptions.java", + "sdk/android/api/org/webrtc/MediaConstraints.java", + "sdk/android/api/org/webrtc/YuvConverter.java", + "sdk/android/api/org/webrtc/JavaI420Buffer.java", + "sdk/android/api/org/webrtc/VideoDecoder.java", + "sdk/android/api/org/webrtc/WrappedNativeVideoDecoder.java", + "sdk/android/api/org/webrtc/Camera2Enumerator.java", + "sdk/android/api/org/webrtc/SurfaceTextureHelper.java", + "sdk/android/api/org/webrtc/EglBase10.java", + "sdk/android/api/org/webrtc/DataChannel.java", + "sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java", + "sdk/android/api/org/webrtc/audio/AudioDeviceModule.java", + "sdk/android/api/org/webrtc/SessionDescription.java", + "sdk/android/api/org/webrtc/GlUtil.java", + "sdk/android/api/org/webrtc/VideoSource.java", + "sdk/android/api/org/webrtc/AudioTrack.java", + "sdk/android/api/org/webrtc/EglRenderer.java", + "sdk/android/api/org/webrtc/EglThread.java", + "sdk/android/api/org/webrtc/VideoEncoder.java", + "sdk/android/api/org/webrtc/VideoCapturer.java", + "sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java", + "sdk/android/api/org/webrtc/AudioSource.java", + "sdk/android/api/org/webrtc/GlRectDrawer.java", + "sdk/android/api/org/webrtc/StatsReport.java", + "sdk/android/api/org/webrtc/CameraVideoCapturer.java", + "sdk/android/api/org/webrtc/NetEqFactoryFactory.java", + "sdk/android/api/org/webrtc/AudioProcessingFactory.java", + "sdk/android/api/org/webrtc/Camera2Capturer.java", + "sdk/android/api/org/webrtc/ScreenCapturerAndroid.java", + "sdk/android/api/org/webrtc/RefCounted.java", + "sdk/android/api/org/webrtc/VideoEncoderFallback.java", + "sdk/android/api/org/webrtc/AudioEncoderFactoryFactory.java", + "sdk/android/api/org/webrtc/EglBase14.java", + "sdk/android/api/org/webrtc/SoftwareVideoEncoderFactory.java", + "sdk/android/api/org/webrtc/VideoEncoderFactory.java", + "sdk/android/api/org/webrtc/StatsObserver.java", + "sdk/android/api/org/webrtc/PlatformSoftwareVideoDecoderFactory.java", + "sdk/android/api/org/webrtc/Camera1Capturer.java", + "sdk/android/api/org/webrtc/AddIceObserver.java", + "sdk/android/api/org/webrtc/SurfaceViewRenderer.java", + "sdk/android/api/org/webrtc/CameraEnumerator.java", + "sdk/android/api/org/webrtc/CameraEnumerationAndroid.java", + "sdk/android/api/org/webrtc/VideoDecoderFallback.java", + "sdk/android/api/org/webrtc/FileVideoCapturer.java", + "sdk/android/api/org/webrtc/NativeLibraryLoader.java", + "sdk/android/api/org/webrtc/Camera1Enumerator.java", + "sdk/android/api/org/webrtc/NativePeerConnectionFactory.java", + "sdk/android/api/org/webrtc/LibaomAv1Encoder.java", + "sdk/android/api/org/webrtc/BuiltinAudioEncoderFactoryFactory.java", + "sdk/android/api/org/webrtc/AudioDecoderFactoryFactory.java", + "sdk/android/api/org/webrtc/FecControllerFactoryFactoryInterface.java", + "sdk/android/api/org/webrtc/VideoFrameBufferType.java", + "sdk/android/api/org/webrtc/SdpObserver.java", + "sdk/android/api/org/webrtc/Predicate.java", + "sdk/android/api/org/webrtc/VideoFileRenderer.java", + "sdk/android/api/org/webrtc/WrappedNativeVideoEncoder.java", + "sdk/android/api/org/webrtc/LibvpxVp8Encoder.java", + "sdk/android/api/org/webrtc/DtmfSender.java", + "sdk/android/api/org/webrtc/VideoTrack.java", + "sdk/android/api/org/webrtc/LibvpxVp8Decoder.java", + "sdk/android/api/org/webrtc/GlShader.java", + "sdk/android/api/org/webrtc/FrameEncryptor.java", + "sdk/android/api/org/webrtc/EglBase.java", + "sdk/android/api/org/webrtc/VideoProcessor.java", + "sdk/android/api/org/webrtc/SSLCertificateVerifier.java", + "sdk/android/api/org/webrtc/VideoSink.java", + "sdk/android/api/org/webrtc/MediaSource.java", + "sdk/android/api/org/webrtc/DefaultVideoDecoderFactory.java", + "sdk/android/api/org/webrtc/VideoCodecInfo.java", + "sdk/android/api/org/webrtc/FrameDecryptor.java", + "sdk/android/api/org/webrtc/VideoDecoderFactory.java", + "sdk/android/api/org/webrtc/TextureBufferImpl.java", + "sdk/android/api/org/webrtc/VideoFrame.java", + "sdk/android/api/org/webrtc/IceCandidateErrorEvent.java", + "sdk/android/api/org/webrtc/CapturerObserver.java", + "sdk/android/api/org/webrtc/MediaStreamTrack.java", + "sdk/android/api/org/webrtc/GlTextureFrameBuffer.java", + "sdk/android/api/org/webrtc/TurnCustomizer.java", + "sdk/android/api/org/webrtc/TimestampAligner.java", + "sdk/android/api/org/webrtc/BuiltinAudioDecoderFactoryFactory.java", + "sdk/android/api/org/webrtc/LibvpxVp9Decoder.java", + "sdk/android/api/org/webrtc/SurfaceEglRenderer.java", + "sdk/android/api/org/webrtc/HardwareVideoDecoderFactory.java", + "sdk/android/api/org/webrtc/VideoCodecStatus.java", + "sdk/android/api/org/webrtc/Dav1dDecoder.java", + "sdk/android/api/org/webrtc/VideoFrameDrawer.java", + "sdk/android/api/org/webrtc/CallSessionFileRotatingLogSink.java", + "sdk/android/api/org/webrtc/EncodedImage.java", + ] + + +def make_github_url(repo, commit): + if not repo.endswith("/"): + repo += "/" + return repo + "archive/" + commit + ".tar.gz" + + +def make_googlesource_url(target, commit): + if target == "libwebrtc": + return "https://webrtc.googlesource.com/src.git/+archive/" + commit + ".tar.gz" + elif target == "build": + return ( + "https://chromium.googlesource.com/chromium/src/build/+archive/" + + commit + + ".tar.gz" + ) + elif target == "third_party": + return ( + "https://chromium.googlesource.com/chromium/src/third_party/+archive/" + + commit + + ".tar.gz" + ) + + +def fetch(target, url): + print("Fetching commit from {}".format(url)) + req = requests.get(url) + if req.status_code == 200: + with open(target + ".tar.gz", "wb") as f: + f.write(req.content) + else: + print( + "Hit status code {} fetching commit. Aborting.".format(req.status_code), + file=sys.stderr, + ) + sys.exit(1) + with open(os.path.join(LIBWEBRTC_DIR, "README.mozilla"), "a") as f: + # write the the command line used + f.write("# ./mach python {}\n".format(" ".join(sys.argv[0:]))) + f.write( + "{} updated from commit {} on {}.\n".format( + target, url, datetime.datetime.utcnow().isoformat() + ) + ) + + +def fetch_local(target, path, commit): + target_archive = target + ".tar.gz" + cp = subprocess.run(["git", "archive", "-o", target_archive, commit], cwd=path) + if cp.returncode != 0: + print( + "Hit return code {} fetching commit. Aborting.".format(cp.returncode), + file=sys.stderr, + ) + sys.exit(1) + + with open(os.path.join(LIBWEBRTC_DIR, "README.mozilla"), "a") as f: + # write the the command line used + f.write("# ./mach python {}\n".format(" ".join(sys.argv[0:]))) + f.write( + "{} updated from {} commit {} on {}.\n".format( + target, path, commit, datetime.datetime.utcnow().isoformat() + ) + ) + shutil.move(os.path.join(path, target_archive), target_archive) + + +def validate_tar_member(member, path): + def _is_within_directory(directory, target): + real_directory = os.path.realpath(directory) + real_target = os.path.realpath(target) + prefix = os.path.commonprefix([real_directory, real_target]) + return prefix == real_directory + + member_path = os.path.join(path, member.name) + if not _is_within_directory(path, member_path): + raise Exception("Attempted path traversal in tar file: " + member.name) + if member.issym(): + link_path = os.path.join(os.path.dirname(member_path), member.linkname) + if not _is_within_directory(path, link_path): + raise Exception("Attempted link path traversal in tar file: " + member.name) + if member.mode & (stat.S_ISUID | stat.S_ISGID): + raise Exception("Attempted setuid or setgid in tar file: " + member.name) + + +def safe_extract(tar, path=".", *, numeric_owner=False): + def _files(tar, path): + for member in tar: + validate_tar_member(member, path) + yield member + + tar.extractall(path, members=_files(tar, path), numeric_owner=numeric_owner) + + +def unpack(target): + target_archive = target + ".tar.gz" + target_path = "tmp-" + target + try: + shutil.rmtree(target_path) + except FileNotFoundError: + pass + with tarfile.open(target_archive) as t: + safe_extract(t, path=target_path) + + if target == "libwebrtc": + # use the top level directories from the tarfile and + # delete those directories in LIBWEBRTC_DIR + libwebrtc_used_in_firefox = os.listdir(target_path) + for path in libwebrtc_used_in_firefox: + try: + shutil.rmtree(os.path.join(LIBWEBRTC_DIR, path)) + except FileNotFoundError: + pass + except NotADirectoryError: + pass + + unused_libwebrtc_in_firefox = get_excluded_files() + get_excluded_dirs() + forced_used_in_firefox = get_included_path_overrides() + + # adjust target_path if GitHub packaging is involved + if not os.path.exists(os.path.join(target_path, libwebrtc_used_in_firefox[0])): + # GitHub packs everything inside a separate directory + target_path = os.path.join(target_path, os.listdir(target_path)[0]) + + # remove any entries found in unused_libwebrtc_in_firefox from the + # tarfile + for path in unused_libwebrtc_in_firefox: + if os.path.isdir(os.path.join(target_path, path)): + shutil.rmtree(os.path.join(target_path, path)) + else: + os.remove(os.path.join(target_path, path)) + + # move remaining top level entries from the tarfile to LIBWEBRTC_DIR + for path in os.listdir(target_path): + shutil.move( + os.path.join(target_path, path), os.path.join(LIBWEBRTC_DIR, path) + ) + + # An easy, but inefficient way to accomplish including specific + # files from directories otherwise removed. Re-extract the tar + # file, and only copy over the exact files requested. + shutil.rmtree(target_path) + with tarfile.open(target_archive) as t: + safe_extract(t, path=target_path) + + # Copy the force included files. Note: the instinctual action + # is to do this prior to removing the excluded paths to avoid + # reextracting the tar file. However, this causes errors due to + # pre-existing paths when other directories are moved out of the + # tar file in the "move all the top level entries from the + # tarfile" phase above. + for path in forced_used_in_firefox: + dest_path = os.path.join(LIBWEBRTC_DIR, path) + dir_path = os.path.dirname(dest_path) + if not os.path.exists(dir_path): + os.makedirs(dir_path) + shutil.move(os.path.join(target_path, path), dest_path) + elif target == "build": + try: + shutil.rmtree(os.path.join(LIBWEBRTC_DIR, "build")) + except FileNotFoundError: + pass + os.makedirs(os.path.join(LIBWEBRTC_DIR, "build")) + + if os.path.exists(os.path.join(target_path, "linux")): + for path in os.listdir(target_path): + shutil.move( + os.path.join(target_path, path), + os.path.join(LIBWEBRTC_DIR, "build", path), + ) + else: + # GitHub packs everything inside a separate directory + target_path = os.path.join(target_path, os.listdir(target_path)[0]) + for path in os.listdir(target_path): + shutil.move( + os.path.join(target_path, path), + os.path.join(LIBWEBRTC_DIR, "build", path), + ) + elif target == "third_party": + # Only delete the THIRDPARTY_USED_IN_FIREFOX paths from + # LIBWEBRTC_DIR/third_party to avoid deleting directories that + # we use to trampoline to libraries already in mozilla's tree. + for path in THIRDPARTY_USED_IN_FIREFOX: + try: + shutil.rmtree(os.path.join(LIBWEBRTC_DIR, "third_party", path)) + except FileNotFoundError: + pass + except NotADirectoryError: + pass + + if os.path.exists(os.path.join(target_path, THIRDPARTY_USED_IN_FIREFOX[0])): + for path in THIRDPARTY_USED_IN_FIREFOX: + shutil.move( + os.path.join(target_path, path), + os.path.join(LIBWEBRTC_DIR, "third_party", path), + ) + else: + # GitHub packs everything inside a separate directory + target_path = os.path.join(target_path, os.listdir(target_path)[0]) + for path in THIRDPARTY_USED_IN_FIREFOX: + shutil.move( + os.path.join(target_path, path), + os.path.join(LIBWEBRTC_DIR, "third_party", path), + ) + + +def cleanup(target): + os.remove(target + ".tar.gz") + shutil.rmtree("tmp-" + target) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Update libwebrtc") + parser.add_argument("target", choices=("libwebrtc", "build", "third_party")) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--from-github", type=str) + group.add_argument("--from-googlesource", action="store_true", default=False) + group.add_argument("--from-local", type=str) + parser.add_argument("--commit", type=str, default="master") + parser.add_argument("--skip-fetch", action="store_true", default=False) + parser.add_argument("--skip-cleanup", action="store_true", default=False) + args = parser.parse_args() + + os.makedirs(LIBWEBRTC_DIR, exist_ok=True) + + if not args.skip_fetch: + if args.from_github: + fetch(args.target, make_github_url(args.from_github, args.commit)) + elif args.from_googlesource: + fetch(args.target, make_googlesource_url(args.target, args.commit)) + elif args.from_local: + fetch_local(args.target, args.from_local, args.commit) + unpack(args.target) + if not args.skip_cleanup: + cleanup(args.target) diff --git a/dom/media/webrtc/third_party_build/vendor_and_commit.py b/dom/media/webrtc/third_party_build/vendor_and_commit.py new file mode 100644 index 0000000000..4de667c6f9 --- /dev/null +++ b/dom/media/webrtc/third_party_build/vendor_and_commit.py @@ -0,0 +1,308 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import argparse +import atexit +import os +import re + +from filter_git_changes import filter_git_changes +from run_operations import get_last_line, run_hg, run_shell, update_resume_state + +# This script vendors moz-libwebrtc, handles add/deletes/renames and +# commits the newly vendored code with the provided commit message. + +error_help = None +script_name = os.path.basename(__file__) + + +def early_exit_handler(): + print("*** ERROR *** {} did not complete successfully".format(script_name)) + if error_help is not None: + print(error_help) + print( + "Please resolve the error and then continue running {}".format(script_name) + ) + + +def log_output_lines(lines, log_dir, filename): + if len(lines) == 0: + return + + with open(os.path.join(log_dir, filename), "w") as ofile: + for line in lines: + ofile.write(line) + ofile.write("\n") + + +def vendor_current_stack(github_branch, github_path, script_dir): + print("-------") + print("------- Vendor {} from {}".format(github_branch, github_path)) + print("-------") + cmd = [ + "./mach", + "python", + "{}/vendor-libwebrtc.py".format(script_dir), + "--from-local", + github_path, + "--commit", + github_branch, + "libwebrtc", + ] + cmd = " ".join(cmd) + run_shell(cmd) + + +def restore_mozbuild_files(target_dir, log_dir): + print("-------") + print("------- Restore moz.build files from repo") + print("-------") + cmd = 'hg revert --include "{}/**moz.build" {}'.format(target_dir, target_dir) + stdout_lines = run_shell(cmd) # run_shell to allow file wildcard + log_output_lines(stdout_lines, log_dir, "log-regen-mozbuild-files.txt") + + +def remove_deleted_upstream_files( + github_path, github_sha, target_dir, log_dir, handle_noop_commit +): + if handle_noop_commit: + return + deleted_paths = filter_git_changes(github_path, github_sha, "D") + deleted_paths = [re.sub("^.\t", "", x) for x in deleted_paths] + deleted_paths = [os.path.join(target_dir, x) for x in deleted_paths] + if len(deleted_paths) != 0: + print("-------") + print("------- Remove deleted upstream files") + print("-------") + cmd = "hg rm {}".format(" ".join(deleted_paths)) + stdout_lines = run_hg(cmd) + log_output_lines(stdout_lines, log_dir, "log-deleted-upstream-files.txt") + + +def add_new_upstream_files( + github_path, github_sha, target_dir, log_dir, handle_noop_commit +): + if handle_noop_commit: + return + added_paths = filter_git_changes(github_path, github_sha, "A") + added_paths = [re.sub("^.\t", "", x) for x in added_paths] + added_paths = [os.path.join(target_dir, x) for x in added_paths] + if len(added_paths) != 0: + print("-------") + print("------- Add new upstream files") + print("-------") + cmd = "hg add {}".format(" ".join(added_paths)) + stdout_lines = run_hg(cmd) + log_output_lines(stdout_lines, log_dir, "log-new-upstream-files.txt") + + +def handle_renamed_upstream_files( + github_path, github_sha, target_dir, log_dir, handle_noop_commit +): + if handle_noop_commit: + return + renamed_paths = filter_git_changes(github_path, github_sha, "R") + renamed_paths = [re.sub("^.[0-9]+\t", "", x) for x in renamed_paths] + renamed_paths = [x.split("\t") for x in renamed_paths] + renamed_paths = [ + " ".join([os.path.join(target_dir, file) for file in rename_pair]) + for rename_pair in renamed_paths + ] + if len(renamed_paths) != 0: + print("-------") + print("------- Handle renamed upstream files") + print("-------") + for x in renamed_paths: + cmd = "hg rename --after {}".format(x) + stdout_lines = run_hg(cmd) + log_output_lines(stdout_lines, log_dir, "log-renamed-upstream-files.txt") + + +def commit_all_changes(github_sha, commit_msg_filename, target_dir): + print("-------") + print("------- Commit vendored changes from {}".format(github_sha)) + print("-------") + cmd = "hg commit -l {} {}".format(commit_msg_filename, target_dir) + run_hg(cmd) + + +def vendor_and_commit( + script_dir, + github_path, + github_branch, + github_sha, + target_dir, + state_dir, + log_dir, + commit_msg_filename, +): + # register the exit handler after the arg parser completes so '--help' doesn't exit with + # an error. + atexit.register(early_exit_handler) + + print("script_dir: {}".format(script_dir)) + print("github_path: {}".format(github_path)) + print("github_branch: {}".format(github_branch)) + print("github_sha: {}".format(github_sha)) + print("target_dir: {}".format(target_dir)) + print("state_dir: {}".format(state_dir)) + print("log_dir: {}".format(log_dir)) + print("commit_msg_filename: {}".format(commit_msg_filename)) + + resume_state_filename = os.path.join(state_dir, "vendor_and_commit.resume") + + noop_commit_path = os.path.join(state_dir, "{}.no-op-cherry-pick-msg").format( + github_sha + ) + handle_noop_commit = os.path.exists(noop_commit_path) + print("noop_commit_path: {}".format(noop_commit_path)) + print("handle_noop_commit: {}".format(handle_noop_commit)) + if handle_noop_commit: + print("***") + print("*** Detected special commit msg, setting handle_noop_commit.") + print("*** This commit is flagged as having been handled by a") + print("*** previous commit, meaning the changed file count between") + print("*** upstream and our vendored commit will not match. The") + print("*** commit message is annotated with info on the previous") + print("*** commit.") + print("***") + + global error_help + resume_state = "" + if os.path.exists(resume_state_filename): + resume_state = get_last_line(resume_state_filename).strip() + print("resume_state: '{}'".format(resume_state)) + + if len(resume_state) == 0: + update_resume_state("resume2", resume_state_filename) + error_help = ( + "Running script '{}/vendor-libwebrtc.py' failed.\n" + "Please manually confirm that all changes from git ({})\n" + "are reflected in the output of 'hg diff'" + ).format(script_dir, github_path) + vendor_current_stack(github_branch, github_path, script_dir) + error_help = None + + if len(resume_state) == 0 or resume_state == "resume2": + resume_state = "" + update_resume_state("resume3", resume_state_filename) + error_help = ( + "An error occurred while restoring moz.build files after vendoring.\n" + "Verify no moz.build files are modified, missing, or changed." + ) + restore_mozbuild_files(target_dir, log_dir) + error_help = None + + if len(resume_state) == 0 or resume_state == "resume3": + resume_state = "" + update_resume_state("resume4", resume_state_filename) + error_help = ( + "An error occurred while removing deleted upstream files.\n" + "Verify files deleted in the newest upstream commit are also\n" + "shown as deleted in the output of 'hg status'" + ) + remove_deleted_upstream_files( + github_path, github_sha, target_dir, log_dir, handle_noop_commit + ) + error_help = None + + if len(resume_state) == 0 or resume_state == "resume4": + resume_state = "" + update_resume_state("resume5", resume_state_filename) + error_help = ( + "An error occurred while adding new upstream files.\n" + "Verify files added in the newest upstream commit are also\n" + "shown as added in the output of 'hg status'" + ) + add_new_upstream_files( + github_path, github_sha, target_dir, log_dir, handle_noop_commit + ) + error_help = None + + if len(resume_state) == 0 or resume_state == "resume5": + resume_state = "" + update_resume_state("resume6", resume_state_filename) + error_help = ( + "An error occurred while adding handling renamed upstream files.\n" + "Verify files renamed in the newest upstream commit are also\n" + "shown as renamed/moved in the output of 'hg status'" + ) + handle_renamed_upstream_files( + github_path, github_sha, target_dir, log_dir, handle_noop_commit + ) + error_help = None + + if len(resume_state) == 0 or resume_state == "resume6": + resume_state = "" + update_resume_state("", resume_state_filename) + error_help = ( + "An error occurred while committing the vendored changes to Mercurial.\n" + ) + commit_all_changes(github_sha, commit_msg_filename, target_dir) + error_help = None + + # unregister the exit handler so the normal exit doesn't falsely + # report as an error. + atexit.unregister(early_exit_handler) + + +if __name__ == "__main__": + default_target_dir = "third_party/libwebrtc" + default_state_dir = ".moz-fast-forward" + default_log_dir = ".moz-fast-forward/logs" + + parser = argparse.ArgumentParser( + description="Vendor from local copy of moz-libwebrtc and commit" + ) + parser.add_argument( + "--repo-path", + required=True, + help="path to libwebrtc repo", + ) + parser.add_argument( + "--script-path", + required=True, + help="path to script directory", + ) + parser.add_argument( + "--commit-sha", + required=True, + help="sha of commit to examine", + ) + parser.add_argument( + "--branch", + default="mozpatches", + help="moz-libwebrtc branch (defaults to mozpatches)", + ) + parser.add_argument( + "--target-path", + default=default_target_dir, + help="target path for vendoring (defaults to {})".format(default_target_dir), + ) + parser.add_argument( + "--state-path", + default=default_state_dir, + help="path to state directory (defaults to {})".format(default_state_dir), + ) + parser.add_argument( + "--log-path", + default=default_log_dir, + help="path to log directory (defaults to {})".format(default_log_dir), + ) + parser.add_argument( + "--commit-msg-path", + required=True, + help="path to file containing commit message", + ) + args = parser.parse_args() + + vendor_and_commit( + args.script_path, + args.repo_path, + args.branch, + args.commit_sha, + args.target_path, # os.path.abspath(args.target_path), + args.state_path, + args.log_path, + args.commit_msg_path, + ) diff --git a/dom/media/webrtc/third_party_build/verify_vendoring.sh b/dom/media/webrtc/third_party_build/verify_vendoring.sh new file mode 100644 index 0000000000..869a2c8c67 --- /dev/null +++ b/dom/media/webrtc/third_party_build/verify_vendoring.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +function show_error_msg() +{ + echo "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +source dom/media/webrtc/third_party_build/use_config_env.sh +export HGPLAIN=1 + +echo "MOZ_LIBWEBRTC_SRC: $MOZ_LIBWEBRTC_SRC" +echo "MOZ_LIBWEBRTC_BRANCH: $MOZ_LIBWEBRTC_BRANCH" +echo "MOZ_FASTFORWARD_BUG: $MOZ_FASTFORWARD_BUG" + +TIP_SHA=`hg id -r tip | awk '{ print $1; }'` +echo "TIP_SHA: $TIP_SHA" + +# we grab the entire firstline description for convenient logging +LAST_PATCHSTACK_UPDATE_COMMIT=`hg log --template "{node|short} {desc|firstline}\n" \ + --include "third_party/libwebrtc/moz-patch-stack/*.patch" | head -1` +echo "LAST_PATCHSTACK_UPDATE_COMMIT: $LAST_PATCHSTACK_UPDATE_COMMIT" + +LAST_PATCHSTACK_UPDATE_COMMIT_SHA=`echo $LAST_PATCHSTACK_UPDATE_COMMIT \ + | awk '{ print $1; }'` +echo "LAST_PATCHSTACK_UPDATE_COMMIT_SHA: $LAST_PATCHSTACK_UPDATE_COMMIT_SHA" + +# grab the oldest, non "Vendor from libwebrtc" line +OLDEST_CANDIDATE_COMMIT=`hg log --template "{node|short} {desc|firstline}\n" \ + -r $LAST_PATCHSTACK_UPDATE_COMMIT_SHA::tip \ + | grep -v "Vendor libwebrtc from" | head -1` +echo "OLDEST_CANDIDATE_COMMIT: $OLDEST_CANDIDATE_COMMIT" + +OLDEST_CANDIDATE_SHA=`echo $OLDEST_CANDIDATE_COMMIT \ + | awk '{ print $1; }'` +echo "OLDEST_CANDIDATE_SHA: $OLDEST_CANDIDATE_SHA" + +EXTRACT_COMMIT_RANGE="{start-commit-sha}::tip" +if [ "x$TIP_SHA" != "x$OLDEST_CANDIDATE_SHA" ]; then + EXTRACT_COMMIT_RANGE="$OLDEST_CANDIDATE_SHA::tip" + echo "EXTRACT_COMMIT_RANGE: $EXTRACT_COMMIT_RANGE" +fi + +# After this point: +# * eE: All commands should succeed. +# * u: All variables should be defined before use. +# * o pipefail: All stages of all pipes should succeed. +set -eEuo pipefail + +./mach python $SCRIPT_DIR/vendor-libwebrtc.py \ + --from-local $MOZ_LIBWEBRTC_SRC \ + --commit $MOZ_LIBWEBRTC_BRANCH \ + libwebrtc + +hg revert -q \ + --include "third_party/libwebrtc/**moz.build" \ + --include "third_party/libwebrtc/README.mozilla" \ + third_party/libwebrtc + +ERROR_HELP=$" +*** +There are changes detected after vendoring libwebrtc from our local copy +of the git repo containing our patch-stack: +$MOZ_LIBWEBRTC_SRC + +Typically this is due to changes made in mercurial to files residing +under third_party/libwebrtc that have not been reflected in +moz-libwebrtc git repo's patch-stack. + +The following commands should help remedy the situation: + ./mach python $SCRIPT_DIR/extract-for-git.py $EXTRACT_COMMIT_RANGE + mv mailbox.patch $MOZ_LIBWEBRTC_SRC + (cd $MOZ_LIBWEBRTC_SRC && \\ + git am mailbox.patch) + +After adding the new changes from mercurial to the moz-libwebrtc +patch stack, you should re-run this command to verify vendoring: + bash $0 +" +FILE_CHANGE_CNT=`hg status third_party/libwebrtc | wc -l | tr -d " "` +if [ "x$FILE_CHANGE_CNT" != "x0" ]; then + echo "$ERROR_HELP" + exit 1 +fi + + +echo "Done - vendoring has been verified." diff --git a/dom/media/webrtc/third_party_build/webrtc.mozbuild b/dom/media/webrtc/third_party_build/webrtc.mozbuild new file mode 100644 index 0000000000..940420e3f8 --- /dev/null +++ b/dom/media/webrtc/third_party_build/webrtc.mozbuild @@ -0,0 +1,48 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file mimics the defines used by any headers in libwebrtc that get +# included in Mozilla code. +# We currently audit these by hand. Searching for the defines upstream is a good +# start: +# https://source.chromium.org/search?q=usage:%23if.*def.*%5B%5E_%5D$%20AND%20path:%5C.h%20AND%20path:%5Ethird_party%2Fwebrtc&ss=chromium + +if CONFIG["MOZ_WEBRTC"]: + DEFINES["HAVE_UINT64_T"] = True + DEFINES["WEBRTC_MOZILLA_BUILD"] = True + DEFINES["RTC_ENABLE_VP9"] = True + + if CONFIG["OS_TARGET"] != "WINNT": + DEFINES["WEBRTC_POSIX"] = True + DEFINES["WEBRTC_BUILD_LIBEVENT"] = True + + if CONFIG["OS_TARGET"] == "Linux": + DEFINES["WEBRTC_LINUX"] = True + if ( + CONFIG["TARGET_CPU"] == "x86" + or CONFIG["TARGET_CPU"] == "x86_64" + or CONFIG["TARGET_CPU"] == "arm" + or CONFIG["TARGET_CPU"] == "aarch64" + or ( + CONFIG["TARGET_ENDIANNESS"] == "little" + and CONFIG["TARGET_CPU"].startswith("mips") + ) + ): + DEFINES["WEBRTC_USE_PIPEWIRE"] = True + elif CONFIG["OS_TARGET"] == "Darwin": + DEFINES["WEBRTC_MAC"] = True + elif CONFIG["OS_TARGET"] == "WINNT": + DEFINES["WEBRTC_WIN"] = True + DEFINES["RTC_ENABLE_WIN_WGC"] = True + DEFINES["HAVE_WINSOCK2_H"] = True + elif CONFIG["OS_TARGET"] in ("DragonFly", "FreeBSD", "NetBSD", "OpenBSD"): + DEFINES["WEBRTC_BSD"] = True + elif CONFIG["OS_TARGET"] == "Android": + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_ANDROID"] = True + + if CONFIG["MOZ_X11"]: + DEFINES["WEBRTC_USE_X11"] = True diff --git a/dom/media/webrtc/third_party_build/write_default_config.py b/dom/media/webrtc/third_party_build/write_default_config.py new file mode 100644 index 0000000000..2339841ff8 --- /dev/null +++ b/dom/media/webrtc/third_party_build/write_default_config.py @@ -0,0 +1,124 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import argparse +import sys +from string import Template + +sys.path.insert(0, "./dom/media/webrtc/third_party_build") +import lookup_branch_head + +text = """#!/bin/bash + +# Edit {path-to} to match the location of your copy of Mozilla's +# fork of libwebrtc (at https://github.com/mozilla/libwebrtc). +export MOZ_LIBWEBRTC_SRC=$$STATE_DIR/moz-libwebrtc + +# The previous fast-forward bug number is used for some error messaging. +export MOZ_PRIOR_FASTFORWARD_BUG="$priorbugnum" + +# Fast-forwarding each Chromium version of libwebrtc should be done +# under a separate bugzilla bug. This bug number is used when crafting +# the commit summary as each upstream commit is vendored into the +# mercurial repository. The bug used for the v106 fast-forward was +# 1800920. +export MOZ_FASTFORWARD_BUG="$bugnum" + +# MOZ_NEXT_LIBWEBRTC_MILESTONE and MOZ_NEXT_FIREFOX_REL_TARGET are +# not used during fast-forward processing, but facilitate generating this +# default config. To generate an default config for the next update, run +# bash dom/media/webrtc/third_party_build/update_default_config_env.sh +export MOZ_NEXT_LIBWEBRTC_MILESTONE=$m2 +export MOZ_NEXT_FIREFOX_REL_TARGET=$t2 + +# For Chromium release branches, see: +# https://chromiumdash.appspot.com/branches + +# Chromium's v$m1 release branch was $bh1. This is used to pre-stack +# the previous release branch's commits onto the appropriate base commit +# (the first common commit between trunk and the release branch). +export MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM="$bh1" + +# New target release branch for v$m2 is branch-heads/$bh2. This is used +# to calculate the next upstream commit. +export MOZ_TARGET_UPSTREAM_BRANCH_HEAD="branch-heads/$bh2" + +# For local development 'mozpatches' is fine for a branch name, but when +# pushing the patch stack to github, it should be named something like +# 'moz-mods-chr$m2-for-rel$t2'. +export MOZ_LIBWEBRTC_BRANCH="mozpatches" + +# After elm has been merged to mozilla-central, the patch stack in +# moz-libwebrtc should be pushed to github. The script +# push_official_branch.sh uses this branch name when pushing to the +# public repo. +export MOZ_LIBWEBRTC_OFFICIAL_BRANCH="moz-mods-chr$m2-for-rel$t2" +""" + + +def build_default_config_env(prior_bug_number, bug_number, milestone, target): + prior_branch_head = lookup_branch_head.get_branch_head(milestone) + if prior_branch_head is None: + sys.exit("error: chromium milestone '{}' is not found.".format(milestone)) + new_branch_head = lookup_branch_head.get_branch_head(milestone + 1) + if new_branch_head is None: + sys.exit( + "error: next chromium milestone '{}' is not found.".format(milestone + 1) + ) + + s = Template(text) + return s.substitute( + priorbugnum=prior_bug_number, + bugnum=bug_number, + m1=milestone, + m2=milestone + 1, + t2=target + 1, + bh1=prior_branch_head, + bh2=new_branch_head, + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Updates the default_config_env file for new release/milestone" + ) + parser.add_argument( + "--prior-bug-number", + required=True, + type=int, + help="integer Bugzilla number (example: 1800920)", + ) + parser.add_argument( + "--bug-number", + required=True, + type=int, + help="integer Bugzilla number (example: 1806510)", + ) + parser.add_argument( + "--milestone", + required=True, + type=int, + help="integer chromium milestone (example: 107)", + ) + parser.add_argument( + "--release-target", + required=True, + type=int, + help="integer firefox release (example: 111)", + ) + parser.add_argument( + "--output-path", + required=True, + help="path name of file to write", + ) + args = parser.parse_args() + + with open(args.output_path, "w") as ofile: + ofile.write( + build_default_config_env( + args.prior_bug_number, + args.bug_number, + args.milestone, + args.release_target, + ) + ) |