diff options
Diffstat (limited to 'dom/media/webrtc/third_party_build')
29 files changed, 2962 insertions, 0 deletions
diff --git a/dom/media/webrtc/third_party_build/README.md b/dom/media/webrtc/third_party_build/README.md new file mode 100644 index 0000000000..9151445c87 --- /dev/null +++ b/dom/media/webrtc/third_party_build/README.md @@ -0,0 +1,17 @@ +# Vendoring libwebrtc and the fast-forward process + +Most of the important information about this process is contained on the fast-forward +automation wiki page +[here](https://wiki.mozilla.org/Media/WebRTC/libwebrtc_Update_Process/automation_plan). + +To skip the history and details and go directly to starting the libwebrtc fast-foward +process, go to the +[Operation Checklist](https://wiki.mozilla.org/Media/WebRTC/libwebrtc_Update_Process/automation_plan#Operation_Checklist). + +# Fixing errors reported in scripts + +In most cases, the scripts report errors including suggestions on how to resolve the +issue. If you're seeing an error message referring you to this README.md file, the +likely issue is that you're missing environment variables that should be set in a +config_env file in .moz-fast-forward. The default for that file can be found at +dom/media/webrtc/third_party_build/default_config_env. diff --git a/dom/media/webrtc/third_party_build/build_no_op_commits.sh b/dom/media/webrtc/third_party_build/build_no_op_commits.sh new file mode 100644 index 0000000000..eff61a5eb5 --- /dev/null +++ b/dom/media/webrtc/third_party_build/build_no_op_commits.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +function show_error_msg() +{ + echo "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +source dom/media/webrtc/third_party_build/use_config_env.sh + +echo "MOZ_LIBWEBRTC_SRC: $MOZ_LIBWEBRTC_SRC" + +# After this point: +# * eE: All commands should succeed. +# * u: All variables should be defined before use. +# * o pipefail: All stages of all pipes should succeed. +set -eEuo pipefail + +CURRENT_DIR=`pwd` +cd $MOZ_LIBWEBRTC_SRC + +MANUAL_INTERVENTION_COMMIT_FILE="$TMP_DIR/manual_commits.txt" +rm -f $MANUAL_INTERVENTION_COMMIT_FILE + +# Find the common commit between our previous work branch and trunk +CURRENT_RELEASE_BASE=`git merge-base branch-heads/$MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM master` + +# Write no-op files for the cherry-picked release branch commits. For more +# details on what this is doing, see make_upstream_revert_noop.sh. +COMMITS=`git log -r $CURRENT_RELEASE_BASE..branch-heads/$MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM --format='%h'` +for commit in $COMMITS; do + + echo "Processing release branch commit $commit for no-op handling" + + # Don't process the commit if the commit message is missing the customary + # line that shows which upstream commit is being cherry-picked. + CNT=`git show $commit | grep "cherry picked from commit" | wc -l | tr -d " " || true` + if [ $CNT != 1 ]; then + # record the commit to list at the end of this script as + # 'needing intervention' + echo " no cherry-pick info found, skipping commit $commit" + echo "$commit" >> $MANUAL_INTERVENTION_COMMIT_FILE + continue + fi + + CHERRY_PICK_COMMIT=`git show $commit | grep "cherry picked from commit" | tr -d "()" | awk '{ print $5; }'` + SHORT_SHA=`git show --name-only $CHERRY_PICK_COMMIT --format='%h' | head -1` + echo " commit $commit cherry-picks $SHORT_SHA" + + echo "We already cherry-picked this when we vendored $commit." \ + > $STATE_DIR/$SHORT_SHA.no-op-cherry-pick-msg + +done + +# This section checks for commits that may have been cherry-picked in more +# than one release branch. +TARGET_RELEASE_BASE=`git merge-base $MOZ_TARGET_UPSTREAM_BRANCH_HEAD master` +NEW_COMMITS=`git log -r $TARGET_RELEASE_BASE..$MOZ_TARGET_UPSTREAM_BRANCH_HEAD --format='%h'` + +# Convert the files that we've already generated for no-op detection into +# something that we can use as a regular expression for searching. +KNOWN_NO_OP_COMMITS=`cd $STATE_DIR ; \ + ls *.no-op-cherry-pick-msg \ + | sed 's/\.no-op-cherry-pick-msg//' \ + | paste -sd '|' /dev/stdin` + +for commit in $NEW_COMMITS; do + + echo "Processing next release branch commit $commit for no-op handling" + + # Don't process the commit if the commit message is missing the customary + # line that shows which upstream commit is being cherry-picked. + CNT=`git show $commit | grep "cherry picked from commit" | wc -l | tr -d " " || true` + if [ $CNT != 1 ]; then + # record the commit to list at the end of this script as + # 'needing intervention' + echo " no cherry-pick info found, skipping commit $commit" + echo "$commit" >> $MANUAL_INTERVENTION_COMMIT_FILE + continue + fi + + CHERRY_PICK_COMMIT=`git show $commit | grep "cherry picked from commit" | tr -d "()" | awk '{ print $5; }'` + SHORT_SHA=`git show --name-only $CHERRY_PICK_COMMIT --format='%h' | head -1` + + # The trick here is that we only want to include no-op processing for the + # commits that appear both here _and_ in the previous release's cherry-pick + # commits. We check the known list of no-op commits to see if it was + # cherry picked in the previous release branch and then create another + # file for the new release branch commit that will ultimately be a no-op. + if [[ "$SHORT_SHA" =~ ^($KNOWN_NO_OP_COMMITS)$ ]]; then + echo " commit $commit cherry-picks $SHORT_SHA" + cp $STATE_DIR/$SHORT_SHA.no-op-cherry-pick-msg $STATE_DIR/$commit.no-op-cherry-pick-msg + fi + +done + +if [ ! -f $MANUAL_INTERVENTION_COMMIT_FILE ]; then + echo "No commits require manual intervention" + exit +fi + +echo $" +Each of the following commits requires manual intervention to +verify the source of the cherry-pick or there may be errors +reported during the fast-forward processing. Without this +intervention, the common symptom is that the vendored commit +file count (0) will not match the upstream commit file count. +" + +for commit in `cat $MANUAL_INTERVENTION_COMMIT_FILE`; do + SUMMARY=`git show --oneline --name-only $commit | head -1` + echo " '$SUMMARY'" +done + +echo $" +To manually create the no-op tracking files needed, +run the following command for each commit in question: + ( export UPSTREAM_COMMIT=\"{sha-of-upstream-commit}\" ; \\ + export PICKED_COMMIT=\"{sha-of-already-used-commit}\" ; \\ + echo \"We already cherry-picked this when we vendored \$PICKED_COMMIT.\" \\ + > $STATE_DIR/\$UPSTREAM_COMMIT.no-op-cherry-pick-msg ) +" diff --git a/dom/media/webrtc/third_party_build/commit-build-file-changes.sh b/dom/media/webrtc/third_party_build/commit-build-file-changes.sh new file mode 100644 index 0000000000..2b4c791fca --- /dev/null +++ b/dom/media/webrtc/third_party_build/commit-build-file-changes.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +function show_error_msg() +{ + echo "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +# All commands should be printed as they are executed +# set -x + +# After this point: +# * eE: All commands should succede. +# * u: All variables should be defined before use. +# * o pipefail: All stages of all pipes should succede. +set -eEuo pipefail + +MOZ_BUILD_CHANGE_CNT=`hg status third_party/libwebrtc | wc -l | tr -d " "` +echo "MOZ_BUILD_CHANGE_CNT: $MOZ_BUILD_CHANGE_CNT" +if [ "x$MOZ_BUILD_CHANGE_CNT" != "x0" ]; then + CURRENT_COMMIT_SHA=`hg id -i | sed 's/+//'` + COMMIT_DESC=`hg --config alias.log=log log -T '{desc|firstline}' -r $CURRENT_COMMIT_SHA` + + # since we have build file changes, touch the CLOBBER file + cat CLOBBER | egrep "^#|^$" > CLOBBER.new + mv CLOBBER.new CLOBBER + echo "Modified build files in third_party/libwebrtc - $COMMIT_DESC" >> CLOBBER + + ADD_CNT=`hg status -nu third_party/libwebrtc | wc -l | tr -d " "` + DEL_CNT=`hg status -nd third_party/libwebrtc | wc -l | tr -d " "` + if [ "x$ADD_CNT" != "x0" ]; then + hg status -nu third_party/libwebrtc | xargs hg add + fi + if [ "x$DEL_CNT" != "x0" ]; then + hg status -nd third_party/libwebrtc | xargs hg rm + fi + + hg commit -m \ + "$COMMIT_DESC - moz.build file updates" \ + third_party/libwebrtc CLOBBER +fi + +echo "Done in $0" diff --git a/dom/media/webrtc/third_party_build/default_config_env b/dom/media/webrtc/third_party_build/default_config_env new file mode 100644 index 0000000000..a50694f0e3 --- /dev/null +++ b/dom/media/webrtc/third_party_build/default_config_env @@ -0,0 +1,42 @@ +#!/bin/bash + +# Edit {path-to} to match the location of your copy of Mozilla's +# fork of libwebrtc (at https://github.com/mozilla/libwebrtc). +export MOZ_LIBWEBRTC_SRC=$STATE_DIR/moz-libwebrtc + +# Fast-forwarding each Chromium version of libwebrtc should be done +# under a separate bugzilla bug. This bug number is used when crafting +# the commit summary as each upstream commit is vendored into the +# mercurial repository. The bug used for the v106 fast-forward was +# 1800920. +export MOZ_FASTFORWARD_BUG="1833237" + +# MOZ_NEXT_LIBWEBRTC_MILESTONE and MOZ_NEXT_FIREFOX_REL_TARGET are +# not used during fast-forward processing, but facilitate generating this +# default config. To generate an default config for the next update, run +# bash dom/media/webrtc/third_party_build/update_default_config_env.sh +export MOZ_NEXT_LIBWEBRTC_MILESTONE=112 +export MOZ_NEXT_FIREFOX_REL_TARGET=116 + +# For Chromium release branches, see: +# https://chromiumdash.appspot.com/branches + +# Chromium's v111 release branch was 5563. This is used to pre-stack +# the previous release branch's commits onto the appropriate base commit +# (the first common commit between trunk and the release branch). +export MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM="5563" + +# New target release branch for v112 is branch-heads/5615. This is used +# to calculate the next upstream commit. +export MOZ_TARGET_UPSTREAM_BRANCH_HEAD="branch-heads/5615" + +# For local development 'mozpatches' is fine for a branch name, but when +# pushing the patch stack to github, it should be named something like +# 'moz-mods-chr112-for-rel116'. +export MOZ_LIBWEBRTC_BRANCH="mozpatches" + +# After elm has been merged to mozilla-central, the patch stack in +# moz-libwebrtc should be pushed to github. The script +# push_official_branch.sh uses this branch name when pushing to the +# public repo. +export MOZ_LIBWEBRTC_OFFICIAL_BRANCH="moz-mods-chr112-for-rel116" diff --git a/dom/media/webrtc/third_party_build/detect_upstream_revert.sh b/dom/media/webrtc/third_party_build/detect_upstream_revert.sh new file mode 100644 index 0000000000..2dc868952d --- /dev/null +++ b/dom/media/webrtc/third_party_build/detect_upstream_revert.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +function show_error_msg() +{ + echo "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +source dom/media/webrtc/third_party_build/use_config_env.sh + +# If DEBUG_GEN is set all commands should be printed as they are executed +if [ ! "x$DEBUG_GEN" = "x" ]; then + set -x +fi + +if [ "x$MOZ_LIBWEBRTC_SRC" = "x" ]; then + echo "MOZ_LIBWEBRTC_SRC is not defined, see README.md" + exit +fi + +if [ -d $MOZ_LIBWEBRTC_SRC ]; then + echo "MOZ_LIBWEBRTC_SRC is $MOZ_LIBWEBRTC_SRC" +else + echo "Path $MOZ_LIBWEBRTC_SRC is not found, see README.md" + exit +fi + +if [ "x$MOZ_LIBWEBRTC_BRANCH" = "x" ]; then + echo "MOZ_LIBWEBRTC_BRANCH is not defined, see README.md" + exit +fi + +if [ "x$AUTO_FIX_REVERT_AS_NOOP" = "x" ]; then + AUTO_FIX_REVERT_AS_NOOP="0" +fi + +find_base_commit +find_next_commit + +MOZ_LIBWEBRTC_COMMIT_MSG=`cd $MOZ_LIBWEBRTC_SRC ; \ +git show --name-only --oneline $MOZ_LIBWEBRTC_NEXT_BASE \ + | head -1 | sed 's/[^ ]* //'` + +echo "MOZ_LIBWEBRTC_BASE: $MOZ_LIBWEBRTC_BASE" +echo "MOZ_LIBWEBRTC_NEXT_BASE: $MOZ_LIBWEBRTC_NEXT_BASE" +echo "MOZ_LIBWEBRTC_COMMIT_MSG: $MOZ_LIBWEBRTC_COMMIT_MSG" +export MOZ_LIBWEBRTC_REVERT_SHA=`cd $MOZ_LIBWEBRTC_SRC ; \ +git log --oneline -r $MOZ_LIBWEBRTC_BASE..$MOZ_TARGET_UPSTREAM_BRANCH_HEAD \ + | grep -F "Revert \"$MOZ_LIBWEBRTC_COMMIT_MSG" \ + | tail -1 | awk '{print $1;}' || true` + +echo "MOZ_LIBWEBRTC_REVERT_SHA: $MOZ_LIBWEBRTC_REVERT_SHA" + +if [ "x$MOZ_LIBWEBRTC_REVERT_SHA" == "x" ]; then + echo "no revert commit detected" + exit +fi + +if [ "x$AUTO_FIX_REVERT_AS_NOOP" = "x1" ]; then + echo "AUTO_FIX_REVERT_AS_NOOP detected, fixing land/revert pair automatically" + bash $SCRIPT_DIR/make_upstream_revert_noop.sh + exit +fi + +echo $" +The next upstream commit has a corresponding future \"Revert\" commit. + +There are 2 common ways forward in this situation: +1. If you're relatively certain there will not be rebase conflicts in the + github repo ($MOZ_LIBWEBRTC_SRC), simply run: + SKIP_NEXT_REVERT_CHK=1 bash $SCRIPT_DIR/loop-ff.sh + +2. The surer method for no rebase conflicts is to cherry-pick both the + next commit, and the commit that reverts the next commit onto the + bottom of our patch stack in github. This pushes the likely rebase + conflict into the future when the upstream fix is relanded, but + ensures we only have to deal with the conflict once. The following + commands will add the necessary commits to the bottom of our patch + stack in github, and leave indicator files in the home directory that + help loop-ff know when to invoke special no-op commit handling: + + MOZ_LIBWEBRTC_BASE=$MOZ_LIBWEBRTC_BASE \\ + MOZ_LIBWEBRTC_NEXT_BASE=$MOZ_LIBWEBRTC_NEXT_BASE \\ + MOZ_LIBWEBRTC_REVERT_SHA=$MOZ_LIBWEBRTC_REVERT_SHA \\ + bash $SCRIPT_DIR/make_upstream_revert_noop.sh + + SKIP_NEXT_REVERT_CHK=1 bash $SCRIPT_DIR/loop-ff.sh +" +exit 1 diff --git a/dom/media/webrtc/third_party_build/elm_arcconfig.patch b/dom/media/webrtc/third_party_build/elm_arcconfig.patch new file mode 100644 index 0000000000..46adb9c6f1 --- /dev/null +++ b/dom/media/webrtc/third_party_build/elm_arcconfig.patch @@ -0,0 +1,10 @@ +diff --git a/.arcconfig b/.arcconfig +--- a/.arcconfig ++++ b/.arcconfig +@@ -1,5 +1,5 @@ + { + "phabricator.uri" : "https://phabricator.services.mozilla.com/", +- "repository.callsign": "MOZILLACENTRAL", ++ "repository.callsign": "ELM", + "history.immutable": false + } diff --git a/dom/media/webrtc/third_party_build/elm_rebase.sh b/dom/media/webrtc/third_party_build/elm_rebase.sh new file mode 100644 index 0000000000..4d53373d35 --- /dev/null +++ b/dom/media/webrtc/third_party_build/elm_rebase.sh @@ -0,0 +1,247 @@ +#!/bin/bash + +# This script exists to help with the rebase process on elm. It rebases +# each patch individually to make it easier to fix rebase conflicts +# without jeopardizing earlier, sucessfully rebased commits. In order to +# limit rebase conflicts around generated moz.build files, it regenerates +# moz.build file commits. It also ensures any commits with 'FLOAT' in the +# commit summary are pushed to the top of the fast-forward stack to help +# the sheriffs more easily merge our commit stack from elm to moz-central. +# +# Occasionally, there will be upstream vendored commits that break the +# build file generation with follow on commits that fix that error. In +# order to allow the rebase process to work more smoothly, it is possible +# to annotate a commit with the string '(skip-generation)' and normal +# build file generation (detected with changes to BUILD.gn files) is +# disabled for that commit. The script outputs instructions for handling +# this situation. +# +# Note: the very first rebase operation will require some manual +# intervention. The user will need to provide, at minimum, the commit that +# corresponds to moz-central upon which the fast-forward stack is based. +# It may also be necessary to provide the first commit of the +# fast-forward stack. Example: +# MOZ_BOTTOM_FF=30f0afb7e4c5 \ +# MOZ_CURRENT_CENTRAL=cad1bd47c273 \ +# bash dom/media/webrtc/third_party_build/elm_rebase.sh +# +# Assumes the top of the fast-forward stack to rebase is the current revision, +# ".". + +function show_error_msg() +{ + echo "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +source dom/media/webrtc/third_party_build/use_config_env.sh + +GENERATION_ERROR=$" +Generating build files has failed. The most common reason for this +failure is that the current commit has an upcoming '(fix-xxxxxx)' commit +that will then allow the build file generation to complete. If the +current situation seems to fit that pattern, adding a line with +'(skip-generation)' to the commit message will ensure that future rebase +operations do not attempt to generate build files for this commit. It may +be as simple as running the following commands: + HGPLAIN=1 hg log -T '{desc}' -r tip > $TMP_DIR/commit_message.txt + ed -s $TMP_DIR/commit_message.txt <<< $'3i\n(skip-generation)\n\n.\nw\nq' + hg commit --amend -l $TMP_DIR/commit_message.txt + bash $0 +" +COMMIT_LIST_FILE=$TMP_DIR/rebase-commit-list.txt +export HGPLAIN=1 + +# After this point: +# * eE: All commands should succeed. +# * o pipefail: All stages of all pipes should succeed. +set -eEo pipefail + +if [ -f $STATE_DIR/rebase_resume_state ]; then + source $STATE_DIR/rebase_resume_state +else + + if [ "x" == "x$MOZ_TOP_FF" ]; then + MOZ_TOP_FF=`hg log -r . -T"{node|short}"` + + ERROR_HELP=$" +The topmost commit to be rebased is not in the public phase. Should it be +pushed to elm first? If this is intentional, please rerun the command and pass +it in explicitly: + MOZ_TOP_FF=$MOZ_TOP_FF bash $0 +" + if [[ $(hg phase -r .) != *public ]]; then + echo "$ERROR_HELP" + exit 1 + fi + ERROR_HELP="" + + ERROR_HELP=$" +The topmost commit to be rebased is public but has descendants. If those +descendants should not be rebased, please rerun the command and pass the commit +in explicitly: + MOZ_TOP_FF=$MOZ_TOP_FF bash $0 +" + if [ "x" != "x$(hg log -r 'descendants(.) and !.' -T'{node|short}')" ]; then + echo "$ERROR_HELP" + exit 1 + fi + ERROR_HELP="" + fi + + ERROR_HELP=$" +An error here is likely because no revision for central is found. +One possible reason for this is this is your first rebase operation. +To 'bootstrap' the first rebase operation, please find the +moz-central commit that the vendoring commits is based on, and +rerun the command: + MOZ_CURRENT_CENTRAL={central-sha} bash $0 + +You may also need to provide the bottom commit of the fast-forward +stack. The bottom commit means the commit following central. This +could be the sha of the .arcconfig commit if it is the bottom commit. +That command looks like: + MOZ_BOTTOM_FF={base-sha} MOZ_CURRENT_CENTRAL={central-sha} bash $0 +" + if [ "x" == "x$MOZ_CURRENT_CENTRAL" ]; then + MOZ_CURRENT_CENTRAL=`hg log -r central -T"{node|short}"` + fi + if [ "x" == "x$MOZ_BOTTOM_FF" ]; then + MOZ_BOTTOM_FF=`hg log -r $MOZ_CURRENT_CENTRAL~-1 -T"{node|short}"` + fi + ERROR_HELP="" + + if [ "x" == "x$MOZ_BOTTOM_FF" ]; then + echo "No value found for the bottom commit of the fast-forward commit stack." + exit 1 + fi + + # After this point: + # * eE: All commands should succeed. + # * u: All variables should be defined before use. + # * o pipefail: All stages of all pipes should succeed. + set -eEuo pipefail + + hg pull central + MOZ_NEW_CENTRAL=`hg log -r central -T"{node|short}"` + + echo "moz-central in elm is currently $MOZ_CURRENT_CENTRAL" + echo "bottom of fast-foward tree is $MOZ_BOTTOM_FF" + echo "top of fast-forward tree (webrtc-fast-forward) is $MOZ_TOP_FF" + echo "new target for elm rebase $MOZ_NEW_CENTRAL (tip of moz-central)" + + hg log -T '{rev}:{node|short} {desc|firstline}\n' \ + -r $MOZ_BOTTOM_FF::$MOZ_TOP_FF > $COMMIT_LIST_FILE + + # move all FLOAT lines to end of file, and delete the "empty" tilde line + # line at the beginning + ed -s $COMMIT_LIST_FILE <<< $'g/- FLOAT -/m$\ng/^~$/d\nw\nq' + + MOZ_BOOKMARK=`date "+webrtc-fast-forward-%Y-%m-%d--%H-%M"` + hg bookmark -r elm $MOZ_BOOKMARK + + hg update $MOZ_NEW_CENTRAL + + # pre-work is complete, let's write out a temporary config file that allows + # us to resume + echo $"export MOZ_CURRENT_CENTRAL=$MOZ_CURRENT_CENTRAL +export MOZ_BOTTOM_FF=$MOZ_BOTTOM_FF +export MOZ_TOP_FF=$MOZ_TOP_FF +export MOZ_NEW_CENTRAL=$MOZ_NEW_CENTRAL +export MOZ_BOOKMARK=$MOZ_BOOKMARK +" > $STATE_DIR/rebase_resume_state +fi # if [ -f $STATE_DIR/rebase_resume_state ]; then ; else + +# grab all commits +COMMITS=`cat $COMMIT_LIST_FILE | awk '{print $1;}'` + +echo -n "Commits: " +for commit in $COMMITS; do +echo -n "$commit " +done +echo "" + +for commit in $COMMITS; do + echo "Processing $commit" + FULL_COMMIT_LINE=`head -1 $COMMIT_LIST_FILE` + + function remove_commit () { + echo "Removing from list '$FULL_COMMIT_LINE'" + ed -s $COMMIT_LIST_FILE <<< $'1d\nw\nq' + } + + IS_BUILD_COMMIT=`hg log -T '{desc|firstline}' -r $commit \ + | grep "file updates" | wc -l | tr -d " " || true` + echo "IS_BUILD_COMMIT: $IS_BUILD_COMMIT" + if [ "x$IS_BUILD_COMMIT" != "x0" ]; then + echo "Skipping $commit:" + hg log -T '{desc|firstline}' -r $commit + remove_commit + continue + fi + + IS_SKIP_GEN_COMMIT=`hg log --verbose \ + -r $commit \ + | grep "skip-generation" | wc -l | tr -d " " || true` + echo "IS_SKIP_GEN_COMMIT: $IS_SKIP_GEN_COMMIT" + + echo "Generate patch for: $commit" + hg export -r $commit > $TMP_DIR/rebase.patch + + echo "Import patch for $commit" + hg import $TMP_DIR/rebase.patch || \ + ( hg log -T '{desc}' -r $commit > $TMP_DIR/rebase_commit_message.txt ; \ + remove_commit ; \ + echo "Error importing: '$FULL_COMMIT_LINE'" ; \ + echo "Please fix import errors, then:" ; \ + echo " hg commit -l $TMP_DIR/rebase_commit_message.txt" ; \ + echo " bash $0" ; \ + exit 1 ) + + remove_commit + + if [ "x$IS_SKIP_GEN_COMMIT" != "x0" ]; then + echo "Skipping build generation for $commit" + continue + fi + + MODIFIED_BUILD_RELATED_FILE_CNT=`hg diff -c tip --stat \ + --include 'third_party/libwebrtc/**BUILD.gn' \ + --include 'third_party/libwebrtc/webrtc.gni' \ + --include 'dom/media/webrtc/third_party_build/gn-configs/webrtc.json' \ + | wc -l | tr -d " "` + echo "MODIFIED_BUILD_RELATED_FILE_CNT: $MODIFIED_BUILD_RELATED_FILE_CNT" + if [ "x$MODIFIED_BUILD_RELATED_FILE_CNT" != "x0" ]; then + echo "Regenerate build files" + ./mach python python/mozbuild/mozbuild/gn_processor.py \ + dom/media/webrtc/third_party_build/gn-configs/webrtc.json || \ + ( echo "$GENERATION_ERROR" ; exit 1 ) + + MOZ_BUILD_CHANGE_CNT=`hg status third_party/libwebrtc \ + --include 'third_party/libwebrtc/**moz.build' | wc -l | tr -d " "` + if [ "x$MOZ_BUILD_CHANGE_CNT" != "x0" ]; then + bash dom/media/webrtc/third_party_build/commit-build-file-changes.sh + NEWEST_COMMIT=`hg log -T '{desc|firstline}' -r tip` + echo "NEWEST_COMMIT: $NEWEST_COMMIT" + echo "NEWEST_COMMIT: $NEWEST_COMMIT" >> $LOG_DIR/rebase-build-changes-commits.log + fi + echo "Done generating build files" + fi + + echo "Done processing $commit" +done + +rm $STATE_DIR/rebase_resume_state + +REMAINING_STEPS=$" +The rebase process is complete. The following steps must be completed manually: + ./mach bootstrap --application=browser --no-system-changes + ./mach build + hg push -r tip --force + hg push -B $MOZ_BOOKMARK +" +echo "$REMAINING_STEPS" diff --git a/dom/media/webrtc/third_party_build/extract-for-git.py b/dom/media/webrtc/third_party_build/extract-for-git.py new file mode 100644 index 0000000000..d2701d7dff --- /dev/null +++ b/dom/media/webrtc/third_party_build/extract-for-git.py @@ -0,0 +1,145 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import argparse +import os +import re +import subprocess + +# This script extracts commits that touch third party webrtc code so they can +# be imported into Git. It filters out commits that are not part of upstream +# code and rewrites the paths to match upstream. Finally, the commits are +# combined into a mailbox file that can be applied with `git am`. +LIBWEBRTC_DIR = "third_party/libwebrtc" + + +def build_commit_list(revset, env): + """Build commit list from the specified revset. + + The revset can be a single revision, e.g. 52bb9bb94661, or a range, + e.g. 8c08a5bb8a99::52bb9bb94661, or any other valid revset + (check hg help revset). Only commits that touch libwebrtc are included. + """ + res = subprocess.run( + ["hg", "log", "-r", revset, "-M", "--template", "{node}\n", LIBWEBRTC_DIR], + capture_output=True, + text=True, + env=env, + ) + return [line.strip() for line in res.stdout.strip().split("\n")] + + +def extract_author_date(sha1, env): + res = subprocess.run( + ["hg", "log", "-r", sha1, "--template", "{author}|{date|isodate}"], + capture_output=True, + text=True, + env=env, + ) + return res.stdout.split("|") + + +def extract_description(sha1, env): + res = subprocess.run( + ["hg", "log", "-r", sha1, "--template", "{desc}"], + capture_output=True, + text=True, + env=env, + ) + return res.stdout + + +def extract_commit(sha1, env): + res = subprocess.run( + ["hg", "log", "-r", sha1, "-pg", "--template", "\n"], + capture_output=True, + text=True, + env=env, + ) + return res.stdout + + +def filter_nonwebrtc(commit): + filtered = [] + skipping = False + for line in commit.split("\n"): + # Extract only patches affecting libwebrtc, but avoid commits that + # touch build, which is tracked by a separate repo, or that affect + # moz.build files which are code generated. + if ( + line.startswith("diff --git a/" + LIBWEBRTC_DIR) + and not line.startswith("diff --git a/" + LIBWEBRTC_DIR + "/build") + and not line.startswith("diff --git a/" + LIBWEBRTC_DIR + "/third_party") + and not line.startswith( + "diff --git a/" + LIBWEBRTC_DIR + "/moz-patch-stack" + ) + and not line.endswith("moz.build") + ): + skipping = False + elif line.startswith("diff --git"): + skipping = True + + if not skipping: + filtered.append(line) + return "\n".join(filtered) + + +def fixup_paths(commit): + # make sure we only rewrite paths in the diff-related or rename lines + commit = re.sub( + f"^rename (from|to) {LIBWEBRTC_DIR}/", "rename \\1 ", commit, flags=re.MULTILINE + ) + return re.sub(f"( [ab])/{LIBWEBRTC_DIR}/", "\\1/", commit) + + +def write_as_mbox(sha1, author, date, description, commit, ofile): + # Use same magic date as git format-patch + ofile.write("From {} Mon Sep 17 00:00:00 2001\n".format(sha1)) + ofile.write("From: {}\n".format(author)) + ofile.write("Date: {}\n".format(date)) + description = description.split("\n") + ofile.write("Subject: {}\n".format(description[0])) + ofile.write("\n".join(description[1:])) + ofile.write( + "\nMercurial Revision: https://hg.mozilla.org/mozilla-central/rev/{}\n".format( + sha1 + ) + ) + ofile.write(commit) + ofile.write("\n") + ofile.write("\n") + + +if __name__ == "__main__": + commits = [] + parser = argparse.ArgumentParser( + description="Format commits for upstream libwebrtc" + ) + parser.add_argument( + "revsets", metavar="revset", type=str, nargs="+", help="A revset to process" + ) + parser.add_argument( + "--target", choices=("libwebrtc", "build", "third_party"), default="libwebrtc" + ) + args = parser.parse_args() + + if args.target != "libwebrtc": + LIBWEBRTC_DIR = os.path.join(LIBWEBRTC_DIR, args.target) + + # must run 'hg' with HGPLAIN=1 to ensure aliases don't interfere with + # command output. + env = os.environ.copy() + env["HGPLAIN"] = "1" + + for revset in args.revsets: + commits.extend(build_commit_list(revset, env)) + + with open("mailbox.patch", "w") as ofile: + for sha1 in commits: + author, date = extract_author_date(sha1, env) + description = extract_description(sha1, env) + filtered_commit = filter_nonwebrtc(extract_commit(sha1, env)) + if len(filtered_commit) == 0: + continue + fixedup_commit = fixup_paths(filtered_commit) + write_as_mbox(sha1, author, date, description, fixedup_commit, ofile) diff --git a/dom/media/webrtc/third_party_build/fast-forward-libwebrtc.sh b/dom/media/webrtc/third_party_build/fast-forward-libwebrtc.sh new file mode 100644 index 0000000000..9ef40a41ac --- /dev/null +++ b/dom/media/webrtc/third_party_build/fast-forward-libwebrtc.sh @@ -0,0 +1,256 @@ +#!/bin/bash + +function show_error_msg() +{ + echo "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +source dom/media/webrtc/third_party_build/use_config_env.sh + +# If DEBUG_GEN is set all commands should be printed as they are executed +if [ ! "x$DEBUG_GEN" = "x" ]; then + set -x +fi + +if [ "x$MOZ_LIBWEBRTC_SRC" = "x" ]; then + echo "MOZ_LIBWEBRTC_SRC is not defined, see README.md" + exit +fi + +if [ -d $MOZ_LIBWEBRTC_SRC ]; then + echo "MOZ_LIBWEBRTC_SRC is $MOZ_LIBWEBRTC_SRC" +else + echo "Path $MOZ_LIBWEBRTC_SRC is not found, see README.md" + exit +fi + +if [ "x$MOZ_LIBWEBRTC_BRANCH" = "x" ]; then + echo "MOZ_LIBWEBRTC_BRANCH is not defined, see README.md" + exit +fi + +if [ "x$HANDLE_NOOP_COMMIT" = "x" ]; then + HANDLE_NOOP_COMMIT="" +fi + +RESUME="" +if [ -f $STATE_DIR/resume_state ]; then + RESUME=`tail -1 $STATE_DIR/resume_state` +fi + +GIT_IS_REBASING=`cd $MOZ_LIBWEBRTC_SRC && git status | grep "interactive rebase in progress" | wc -l | tr -d " " || true` +if [ "x$GIT_IS_REBASING" != "x0" ]; then + echo "There is currently a git rebase operation in progress at $MOZ_LIBWEBRTC_SRC." + echo "Please resolve the rebase before attempting to continue the fast-forward" + echo "operation." + exit 1 +fi + +if [ "x$RESUME" = "x" ]; then + SKIP_TO="run" + # Check for modified files and abort if present. + MODIFIED_FILES=`hg status --exclude "third_party/libwebrtc/**.orig" third_party/libwebrtc` + if [ "x$MODIFIED_FILES" = "x" ]; then + # Completely clean the mercurial checkout before proceeding + hg update -C -r . + hg purge + else + echo "There are modified files in the checkout. Cowardly aborting!" + echo "$MODIFIED_FILES" + exit 1 + fi +else + SKIP_TO=$RESUME + hg revert -C third_party/libwebrtc/README.moz-ff-commit &> /dev/null +fi + +find_base_commit +find_next_commit + +echo "looking for $STATE_DIR/$MOZ_LIBWEBRTC_NEXT_BASE.no-op-cherry-pick-msg" +if [ -f $STATE_DIR/$MOZ_LIBWEBRTC_NEXT_BASE.no-op-cherry-pick-msg ]; then + echo "***" + echo "*** detected special commit msg, setting HANDLE_NOOP_COMMIT" + echo "***" + HANDLE_NOOP_COMMIT="1" +fi + +UPSTREAM_ADDED_FILES="" + +# Grab the filtered changes from git based on what we vendor. +FILTERED_GIT_CHANGES=`./mach python $SCRIPT_DIR/filter_git_changes.py \ + --repo-path $MOZ_LIBWEBRTC_SRC --commit-sha $MOZ_LIBWEBRTC_NEXT_BASE` + +# After this point: +# * eE: All commands should succeed. +# * u: All variables should be defined before use. +# * o pipefail: All stages of all pipes should succeed. +set -eEuo pipefail + +echo " MOZ_LIBWEBRTC_BASE: $MOZ_LIBWEBRTC_BASE" +echo "MOZ_LIBWEBRTC_NEXT_BASE: $MOZ_LIBWEBRTC_NEXT_BASE" +echo "HANDLE_NOOP_COMMIT: $HANDLE_NOOP_COMMIT" +echo " RESUME: $RESUME" +echo "SKIP_TO: $SKIP_TO" + +echo "-------" +echo "------- Write cmd-line to third_party/libwebrtc/README.moz-ff-commit" +echo "-------" +echo "# MOZ_LIBWEBRTC_SRC=$MOZ_LIBWEBRTC_SRC MOZ_LIBWEBRTC_BRANCH=$MOZ_LIBWEBRTC_BRANCH bash $0" \ + >> third_party/libwebrtc/README.moz-ff-commit + +echo "-------" +echo "------- Write new-base to last line of third_party/libwebrtc/README.moz-ff-commit" +echo "-------" +echo "# base of lastest vendoring" >> third_party/libwebrtc/README.moz-ff-commit +echo "$MOZ_LIBWEBRTC_NEXT_BASE" >> third_party/libwebrtc/README.moz-ff-commit + +REBASE_HELP=$" +The rebase operation onto $MOZ_LIBWEBRTC_NEXT_BASE has failed. Please +resolve all the rebase conflicts. To fix this issue, you will need to +jump to the github repo at $MOZ_LIBWEBRTC_SRC . +When the github rebase is complete, re-run the script to resume the +fast-forward process. +" +#"rebase_mozlibwebrtc_stack help for rebase failure" +function rebase_mozlibwebrtc_stack { + echo "-------" + echo "------- Rebase $MOZ_LIBWEBRTC_BRANCH to $MOZ_LIBWEBRTC_NEXT_BASE" + echo "-------" + ERROR_HELP=$REBASE_HELP + ( cd $MOZ_LIBWEBRTC_SRC && \ + git checkout -q $MOZ_LIBWEBRTC_BRANCH && \ + git rebase $MOZ_LIBWEBRTC_NEXT_BASE \ + &> $LOG_DIR/log-rebase-moz-libwebrtc.txt \ + ) + ERROR_HELP="" +} + +function vendor_off_next_commit { + echo "-------" + echo "------- Vendor $MOZ_LIBWEBRTC_BRANCH from $MOZ_LIBWEBRTC_SRC" + echo "-------" + ./mach python $SCRIPT_DIR/vendor-libwebrtc.py \ + --from-local $MOZ_LIBWEBRTC_SRC \ + --commit $MOZ_LIBWEBRTC_BRANCH \ + libwebrtc +} + +# The vendoring script (called above in vendor_off_next_commit) replaces +# the entire third_party/libwebrtc directory, which effectively removes +# all the generated moz.build files. It is easier (less error prone), +# to revert only those missing moz.build files rather than attempt to +# rebuild them because the rebuild may need json updates to work properly. +function regen_mozbuild_files { + echo "-------" + echo "------- Restore moz.build files from repo" + echo "-------" + hg revert --include "third_party/libwebrtc/**moz.build" \ + third_party/libwebrtc &> $LOG_DIR/log-regen-mozbuild-files.txt +} + +function add_new_upstream_files { + if [ "x$HANDLE_NOOP_COMMIT" == "x1" ]; then + return + fi + UPSTREAM_ADDED_FILES=`echo "$FILTERED_GIT_CHANGES" | grep "^A" \ + | awk '{print $2;}' || true` + if [ "x$UPSTREAM_ADDED_FILES" != "x" ]; then + echo "-------" + echo "------- Add new upstream files" + echo "-------" + (cd third_party/libwebrtc && hg add $UPSTREAM_ADDED_FILES) + echo "$UPSTREAM_ADDED_FILES" &> $LOG_DIR/log-new-upstream-files.txt + fi +} + +function remove_deleted_upstream_files { + if [ "x$HANDLE_NOOP_COMMIT" == "x1" ]; then + return + fi + UPSTREAM_DELETED_FILES=`echo "$FILTERED_GIT_CHANGES" | grep "^D" \ + | awk '{print $2;}' || true` + if [ "x$UPSTREAM_DELETED_FILES" != "x" ]; then + echo "-------" + echo "------- Remove deleted upstream files" + echo "-------" + (cd third_party/libwebrtc && hg rm $UPSTREAM_DELETED_FILES) + echo "$UPSTREAM_DELETED_FILES" &> $LOG_DIR/log-deleted-upstream-files.txt + fi +} + +function handle_renamed_upstream_files { + if [ "x$HANDLE_NOOP_COMMIT" == "x1" ]; then + return + fi + UPSTREAM_RENAMED_FILES=`echo "$FILTERED_GIT_CHANGES" | grep "^R" \ + | awk '{print $2 " " $3;}' || true` + if [ "x$UPSTREAM_RENAMED_FILES" != "x" ]; then + echo "-------" + echo "------- Handle renamed upstream files" + echo "-------" + (cd third_party/libwebrtc && echo "$UPSTREAM_RENAMED_FILES" | while read line; do hg rename --after $line; done) + echo "$UPSTREAM_RENAMED_FILES" &> $LOG_DIR/log-renamed-upstream-files.txt + fi +} + +if [ $SKIP_TO = "run" ]; then + echo "resume2" > $STATE_DIR/resume_state + rebase_mozlibwebrtc_stack; +fi + +if [ $SKIP_TO = "resume2" ]; then SKIP_TO="run"; fi +if [ $SKIP_TO = "run" ]; then + echo "resume3" > $STATE_DIR/resume_state + vendor_off_next_commit; +fi + +if [ $SKIP_TO = "resume3" ]; then SKIP_TO="run"; fi +if [ $SKIP_TO = "run" ]; then + echo "resume4" > $STATE_DIR/resume_state + regen_mozbuild_files; +fi + +if [ $SKIP_TO = "resume4" ]; then SKIP_TO="run"; fi +if [ $SKIP_TO = "run" ]; then + echo "resume5" > $STATE_DIR/resume_state + remove_deleted_upstream_files; +fi + +if [ $SKIP_TO = "resume5" ]; then SKIP_TO="run"; fi +if [ $SKIP_TO = "run" ]; then + echo "resume6" > $STATE_DIR/resume_state + add_new_upstream_files; +fi + +if [ $SKIP_TO = "resume6" ]; then SKIP_TO="run"; fi +if [ $SKIP_TO = "run" ]; then + echo "resume7" > $STATE_DIR/resume_state + handle_renamed_upstream_files; +fi + +echo "" > $STATE_DIR/resume_state +echo "-------" +echo "------- Commit vendored changes from $MOZ_LIBWEBRTC_NEXT_BASE" +echo "-------" +UPSTREAM_SHA=`cd $MOZ_LIBWEBRTC_SRC && \ + git show --name-only $MOZ_LIBWEBRTC_NEXT_BASE \ + | grep "^commit " | awk '{ print $NF }'` +echo "Bug $MOZ_FASTFORWARD_BUG - Vendor libwebrtc from $MOZ_LIBWEBRTC_NEXT_BASE" \ + > $TMP_DIR/commit_msg.txt +echo "" >> $TMP_DIR/commit_msg.txt +if [ -f $STATE_DIR/$MOZ_LIBWEBRTC_NEXT_BASE.no-op-cherry-pick-msg ]; then + cat $STATE_DIR/$MOZ_LIBWEBRTC_NEXT_BASE.no-op-cherry-pick-msg >> $TMP_DIR/commit_msg.txt + echo "" >> $TMP_DIR/commit_msg.txt + rm $STATE_DIR/$MOZ_LIBWEBRTC_NEXT_BASE.no-op-cherry-pick-msg +fi +echo "Upstream commit: https://webrtc.googlesource.com/src/+/$UPSTREAM_SHA" >> $TMP_DIR/commit_msg.txt +(cd $MOZ_LIBWEBRTC_SRC && \ +git show --name-only $MOZ_LIBWEBRTC_NEXT_BASE | grep "^ ") >> $TMP_DIR/commit_msg.txt + +hg commit -l $TMP_DIR/commit_msg.txt third_party/libwebrtc diff --git a/dom/media/webrtc/third_party_build/fetch_github_repo.py b/dom/media/webrtc/third_party_build/fetch_github_repo.py new file mode 100644 index 0000000000..bfa40f5c7b --- /dev/null +++ b/dom/media/webrtc/third_party_build/fetch_github_repo.py @@ -0,0 +1,122 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import argparse +import os +import re +import shutil + +from run_operations import run_git, run_shell + +# This script fetches the moz-libwebrtc github repro with the expected +# upstream remote and branch-heads setup. This is used by both the +# prep_repo.sh script as well as the restore_patch_stack.py script. +# +# For speed and conservation of network resources, after fetching all +# the data, a tar of the repo is made and used if available. + + +def fetch_repo(github_path, force_fetch, tar_path): + capture_output = False + + # check for pre-existing repo - make sure we force the removal + if force_fetch and os.path.exists(github_path): + print("Removing existing repo: {}".format(github_path)) + shutil.rmtree(github_path) + + # clone https://github.com/mozilla/libwebrtc + if not os.path.exists(github_path): + # check for pre-existing tar, use it if we have it + if os.path.exists(tar_path): + print("Using tar file to reconstitute repo") + cmd = "cd {} ; tar --extract --gunzip --file={}".format( + os.path.dirname(github_path), os.path.basename(tar_path) + ) + run_shell(cmd, capture_output) + else: + print("Cloning github repo") + run_shell( + "git clone https://github.com/mozilla/libwebrtc {}".format(github_path), + capture_output, + ) + + # setup upstream (https://webrtc.googlesource.com/src) + stdout_lines = run_git("git config --local --list", github_path) + stdout_lines = [ + path for path in stdout_lines if re.findall("^remote.upstream.url.*", path) + ] + if len(stdout_lines) == 0: + print("Fetching upstream") + run_git("git checkout master", github_path) + run_git( + "git remote add upstream https://webrtc.googlesource.com/src", github_path + ) + run_git("git fetch upstream", github_path) + run_git("git merge upstream/master", github_path) + else: + print( + "Upstream remote (https://webrtc.googlesource.com/src) already configured" + ) + + # setup upstream branch-heads + stdout_lines = run_git( + "git config --local --get-all remote.upstream.fetch", github_path + ) + if len(stdout_lines) == 1: + print("Fetching upstream branch-heads") + run_git( + "git config --local --add remote.upstream.fetch +refs/branch-heads/*:refs/remotes/branch-heads/*", + github_path, + ) + run_git("git fetch upstream", github_path) + else: + print("Upstream remote branch-heads already configured") + + # do a sanity fetch in case this was not a freshly cloned copy of the + # repo, meaning it may not have all the mozilla branches present. + run_git("git fetch --all", github_path) + + # create tar to avoid time refetching + if not os.path.exists(tar_path): + print("Creating tar file for quicker restore") + cmd = "cd {} ; tar --create --gzip --file={} {}".format( + os.path.dirname(github_path), + os.path.basename(tar_path), + os.path.basename(github_path), + ) + run_shell(cmd, capture_output) + + +if __name__ == "__main__": + default_state_dir = ".moz-fast-forward" + default_tar_name = "moz-libwebrtc.tar.gz" + + parser = argparse.ArgumentParser( + description="Restore moz-libwebrtc github patch stack" + ) + parser.add_argument( + "--repo-path", + required=True, + help="path to libwebrtc repo", + ) + parser.add_argument( + "--force-fetch", + action="store_true", + default=False, + help="force rebuild an existing repo directory", + ) + parser.add_argument( + "--tar-name", + default=default_tar_name, + help="name of tar file (defaults to {})".format(default_tar_name), + ) + parser.add_argument( + "--state-path", + default=default_state_dir, + help="path to state directory (defaults to {})".format(default_state_dir), + ) + args = parser.parse_args() + + fetch_repo( + args.repo_path, args.force_fetch, os.path.join(args.state_path, args.tar_name) + ) diff --git a/dom/media/webrtc/third_party_build/filter_git_changes.py b/dom/media/webrtc/third_party_build/filter_git_changes.py new file mode 100644 index 0000000000..f8964e145b --- /dev/null +++ b/dom/media/webrtc/third_party_build/filter_git_changes.py @@ -0,0 +1,72 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import argparse +import importlib +import re +import subprocess +import sys + +sys.path.insert(0, "./dom/media/webrtc/third_party_build") +vendor_libwebrtc = importlib.import_module("vendor-libwebrtc") + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Get relevant change count from an upstream git commit" + ) + parser.add_argument( + "--repo-path", + required=True, + help="path to libwebrtc repo", + ) + parser.add_argument("--commit-sha", required=True, help="sha of commit to examine") + parser.add_argument("--diff-filter", choices=("A", "D", "R")) + args = parser.parse_args() + + command = [ + "git", + "show", + "--oneline", + "--name-status", + "--pretty=format:", + None if not args.diff_filter else "--diff-filter={}".format(args.diff_filter), + args.commit_sha, + ] + # strip possible empty elements from command list + command = [x for x in command if x is not None] + + # Get the list of changes in the upstream commit. + res = subprocess.run( + command, + capture_output=True, + text=True, + cwd=args.repo_path, + ) + if res.returncode != 0: + sys.exit("error: {}".format(res.stderr.strip())) + + changed_files = [line.strip() for line in res.stdout.strip().split("\n")] + changed_files = [line for line in changed_files if line != ""] + + # Fetch the list of excludes and includes used in the vendoring script. + exclude_list = vendor_libwebrtc.get_excluded_paths() + include_list = vendor_libwebrtc.get_included_path_overrides() + + # First, search for changes in files that are specifically included. + # Do this first, because some of these files might be filtered out + # by the exclude list. + regex_includes = "|".join(["^.\t{}".format(i) for i in include_list]) + included_files = [ + path for path in changed_files if re.findall(regex_includes, path) + ] + + # Convert the exclude list to a regex string. + regex_excludes = "|".join(["^.\t{}".format(i) for i in exclude_list]) + + # Filter out the excluded files/paths. + files_not_excluded = [ + path for path in changed_files if not re.findall(regex_excludes, path) + ] + + for path in included_files + files_not_excluded: + print(path) diff --git a/dom/media/webrtc/third_party_build/gn-configs/README.md b/dom/media/webrtc/third_party_build/gn-configs/README.md new file mode 100644 index 0000000000..111d2d022e --- /dev/null +++ b/dom/media/webrtc/third_party_build/gn-configs/README.md @@ -0,0 +1,16 @@ +# Generate new gn json files and moz.build files for building libwebrtc in our tree + +/!\ This is only supported on Linux and macOS. If you are on Windows, you can run +the script under [WSL](https://docs.microsoft.com/en-us/windows/wsl/install). + +1. The script should be run from the top directory of our firefox tree. + + ``` + ./mach python python/mozbuild/mozbuild/gn_processor.py dom/media/webrtc/third_party_build/gn-configs/webrtc.json + ``` + +2. Checkin all the generated/modified files and try your build! + +# Adding new configurations to the build + +Edit the `main` function in the `python/mozbuild/mozbuild/gn_processor.py` file. diff --git a/dom/media/webrtc/third_party_build/gn-configs/webrtc.json b/dom/media/webrtc/third_party_build/gn-configs/webrtc.json new file mode 100644 index 0000000000..9744558025 --- /dev/null +++ b/dom/media/webrtc/third_party_build/gn-configs/webrtc.json @@ -0,0 +1,83 @@ +{ + "target_dir": "third_party/libwebrtc", + "gn_target": "//:webrtc", + "gn_sandbox_variables": { + "COMPILE_FLAGS": { + "WARNINGS_AS_ERRORS": [] + }, + "FINAL_LIBRARY": "webrtc" + }, + "mozilla_flags": ["-fobjc-arc", "-mavx2", "-mfma", "-mfpu=neon", "-msse2"], + "write_mozbuild_variables": { + "INCLUDE_TK_CFLAGS_DIRS": [ + "third_party/libwebrtc/modules/desktop_capture/desktop_capture_gn", + "third_party/libwebrtc/modules/portal/portal_gn" + ] + }, + "non_unified_sources": [ + "third_party/libwebrtc/api/audio_codecs/opus/audio_encoder_opus_config.cc", + "third_party/libwebrtc/api/video/i210_buffer.cc", + "third_party/libwebrtc/api/video/i410_buffer.cc", + "third_party/libwebrtc/api/video/i422_buffer.cc", + "third_party/libwebrtc/api/video/i444_buffer.cc", + "third_party/libwebrtc/audio/channel_send_frame_transformer_delegate.cc", + "third_party/libwebrtc/common_audio/vad/vad_core.c", + "third_party/libwebrtc/common_audio/vad/webrtc_vad.c", + "third_party/libwebrtc/common_audio/signal_processing/resample_by_2_mips.c", + "third_party/libwebrtc/modules/audio_coding/codecs/isac/fix/source/decode_plc.c", + "third_party/libwebrtc/modules/audio_coding/codecs/isac/fix/source/lpc_masking_model.c", + "third_party/libwebrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter.c", + "third_party/libwebrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c", + "third_party/libwebrtc/modules/audio_coding/neteq/audio_vector.cc", + "third_party/libwebrtc/modules/audio_coding/neteq/underrun_optimizer.cc", + "third_party/libwebrtc/modules/audio_device/android/audio_manager.cc", + "third_party/libwebrtc/modules/audio_device/android/audio_record_jni.cc", + "third_party/libwebrtc/modules/audio_device/android/audio_track_jni.cc", + "third_party/libwebrtc/modules/audio_device/android/opensles_player.cc", + "third_party/libwebrtc/modules/audio_device/linux/audio_device_pulse_linux.cc", + "third_party/libwebrtc/modules/audio_device/linux/audio_mixer_manager_pulse_linux.cc", + "third_party/libwebrtc/modules/audio_device/win/audio_device_core_win.cc", + "third_party/libwebrtc/modules/audio_processing/aecm/aecm_core.cc", + "third_party/libwebrtc/modules/audio_processing/aecm/aecm_core_c.cc", + "third_party/libwebrtc/modules/audio_processing/aecm/aecm_core_mips.cc", + "third_party/libwebrtc/modules/audio_processing/aecm/aecm_core_neon.cc", + "third_party/libwebrtc/modules/audio_processing/aecm/echo_control_mobile.cc", + "third_party/libwebrtc/modules/audio_processing/echo_control_mobile_impl.cc", + "third_party/libwebrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator.cc", + "third_party/libwebrtc/modules/audio_processing/gain_control_impl.cc", + "third_party/libwebrtc/modules/audio_processing/rms_level.cc", + "third_party/libwebrtc/modules/congestion_controller/goog_cc/loss_based_bandwidth_estimation.cc", + "third_party/libwebrtc/modules/congestion_controller/goog_cc/send_side_bandwidth_estimation.cc", + "third_party/libwebrtc/modules/desktop_capture/fallback_desktop_capturer_wrapper.cc", + "third_party/libwebrtc/modules/desktop_capture/linux/wayland/base_capturer_pipewire.cc", + "third_party/libwebrtc/modules/desktop_capture/linux/wayland/moz_base_capturer_pipewire.cc", + "third_party/libwebrtc/modules/desktop_capture/mouse_cursor_monitor_linux.cc", + "third_party/libwebrtc/modules/desktop_capture/win/screen_capturer_win_gdi.cc", + "third_party/libwebrtc/modules/pacing/prioritized_packet_queue.cc", + "third_party/libwebrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator_single_stream.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/flexfec_receiver.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/rtcp_packet/tmmbn.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/rtcp_packet/tmmbr.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/rtcp_sender.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_vp8.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/rtp_rtcp_impl2.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_egress.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_video.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/rtp_util.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/ulpfec_generator.cc", + "third_party/libwebrtc/modules/rtp_rtcp/source/video_rtp_depacketizer_vp9.cc", + "third_party/libwebrtc/modules/third_party/g722/g722_encode.c", + "third_party/libwebrtc/modules/video_capture/windows/device_info_ds.cc", + "third_party/libwebrtc/modules/video_capture/windows/help_functions_ds.cc", + "third_party/libwebrtc/modules/video_capture/windows/sink_filter_ds.cc", + "third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc", + "third_party/libwebrtc/modules/video_coding/svc/scalability_structure_key_svc.cc", + "third_party/libwebrtc/modules/video_coding/svc/scalability_structure_simulcast.cc", + "third_party/libwebrtc/rtc_base/win/hstring.cc", + "third_party/libwebrtc/third_party/abseil-cpp/absl/strings/numbers.cc", + "third_party/libwebrtc/third_party/abseil-cpp/absl/synchronization/blocking_counter.cc", + "third_party/libwebrtc/third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_posix.cc", + "third_party/libwebrtc/third_party/abseil-cpp/absl/time/time.cc", + "third_party/libwebrtc/video/rtp_video_stream_receiver2.cc" + ] +} diff --git a/dom/media/webrtc/third_party_build/lookup_branch_head.py b/dom/media/webrtc/third_party_build/lookup_branch_head.py new file mode 100644 index 0000000000..afd0e6c791 --- /dev/null +++ b/dom/media/webrtc/third_party_build/lookup_branch_head.py @@ -0,0 +1,98 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import argparse +import json +import os +import pathlib +import sys +import urllib.request + +# default cache file location in STATE_DIR location +default_cache_path = ".moz-fast-forward/milestone.cache" + + +def fetch_branch_head_dict(): + milestone_url = ( + "https://chromiumdash.appspot.com/fetch_milestones?only_branched=true" + ) + uf = urllib.request.urlopen(milestone_url) + html = uf.read() + milestone_dict = json.loads(html) + + # There is more information in the json dictionary, but we only care + # about the milestone (version) to branch "name" (webrtc_branch) + # info. For example: + # v106 -> 5249 (which translates to branch-heads/5249) + # v107 -> 5304 (which translates to branch-heads/5304) + # + # As returned from web query, milestones are integers and branch + # "names" are strings. + new_dict = {} + for row in milestone_dict: + new_dict[row["milestone"]] = row["webrtc_branch"] + + return new_dict + + +def read_dict_from_cache(cache_path): + if cache_path is not None and os.path.exists(cache_path): + with open(cache_path, "r") as ifile: + return json.loads(ifile.read(), object_hook=jsonKeys2int) + return {} + + +def write_dict_to_cache(cache_path, milestones): + with open(cache_path, "w") as ofile: + ofile.write(json.dumps(milestones)) + + +def get_branch_head(milestone, cache_path=default_cache_path): + milestones = read_dict_from_cache(cache_path) + + # if the cache didn't exist or is stale, try to fetch using a web query + if milestone not in milestones: + try: + milestones = fetch_branch_head_dict() + write_dict_to_cache(cache_path, milestones) + except Exception: + pass + + if milestone in milestones: + return milestones[milestone] + return None + + +# From https://stackoverflow.com/questions/1450957/pythons-json-module-converts-int-dictionary-keys-to-strings +def jsonKeys2int(x): + if isinstance(x, dict): + return {int(k): v for k, v in x.items()} + return x + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Get libwebrtc branch-head for given chromium milestone" + ) + parser.add_argument( + "milestone", type=int, help="integer chromium milestone (example: 106)" + ) + parser.add_argument("-v", "--verbose", action="store_true") + parser.add_argument("-c", "--cache", type=pathlib.Path, help="path to cache file") + args = parser.parse_args() + + # if the user provided a cache path use it, otherwise use the default + local_cache_path = args.cache or default_cache_path + + branch_head = get_branch_head(args.milestone, local_cache_path) + if branch_head is None: + sys.exit("error: chromium milestone '{}' is not found.".format(args.milestone)) + + if args.verbose: + print( + "chromium milestone {} uses branch-heads/{}".format( + args.milestone, branch_head + ) + ) + else: + print(branch_head) diff --git a/dom/media/webrtc/third_party_build/loop-ff.sh b/dom/media/webrtc/third_party_build/loop-ff.sh new file mode 100644 index 0000000000..72a8d88091 --- /dev/null +++ b/dom/media/webrtc/third_party_build/loop-ff.sh @@ -0,0 +1,236 @@ +#!/bin/bash + +source dom/media/webrtc/third_party_build/use_config_env.sh + +# file for logging loop script output +LOOP_OUTPUT_LOG=$LOG_DIR/log-loop-ff.txt + +function echo_log() +{ + echo "===loop-ff=== $@" 2>&1| tee -a $LOOP_OUTPUT_LOG +} + +function show_error_msg() +{ + echo_log "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo_log "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +# If DEBUG_LOOP_FF is set all commands should be printed as they are executed +if [ ! "x$DEBUG_LOOP_FF" = "x" ]; then + set -x +fi + +if [ "x$MOZ_LIBWEBRTC_SRC" = "x" ]; then + echo "MOZ_LIBWEBRTC_SRC is not defined, see README.md" + exit +fi + +if [ ! -d $MOZ_LIBWEBRTC_SRC ]; then + echo "Path $MOZ_LIBWEBRTC_SRC is not found, see README.md" + exit +fi + +if [ "x$MOZ_LIBWEBRTC_BRANCH" = "x" ]; then + echo "MOZ_LIBWEBRTC_BRANCH is not defined, see README.md" + exit +fi + +if [ "x$MOZ_STOP_AFTER_COMMIT" = "x" ]; then + MOZ_STOP_AFTER_COMMIT=`cd $MOZ_LIBWEBRTC_SRC ; git show $MOZ_TARGET_UPSTREAM_BRANCH_HEAD --format='%h' --name-only | head -1` + echo "No MOZ_STOP_AFTER_COMMIT variable defined - stopping at $MOZ_TARGET_UPSTREAM_BRANCH_HEAD" +fi + +if [ "x$MOZ_ADVANCE_ONE_COMMIT" = "x" ]; then + MOZ_ADVANCE_ONE_COMMIT="" +fi + +if [ "x$SKIP_NEXT_REVERT_CHK" = "x" ]; then + SKIP_NEXT_REVERT_CHK="0" +fi + +MOZ_CHANGED=0 +GIT_CHANGED=0 +HANDLE_NOOP_COMMIT="" + +# After this point: +# * eE: All commands should succeed. +# * u: All variables should be defined before use. +# * o pipefail: All stages of all pipes should succeed. +set -eEuo pipefail + +# start a new log with every run of this script +rm -f $LOOP_OUTPUT_LOG +# make sure third_party/libwebrtc/README.moz-ff-commit is the committed version +# so we properly determine MOZ_LIBWEBRTC_BASE and MOZ_LIBWEBRTC_NEXT_BASE +# in the loop below +hg revert -C third_party/libwebrtc/README.moz-ff-commit &> /dev/null + +# check for a resume situation from fast-forward-libwebrtc.sh +RESUME="" +if [ -f $STATE_DIR/resume_state ]; then + RESUME=`tail -1 $STATE_DIR/resume_state` +fi + +ERROR_HELP=$" +It appears that initial vendoring verification has failed. +- If you have never run the fast-forward process before, you may need to + prepare the github repository by running prep_repo.sh. +- If you have previously run loop-ff.sh successfully, there may be a new + change to third_party/libwebrtc that should be extracted and added to + the patch stack in github. It may be as easy as running: + ./mach python $SCRIPT_DIR/extract-for-git.py tip::tip + mv mailbox.patch $MOZ_LIBWEBRTC_SRC + (cd $MOZ_LIBWEBRTC_SRC && \\ + git am mailbox.patch) +" +# if we're not in the resume situation from fast-forward-libwebrtc.sh +if [ "x$RESUME" = "x" ]; then + # start off by verifying the vendoring process to make sure no changes have + # been added to elm to fix bugs. + echo_log "Verifying vendoring..." + bash $SCRIPT_DIR/verify_vendoring.sh + echo_log "Done verifying vendoring." +fi +ERROR_HELP="" + +for (( ; ; )); do + +find_base_commit +find_next_commit + +if [ $MOZ_LIBWEBRTC_BASE == $MOZ_LIBWEBRTC_NEXT_BASE ]; then + echo_log "Processing complete, already at upstream $MOZ_LIBWEBRTC_BASE" + exit +fi + +echo_log "===================" + +COMMITS_REMAINING=`cd $MOZ_LIBWEBRTC_SRC ; \ + git log --oneline $MOZ_LIBWEBRTC_BASE..$MOZ_TARGET_UPSTREAM_BRANCH_HEAD \ + | wc -l | tr -d " "` +echo_log "Commits remaining: $COMMITS_REMAINING" + +ERROR_HELP=$"Some portion of the detection and/or fixing of upstream revert commits +has failed. Please fix the state of the git hub repo at: $MOZ_LIBWEBRTC_SRC. +When fixed, please resume this script with the following command: + SKIP_NEXT_REVERT_CHK=1 bash $SCRIPT_DIR/loop-ff.sh +" +if [ "x$SKIP_NEXT_REVERT_CHK" == "x0" ]; then + echo_log "Check for upcoming revert commit" + AUTO_FIX_REVERT_AS_NOOP=1 bash $SCRIPT_DIR/detect_upstream_revert.sh \ + 2>&1| tee -a $LOOP_OUTPUT_LOG +fi +SKIP_NEXT_REVERT_CHK="0" +ERROR_HELP="" + +echo_log "Looking for $STATE_DIR/$MOZ_LIBWEBRTC_NEXT_BASE.no-op-cherry-pick-msg" +if [ -f $STATE_DIR/$MOZ_LIBWEBRTC_NEXT_BASE.no-op-cherry-pick-msg ]; then + echo_log "Detected special commit msg, setting HANDLE_NOOP_COMMIT=1" + HANDLE_NOOP_COMMIT="1" +fi + +echo_log "Moving from moz-libwebrtc commit $MOZ_LIBWEBRTC_BASE to $MOZ_LIBWEBRTC_NEXT_BASE" +bash $SCRIPT_DIR/fast-forward-libwebrtc.sh 2>&1| tee -a $LOOP_OUTPUT_LOG + +MOZ_CHANGED=`hg diff -c tip --stat \ + | egrep -ve "README.moz-ff-commit|README.mozilla|files changed," \ + | wc -l | tr -d " " || true` +GIT_CHANGED=`./mach python $SCRIPT_DIR/filter_git_changes.py \ + --repo-path $MOZ_LIBWEBRTC_SRC --commit-sha $MOZ_LIBWEBRTC_NEXT_BASE \ + | wc -l | tr -d " "` +FILE_CNT_MISMATCH_MSG=$" +The number of files changed in the upstream commit ($GIT_CHANGED) does +not match the number of files changed in the local Mozilla repo +commit ($MOZ_CHANGED). This may indicate a mismatch between the vendoring +script and this script, or it could be a true error in the import +processing. Once the issue has been resolved, the following steps +remain for this commit: + # generate moz.build files (may not be necessary) + ./mach python python/mozbuild/mozbuild/gn_processor.py \\ + $SCRIPT_DIR/gn-configs/webrtc.json + # commit the updated moz.build files with the appropriate commit msg + bash $SCRIPT_DIR/commit-build-file-changes.sh + # do a (hopefully) quick test build + ./mach build +" +echo_log "Verify number of files changed MOZ($MOZ_CHANGED) GIT($GIT_CHANGED)" +if [ "x$HANDLE_NOOP_COMMIT" == "x1" ]; then + echo_log "NO-OP commit detected, we expect file changed counts to differ" +elif [ $MOZ_CHANGED -ne $GIT_CHANGED ]; then + echo_log "MOZ_CHANGED $MOZ_CHANGED should equal GIT_CHANGED $GIT_CHANGED" + echo "$FILE_CNT_MISMATCH_MSG" + exit 1 +fi +HANDLE_NOOP_COMMIT="" + +# save the current patch stack in case we need to reconstitute it later +./mach python $SCRIPT_DIR/save_patch_stack.py \ + --repo-path $MOZ_LIBWEBRTC_SRC \ + --branch $MOZ_LIBWEBRTC_BRANCH \ + --patch-path "third_party/libwebrtc/moz-patch-stack" \ + --state-path $STATE_DIR \ + --target-branch-head $MOZ_TARGET_UPSTREAM_BRANCH_HEAD \ + 2>&1| tee -a $LOOP_OUTPUT_LOG + +MODIFIED_BUILD_RELATED_FILE_CNT=`hg diff -c tip --stat \ + --include 'third_party/libwebrtc/**BUILD.gn' \ + --include 'third_party/libwebrtc/webrtc.gni' \ + | grep -v "files changed" \ + | wc -l | tr -d " " || true` +ERROR_HELP=$" +Generating build files has failed. This likely means changes to one or more +BUILD.gn files are required. Commit those changes following the instructions +in https://wiki.mozilla.org/Media/WebRTC/libwebrtc_Update_Process#Operational_notes +Then complete these steps: + # generate moz.build files (may not be necessary) + ./mach python python/mozbuild/mozbuild/gn_processor.py \\ + $SCRIPT_DIR/gn-configs/webrtc.json + # commit the updated moz.build files with the appropriate commit msg + bash $SCRIPT_DIR/commit-build-file-changes.sh + # do a (hopefully) quick test build + ./mach build +After a successful build, you may resume this script. +" +echo_log "Modified BUILD.gn (or webrtc.gni) files: $MODIFIED_BUILD_RELATED_FILE_CNT" +if [ "x$MODIFIED_BUILD_RELATED_FILE_CNT" != "x0" ]; then + echo_log "Regenerate build files" + ./mach python python/mozbuild/mozbuild/gn_processor.py \ + $SCRIPT_DIR/gn-configs/webrtc.json 2>&1| tee -a $LOOP_OUTPUT_LOG + + MOZ_BUILD_CHANGE_CNT=`hg status third_party/libwebrtc \ + --include 'third_party/libwebrtc/**moz.build' | wc -l | tr -d " "` + if [ "x$MOZ_BUILD_CHANGE_CNT" != "x0" ]; then + echo_log "Detected modified moz.build files, commiting" + fi + + bash $SCRIPT_DIR/commit-build-file-changes.sh 2>&1| tee -a $LOOP_OUTPUT_LOG +fi +ERROR_HELP="" + +ERROR_HELP=$" +The test build has failed. Most likely this is due to an upstream api change that +must be reflected in Mozilla code outside of the third_party/libwebrtc directory. +" +echo_log "Test build" +./mach build 2>&1| tee -a $LOOP_OUTPUT_LOG +ERROR_HELP="" + +if [ ! "x$MOZ_STOP_AFTER_COMMIT" = "x" ]; then +if [ $MOZ_LIBWEBRTC_NEXT_BASE = $MOZ_STOP_AFTER_COMMIT ]; then + break +fi +fi + +if [ ! "x$MOZ_ADVANCE_ONE_COMMIT" = "x" ]; then + echo_log "Done advancing one commit." + exit +fi + +done + +echo_log "Completed fast-foward to $MOZ_STOP_AFTER_COMMIT" diff --git a/dom/media/webrtc/third_party_build/make_upstream_revert_noop.sh b/dom/media/webrtc/third_party_build/make_upstream_revert_noop.sh new file mode 100755 index 0000000000..9f5fdd7b82 --- /dev/null +++ b/dom/media/webrtc/third_party_build/make_upstream_revert_noop.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +# This script takes the current base sha, the next base sha, and the sha +# of the commit that reverts the next base as determined by +# detect_upstream_revert.sh and "inserts" two commits at the bottom of the +# moz_libwebrtc GitHub patch stack. The two commits are exact copies of +# the upcoming commit and its corresponding revert commit, with commit +# messages indicating they are temporary commits. Additionally, 2 files +# are written that act as markers for the fast-forward script and contain +# supplemental commit message text. +# +# When the fast-forward script runs, it will rebase onto the next base +# sha. Since we have a corresponding, identical temp commit at the bottom +# of our patch stack, the temp commit will be absorbed as unnecessary. +# Since the patch stack now has the temp revert commit at the bottom, this +# results in a “no-op” commit. The marker file indicates that specially +# handling should occur in the fast-forward-libwebrtc.sh and loop-ff.sh +# scripts. This special handling includes adding the supplemental commit +# text that explains why the commit is a no-op (or empty) commit and +# skipping the verification of the number of files changed between our +# mercurial repository and the GitHub repository. + +function show_error_msg() +{ + echo "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +source dom/media/webrtc/third_party_build/use_config_env.sh + +# If DEBUG_GEN is set all commands should be printed as they are executed +if [ ! "x$DEBUG_GEN" = "x" ]; then + set -x +fi + +if [ "x$MOZ_LIBWEBRTC_SRC" = "x" ]; then + echo "MOZ_LIBWEBRTC_SRC is not defined, see README.md" + exit +fi + +if [ -d $MOZ_LIBWEBRTC_SRC ]; then + echo "MOZ_LIBWEBRTC_SRC is $MOZ_LIBWEBRTC_SRC" +else + echo "Path $MOZ_LIBWEBRTC_SRC is not found, see README.md" + exit +fi + +if [ "x$MOZ_LIBWEBRTC_BRANCH" = "x" ]; then + echo "MOZ_LIBWEBRTC_BRANCH is not defined, see README.md" + exit +fi + +# After this point: +# * eE: All commands should succeed. +# * u: All variables should be defined before use. +# * o pipefail: All stages of all pipes should succeed. +set -eEuo pipefail + +find_base_commit +find_next_commit +echo "MOZ_LIBWEBRTC_BASE: $MOZ_LIBWEBRTC_BASE" +echo "MOZ_LIBWEBRTC_NEXT_BASE: $MOZ_LIBWEBRTC_NEXT_BASE" +echo "MOZ_LIBWEBRTC_REVERT_SHA: $MOZ_LIBWEBRTC_REVERT_SHA" + +# These files serve dual purposes: +# 1) They serve as marker/indicator files to loop-ff.sh to +# know to process the commit differently, accounting for +# the no-op nature of the commit and it's corresponding +# revert commit. +# 2) The contain supplemental commit message text to explain +# why the commits are essentially empty. +# They are written first on the off chance that the rebase +# operation below fails and requires manual intervention, +# thus avoiding the operator of these scripts to remember to +# generate these two files. +echo $"Essentially a no-op since we're going to see this change +reverted when we vendor in $MOZ_LIBWEBRTC_REVERT_SHA." \ +> $STATE_DIR/$MOZ_LIBWEBRTC_NEXT_BASE.no-op-cherry-pick-msg + +echo "We already cherry-picked this when we vendored $MOZ_LIBWEBRTC_NEXT_BASE." \ +> $STATE_DIR/$MOZ_LIBWEBRTC_REVERT_SHA.no-op-cherry-pick-msg + +cd $MOZ_LIBWEBRTC_SRC +rm -f $TMP_DIR/*.patch $TMP_DIR/*.patch.bak +git checkout -b moz-cherry-pick $MOZ_LIBWEBRTC_BASE +git format-patch -o $TMP_DIR -k --start-number 1 \ + $MOZ_LIBWEBRTC_NEXT_BASE^..$MOZ_LIBWEBRTC_NEXT_BASE +git format-patch -o $TMP_DIR -k --start-number 2 \ + $MOZ_LIBWEBRTC_REVERT_SHA^..$MOZ_LIBWEBRTC_REVERT_SHA +sed -i.bak -e "/^Subject: / s/$/ ($MOZ_LIBWEBRTC_NEXT_BASE)/" $TMP_DIR/0001*.patch +sed -i.bak -e "/^Subject: / s/$/ ($MOZ_LIBWEBRTC_REVERT_SHA)/" $TMP_DIR/0002*.patch +sed -i.bak -e 's/^Subject: /Subject: (tmp-cherry-pick) /' $TMP_DIR/*.patch +git am $TMP_DIR/*.patch +git checkout $MOZ_LIBWEBRTC_BRANCH +git rebase moz-cherry-pick +git branch -d moz-cherry-pick + diff --git a/dom/media/webrtc/third_party_build/pre-warmed-milestone.cache b/dom/media/webrtc/third_party_build/pre-warmed-milestone.cache new file mode 100644 index 0000000000..826268db47 --- /dev/null +++ b/dom/media/webrtc/third_party_build/pre-warmed-milestone.cache @@ -0,0 +1 @@ +{"109": "5414", "108": "5359", "107": "5304", "106": "5249", "105": "5195"} diff --git a/dom/media/webrtc/third_party_build/prep_repo.sh b/dom/media/webrtc/third_party_build/prep_repo.sh new file mode 100644 index 0000000000..9f22f63253 --- /dev/null +++ b/dom/media/webrtc/third_party_build/prep_repo.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +function show_error_msg() +{ + echo "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +source dom/media/webrtc/third_party_build/use_config_env.sh + +echo "MOZ_LIBWEBRTC_SRC: $MOZ_LIBWEBRTC_SRC" +echo "MOZ_LIBWEBRTC_BRANCH: $MOZ_LIBWEBRTC_BRANCH" +echo "MOZ_FASTFORWARD_BUG: $MOZ_FASTFORWARD_BUG" + +# After this point: +# * eE: All commands should succeed. +# * u: All variables should be defined before use. +# * o pipefail: All stages of all pipes should succeed. +set -eEuo pipefail + +# wipe resume_state for new run +rm -f $STATE_DIR/resume_state + +# If there is no cache file for the branch-head lookups done in +# update_default_config.sh, go ahead and copy our small pre-warmed +# version. +if [ ! -f $STATE_DIR/milestone.cache ]; then + cp $SCRIPT_DIR/pre-warmed-milestone.cache $STATE_DIR/milestone.cache +fi + +# fetch the github repro +./mach python $SCRIPT_DIR/fetch_github_repo.py \ + --repo-path $MOZ_LIBWEBRTC_SRC \ + --state-path $STATE_DIR + +CURRENT_DIR=`pwd` +cd $MOZ_LIBWEBRTC_SRC + +# clear any possible previous patches +rm -f *.patch + +# create a new work branch and "export" a new patch stack to rebase +# find the common commit between our upstream release branch and trunk +CHERRY_PICK_BASE=`git merge-base branch-heads/$MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM master` +echo "common commit: $CHERRY_PICK_BASE" + +# create a new branch at the common commit and checkout the new branch +ERROR_HELP=$" +Unable to create branch '$MOZ_LIBWEBRTC_BRANCH'. This probably means +that prep_repo.sh is being called on a repo that already has a patch +stack in progress. If you're sure you want to do this, the following +commands will allow the process to continue: + ( cd $MOZ_LIBWEBRTC_SRC && \\ + git checkout $MOZ_LIBWEBRTC_BRANCH && \\ + git checkout -b $MOZ_LIBWEBRTC_BRANCH-old && \\ + git branch -D $MOZ_LIBWEBRTC_BRANCH ) && \\ + bash $0 +" +git branch $MOZ_LIBWEBRTC_BRANCH $CHERRY_PICK_BASE +ERROR_HELP="" +git checkout $MOZ_LIBWEBRTC_BRANCH + +# make sure we're starting with a clean tmp directory +rm -f $TMP_DIR/*.patch $TMP_DIR/*.patch.bak + +# grab the patches for all the commits in chrome's release branch for libwebrtc +git format-patch -o $TMP_DIR -k $CHERRY_PICK_BASE..branch-heads/$MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM +# tweak the release branch commit summaries to show they were cherry picked +sed -i.bak -e "/^Subject: / s/^Subject: /Subject: (cherry-pick-branch-heads\/$MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM) /" $TMP_DIR/*.patch +git am $TMP_DIR/*.patch # applies to branch mozpatches +rm $TMP_DIR/*.patch $TMP_DIR/*.patch.bak + +# we don't use restore_patch_stack.py here because it would overwrite the patches +# from the previous release branch we just added in the above step. + +# grab all the moz patches and apply +git am $CURRENT_DIR/third_party/libwebrtc/moz-patch-stack/*.patch + +cd $CURRENT_DIR + +# cp all the no-op files to STATE_DIR +NO_OP_FILE_COUNT=`ls third_party/libwebrtc/moz-patch-stack \ + | grep "no-op-cherry-pick-msg" | wc -l | tr -d " " || true` +if [ "x$NO_OP_FILE_COUNT" != "x0" ]; then + cp $CURRENT_DIR/third_party/libwebrtc/moz-patch-stack/*.no-op-cherry-pick-msg \ + $STATE_DIR +fi + +bash $SCRIPT_DIR/verify_vendoring.sh || true diff --git a/dom/media/webrtc/third_party_build/push_official_branch.sh b/dom/media/webrtc/third_party_build/push_official_branch.sh new file mode 100644 index 0000000000..e7ed17df2b --- /dev/null +++ b/dom/media/webrtc/third_party_build/push_official_branch.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# This script is a simple helper script that creates a branch name +# in moz-libwebrtc and pushes it to origin. You will need permissions +# to the mozilla fork of libwebrtc in order to complete this operation. +# The repo is: https://github.com/mozilla/libwebrtc/ +# +# Note: this should only be run after elm has been merged to mozilla-central + +function show_error_msg() +{ + echo "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +source dom/media/webrtc/third_party_build/use_config_env.sh + +# After this point: +# * eE: All commands should succeed. +# * o pipefail: All stages of all pipes should succeed. +set -eEo pipefail + +cd $MOZ_LIBWEBRTC_SRC + +git fetch + +if git show-ref --quiet refs/remotes/origin/$MOZ_LIBWEBRTC_OFFICIAL_BRANCH; then + echo "Branch '$MOZ_LIBWEBRTC_OFFICIAL_BRANCH' already exists remotely. Nothing to do." + + echo "Please ensure this branch information can be found on Bug $MOZ_FASTFORWARD_BUG" + echo "https://github.com/mozilla/libwebrtc/tree/$MOZ_LIBWEBRTC_OFFICIAL_BRANCH" + exit 0 +fi + +if git show-ref --quiet refs/heads/$MOZ_LIBWEBRTC_OFFICIAL_BRANCH; then + echo "Branch '$MOZ_LIBWEBRTC_OFFICIAL_BRANCH' already exists locally. No need to create it." +else + echo "Creating branch '$MOZ_LIBWEBRTC_OFFICIAL_BRANCH'" + git branch $MOZ_LIBWEBRTC_OFFICIAL_BRANCH +fi + +echo "Pushing the branch to https://github.com/mozilla/libwebrtc" +git push origin $MOZ_LIBWEBRTC_OFFICIAL_BRANCH + +echo "Please add this new branch information to Bug $MOZ_FASTFORWARD_BUG" +echo "https://github.com/mozilla/libwebrtc/tree/$MOZ_LIBWEBRTC_OFFICIAL_BRANCH" diff --git a/dom/media/webrtc/third_party_build/restore_elm_arcconfig.py b/dom/media/webrtc/third_party_build/restore_elm_arcconfig.py new file mode 100644 index 0000000000..00c0bd7309 --- /dev/null +++ b/dom/media/webrtc/third_party_build/restore_elm_arcconfig.py @@ -0,0 +1,27 @@ +#!/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from subprocess import run + +# This script sets the Arcanist configuration for the elm repo. This script +# should be run after each repository reset. +# +# Usage: from the root of the repo `./mach python dom/media/webrtc/third_party_build/restore_elm_arcconfig.py` +# + +ret = run( + [ + "hg", + "import", + "-m", + "Bug 1729988 - FLOAT - REPO-elm - update .arcconfig repo callsign r=bgrins", + "dom/media/webrtc/third_party_build/elm_arcconfig.patch", + ] +).returncode +if ret != 0: + raise Exception(f"Failed to add FLOATing arcconfig patch for ELM: { ret }") +else: + print("ELM .arcconfig restored. Please push this change to ELM") diff --git a/dom/media/webrtc/third_party_build/restore_patch_stack.py b/dom/media/webrtc/third_party_build/restore_patch_stack.py new file mode 100644 index 0000000000..6db3a64192 --- /dev/null +++ b/dom/media/webrtc/third_party_build/restore_patch_stack.py @@ -0,0 +1,107 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import argparse +import os +import re +import shutil + +from fetch_github_repo import fetch_repo +from run_operations import run_git, run_shell + +# This script restores the mozilla patch stack and no-op commit tracking +# files. In the case of repo corruption or a mistake made during +# various rebase conflict resolution operations, the patch-stack can be +# restored rather than requiring the user to restart the fast-forward +# process from the beginning. + + +def get_last_line(file_path): + # technique from https://stackoverflow.com/questions/46258499/how-to-read-the-last-line-of-a-file-in-python + with open(file_path, "rb") as f: + try: # catch OSError in case of a one line file + f.seek(-2, os.SEEK_END) + while f.read(1) != b"\n": + f.seek(-2, os.SEEK_CUR) + except OSError: + f.seek(0) + return f.readline().decode().strip() + + +def restore_patch_stack( + github_path, github_branch, patch_directory, state_directory, tar_name +): + # first, refetch the repo (hopefully utilizing the tarfile for speed) so + # the patches apply cleanly + fetch_repo(github_path, True, os.path.join(state_directory, tar_name)) + + # remove any stale no-op-cherry-pick-msg files in state_directory + run_shell("rm {}/*.no-op-cherry-pick-msg || true".format(state_directory)) + + # lookup latest vendored commit from third_party/libwebrtc/README.moz-ff-commit + file = os.path.abspath("third_party/libwebrtc/README.moz-ff-commit") + last_vendored_commit = get_last_line(file) + + # checkout the previous vendored commit with proper branch name + cmd = "git checkout -b {} {}".format(github_branch, last_vendored_commit) + run_git(cmd, github_path) + + # restore the patches to moz-libwebrtc repo, use run_shell instead of + # run_hg to allow filepath wildcard + print("Restoring patch stack") + run_shell("cd {} && git am {}/*.patch".format(github_path, patch_directory)) + + # it is also helpful to restore the no-op-cherry-pick-msg files to + # the state directory so that if we're restoring a patch-stack we + # also restore the possibly consumed no-op tracking files. + no_op_files = [ + path + for path in os.listdir(patch_directory) + if re.findall(".*no-op-cherry-pick-msg$", path) + ] + for file in no_op_files: + shutil.copy(os.path.join(patch_directory, file), state_directory) + + +if __name__ == "__main__": + default_patch_dir = "third_party/libwebrtc/moz-patch-stack" + default_state_dir = ".moz-fast-forward" + default_tar_name = "moz-libwebrtc.tar.gz" + + parser = argparse.ArgumentParser( + description="Restore moz-libwebrtc github patch stack" + ) + parser.add_argument( + "--repo-path", + required=True, + help="path to libwebrtc repo", + ) + parser.add_argument( + "--branch", + default="mozpatches", + help="moz-libwebrtc branch (defaults to mozpatches)", + ) + parser.add_argument( + "--patch-path", + default=default_patch_dir, + help="path to save patches (defaults to {})".format(default_patch_dir), + ) + parser.add_argument( + "--tar-name", + default=default_tar_name, + help="name of tar file (defaults to {})".format(default_tar_name), + ) + parser.add_argument( + "--state-path", + default=default_state_dir, + help="path to state directory (defaults to {})".format(default_state_dir), + ) + args = parser.parse_args() + + restore_patch_stack( + args.repo_path, + args.branch, + os.path.abspath(args.patch_path), + args.state_path, + args.tar_name, + ) diff --git a/dom/media/webrtc/third_party_build/run_operations.py b/dom/media/webrtc/third_party_build/run_operations.py new file mode 100644 index 0000000000..794ea268d8 --- /dev/null +++ b/dom/media/webrtc/third_party_build/run_operations.py @@ -0,0 +1,78 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import os +import subprocess +import sys + +# This is a collection of helper functions that use the subprocess.run +# command to execute commands. In the future, if we have cases where +# we find common python functionality in utilizing the data returned +# from these functions, that functionality can live here for easy reuse +# between other python scripts in dom/media/webrtc/third_party_build. + + +# must run 'hg' with HGPLAIN=1 to ensure aliases don't interfere with +# command output. +env = os.environ.copy() +env["HGPLAIN"] = "1" + + +def run_hg(cmd): + cmd_list = cmd.split(" ") + res = subprocess.run( + cmd_list, + capture_output=True, + text=True, + env=env, + ) + if res.returncode != 0: + print( + "Hit return code {} running '{}'. Aborting.".format(res.returncode, cmd), + file=sys.stderr, + ) + print(res.stderr) + sys.exit(1) + stdout = res.stdout.strip() + return [] if len(stdout) == 0 else stdout.split("\n") + + +def run_git(cmd, working_dir): + cmd_list = cmd.split(" ") + res = subprocess.run( + cmd_list, + capture_output=True, + text=True, + cwd=working_dir, + ) + if res.returncode != 0: + print( + "Hit return code {} running '{}'. Aborting.".format(res.returncode, cmd), + file=sys.stderr, + ) + print(res.stderr) + sys.exit(1) + stdout = res.stdout.strip() + return [] if len(stdout) == 0 else stdout.split("\n") + + +def run_shell(cmd, capture_output=True): + res = subprocess.run( + cmd, + shell=True, + capture_output=capture_output, + text=True, + ) + if res.returncode != 0: + print( + "Hit return code {} running '{}'. Aborting.".format(res.returncode, cmd), + file=sys.stderr, + ) + print(res.stderr) + sys.exit(1) + output_lines = [] + if capture_output: + stdout = res.stdout.strip() + output_lines = [] if len(stdout) == 0 else stdout.split("\n") + + return output_lines diff --git a/dom/media/webrtc/third_party_build/save_patch_stack.py b/dom/media/webrtc/third_party_build/save_patch_stack.py new file mode 100644 index 0000000000..11c007109c --- /dev/null +++ b/dom/media/webrtc/third_party_build/save_patch_stack.py @@ -0,0 +1,142 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import argparse +import os +import re +import shutil + +from run_operations import run_git, run_hg, run_shell + +# This script saves the mozilla patch stack and no-op commit tracking +# files. This makes our fast-forward process much more resilient by +# saving the intermediate state after each upstream commit is processed. + + +def save_patch_stack( + github_path, + github_branch, + patch_directory, + state_directory, + target_branch_head, + bug_number, +): + # remove the current patch files + files_to_remove = os.listdir(patch_directory) + for file in files_to_remove: + os.remove(os.path.join(patch_directory, file)) + + # find the base of the patch stack + cmd = "git merge-base {} {}".format(github_branch, target_branch_head) + stdout_lines = run_git(cmd, github_path) + merge_base = stdout_lines[0] + + # grab patch stack + cmd = "git format-patch --keep-subject --output-directory {} {}..{}".format( + patch_directory, merge_base, github_branch + ) + run_git(cmd, github_path) + + # remove the commit summary from the file name + patches_to_rename = os.listdir(patch_directory) + for file in patches_to_rename: + shortened_name = re.sub("^(\d\d\d\d)-.*\.patch", "\\1.patch", file) + os.rename( + os.path.join(patch_directory, file), + os.path.join(patch_directory, shortened_name), + ) + + # remove the unhelpful first line of the patch files that only + # causes diff churn. For reasons why we can't skip creating backup + # files during the in-place editing, see: + # https://stackoverflow.com/questions/5694228/sed-in-place-flag-that-works-both-on-mac-bsd-and-linux + run_shell("sed -i'.bak' -e '1d' {}/*.patch".format(patch_directory)) + run_shell("rm {}/*.patch.bak".format(patch_directory)) + + # it is also helpful to save the no-op-cherry-pick-msg files from + # the state directory so that if we're restoring a patch-stack we + # also restore the possibly consumed no-op tracking files. + no_op_files = [ + path + for path in os.listdir(state_directory) + if re.findall(".*no-op-cherry-pick-msg$", path) + ] + for file in no_op_files: + shutil.copy(os.path.join(state_directory, file), patch_directory) + + # get missing files (that should be marked removed) + cmd = "hg status --no-status --deleted {}".format(patch_directory) + stdout_lines = run_hg(cmd) + if len(stdout_lines) != 0: + cmd = "hg rm {}".format(" ".join(stdout_lines)) + run_hg(cmd) + + # get unknown files (that should be marked added) + cmd = "hg status --no-status --unknown {}".format(patch_directory) + stdout_lines = run_hg(cmd) + if len(stdout_lines) != 0: + cmd = "hg add {}".format(" ".join(stdout_lines)) + run_hg(cmd) + + # if any files are marked for add/remove/modify, commit them + cmd = "hg status --added --removed --modified {}".format(patch_directory) + stdout_lines = run_hg(cmd) + if (len(stdout_lines)) != 0: + print("Updating {} files in {}".format(len(stdout_lines), patch_directory)) + if bug_number is None: + run_hg("hg amend") + else: + run_shell( + "hg commit --message 'Bug {} - updated libwebrtc patch stack'".format( + bug_number + ) + ) + + +if __name__ == "__main__": + default_patch_dir = "third_party/libwebrtc/moz-patch-stack" + default_state_dir = ".moz-fast-forward" + + parser = argparse.ArgumentParser( + description="Save moz-libwebrtc github patch stack" + ) + parser.add_argument( + "--repo-path", + required=True, + help="path to libwebrtc repo", + ) + parser.add_argument( + "--branch", + default="mozpatches", + help="moz-libwebrtc branch (defaults to mozpatches)", + ) + parser.add_argument( + "--patch-path", + default=default_patch_dir, + help="path to save patches (defaults to {})".format(default_patch_dir), + ) + parser.add_argument( + "--state-path", + default=default_state_dir, + help="path to state directory (defaults to {})".format(default_state_dir), + ) + parser.add_argument( + "--target-branch-head", + required=True, + help="target branch head for fast-forward, should match MOZ_TARGET_UPSTREAM_BRANCH_HEAD in config_env", + ) + parser.add_argument( + "--separate-commit-bug-number", + type=int, + help="integer Bugzilla number (example: 1800920), if provided will write patch stack as separate commit", + ) + args = parser.parse_args() + + save_patch_stack( + args.repo_path, + args.branch, + os.path.abspath(args.patch_path), + args.state_path, + args.target_branch_head, + args.separate_commit_bug_number, + ) diff --git a/dom/media/webrtc/third_party_build/update_default_config.sh b/dom/media/webrtc/third_party_build/update_default_config.sh new file mode 100644 index 0000000000..f4020f9b45 --- /dev/null +++ b/dom/media/webrtc/third_party_build/update_default_config.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +function show_error_msg() +{ + echo "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +if [ "x" = "x$NEW_BUG_NUMBER" ]; then + echo "NEW_BUG_NUMBER is not defined. You should probably have a new bug" + echo "number defined for the next fast-forward update. Then do:" + echo " NEW_BUG_NUMBER={new-bug-number} bash $0" + exit +fi + +source dom/media/webrtc/third_party_build/use_config_env.sh + +if [ "x$MOZ_NEXT_LIBWEBRTC_MILESTONE" = "x" ]; then + echo "MOZ_NEXT_LIBWEBRTC_MILESTONE is not defined, see README.md" + exit +fi + +if [ "x$MOZ_NEXT_FIREFOX_REL_TARGET" = "x" ]; then + echo "MOZ_NEXT_FIREFOX_REL_TARGET is not defined, see README.md" + exit +fi + + +# After this point: +# * eE: All commands should succeed. +# * u: All variables should be defined before use. +# * o pipefail: All stages of all pipes should succeed. +set -eEuo pipefail + +ERROR_HELP=$" +An error has occurred running $SCRIPT_DIR/write_default_config.py +" +./mach python $SCRIPT_DIR/write_default_config.py \ + --bug-number $NEW_BUG_NUMBER \ + --milestone $MOZ_NEXT_LIBWEBRTC_MILESTONE \ + --release-target $MOZ_NEXT_FIREFOX_REL_TARGET \ + > $SCRIPT_DIR/default_config_env diff --git a/dom/media/webrtc/third_party_build/use_config_env.sh b/dom/media/webrtc/third_party_build/use_config_env.sh new file mode 100644 index 0000000000..bd78dc7e0e --- /dev/null +++ b/dom/media/webrtc/third_party_build/use_config_env.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +# Assume that if STATE_DIR is already defined, we do not need to +# execute this file again. +if [ "x$STATE_DIR" != "x" ]; then + # no need to run script since we've already run + return +fi + +export SCRIPT_DIR="dom/media/webrtc/third_party_build" +# first, make sure we're running from the top of moz-central repo +if [ ! -d $SCRIPT_DIR ]; then + echo "Error: unable to find directory $SCRIPT_DIR" + exit 1 +fi + +# Should we tie the location of the STATE_DIR to the path +# in MOZ_CONFIG_PATH? Probably. +export STATE_DIR=`pwd`/.moz-fast-forward +export LOG_DIR=$STATE_DIR/logs +export TMP_DIR=$STATE_DIR/tmp + +if [ ! -d $STATE_DIR ]; then + echo "Creating missing $STATE_DIR" + mkdir -p $STATE_DIR + if [ ! -d $STATE_DIR ]; then + echo "error: unable to find (or create) $STATE_DIR" + exit 1 + fi +fi +echo "Using STATE_DIR=$STATE_DIR" + +if [ ! -d $LOG_DIR ]; then + echo "Creating missing $LOG_DIR" + mkdir -p $LOG_DIR + if [ ! -d $LOG_DIR ]; then + echo "error: unable to find (or create) $LOG_DIR" + exit 1 + fi +fi +echo "Using LOG_DIR=$LOG_DIR" + +if [ ! -d $TMP_DIR ]; then + echo "Creating missing $TMP_DIR" + mkdir -p $TMP_DIR + if [ ! -d $TMP_DIR ]; then + echo "error: unable to find (or create) $TMP_DIR" + exit 1 + fi +fi +echo "Using TMP_DIR=$TMP_DIR" + +# Allow user to override default path to config_env +if [ "x$MOZ_CONFIG_PATH" = "x" ]; then + MOZ_CONFIG_PATH=$STATE_DIR/config_env + echo "Using default MOZ_CONFIG_PATH=$MOZ_CONFIG_PATH" +fi + +if [ ! -f $MOZ_CONFIG_PATH ]; then + echo "Creating default config file at $MOZ_CONFIG_PATH" + cp $SCRIPT_DIR/default_config_env $MOZ_CONFIG_PATH +fi +source $MOZ_CONFIG_PATH + + +function find_base_commit() +{ + # read the last line of README.moz-ff-commit to retrieve our current base + # commit in moz-libwebrtc + MOZ_LIBWEBRTC_BASE=`tail -1 third_party/libwebrtc/README.moz-ff-commit` + echo "prelim MOZ_LIBWEBRTC_BASE: $MOZ_LIBWEBRTC_BASE" + # if we've advanced into a chrome release branch, we need to adjust the + # MOZ_LIBWEBRTC_BASE to the last common commit so we can now advance up + # the trunk commits. + MOZ_LIBWEBRTC_BASE=`cd $MOZ_LIBWEBRTC_SRC ; git merge-base $MOZ_LIBWEBRTC_BASE $MOZ_TARGET_UPSTREAM_BRANCH_HEAD` + # now make it a short hash + MOZ_LIBWEBRTC_BASE=`cd $MOZ_LIBWEBRTC_SRC ; git rev-parse --short $MOZ_LIBWEBRTC_BASE` + echo "adjusted MOZ_LIBWEBRTC_BASE: $MOZ_LIBWEBRTC_BASE" +} +export -f find_base_commit + +function find_next_commit() +{ + # identify the next commit above our current base commit + MOZ_LIBWEBRTC_NEXT_BASE=`cd $MOZ_LIBWEBRTC_SRC ; \ + git log --oneline --ancestry-path $MOZ_LIBWEBRTC_BASE^..$MOZ_TARGET_UPSTREAM_BRANCH_HEAD \ + | tail -2 | head -1 | awk '{print $1;}'` +} +export -f find_next_commit diff --git a/dom/media/webrtc/third_party_build/vendor-libwebrtc.py b/dom/media/webrtc/third_party_build/vendor-libwebrtc.py new file mode 100644 index 0000000000..99723849c7 --- /dev/null +++ b/dom/media/webrtc/third_party_build/vendor-libwebrtc.py @@ -0,0 +1,419 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import argparse +import datetime +import os +import shutil +import stat +import subprocess +import sys +import tarfile + +import requests + +THIRDPARTY_USED_IN_FIREFOX = [ + "abseil-cpp", + "google_benchmark", + "pffft", + "rnnoise", +] + +LIBWEBRTC_DIR = os.path.normpath("third_party/libwebrtc") + + +def get_excluded_paths(): + return [ + ".clang-format", + ".git-blame-ignore-revs", + ".gitignore", + ".vpython", + "CODE_OF_CONDUCT.md", + "ENG_REVIEW_OWNERS", + "PRESUBMIT.py", + "README.chromium", + "WATCHLISTS", + "codereview.settings", + "license_template.txt", + "native-api.md", + "presubmit_test.py", + "presubmit_test_mocks.py", + "pylintrc", + # Only the camera code under sdk/android/api/org/webrtc is used, so + # we remove sdk/android and add back the specific files we want. + "sdk/android", + ] + + +# Paths in this list are included even if their parent directory is +# excluded in get_excluded_paths() +def get_included_path_overrides(): + return [ + "sdk/android/src/java/org/webrtc/NativeLibrary.java", + "sdk/android/src/java/org/webrtc/FramerateBitrateAdjuster.java", + "sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java", + "sdk/android/src/java/org/webrtc/BitrateAdjuster.java", + "sdk/android/src/java/org/webrtc/MediaCodecWrapperFactory.java", + "sdk/android/src/java/org/webrtc/WebRtcClassLoader.java", + "sdk/android/src/java/org/webrtc/audio/WebRtcAudioRecord.java", + "sdk/android/src/java/org/webrtc/audio/WebRtcAudioTrack.java", + "sdk/android/src/java/org/webrtc/audio/WebRtcAudioManager.java", + "sdk/android/src/java/org/webrtc/audio/LowLatencyAudioBufferManager.java", + "sdk/android/src/java/org/webrtc/audio/WebRtcAudioUtils.java", + "sdk/android/src/java/org/webrtc/audio/WebRtcAudioEffects.java", + "sdk/android/src/java/org/webrtc/audio/VolumeLogger.java", + "sdk/android/src/java/org/webrtc/NativeCapturerObserver.java", + "sdk/android/src/java/org/webrtc/MediaCodecWrapper.java", + "sdk/android/src/java/org/webrtc/CalledByNative.java", + "sdk/android/src/java/org/webrtc/Histogram.java", + "sdk/android/src/java/org/webrtc/EglBase10Impl.java", + "sdk/android/src/java/org/webrtc/EglBase14Impl.java", + "sdk/android/src/java/org/webrtc/MediaCodecWrapperFactoryImpl.java", + "sdk/android/src/java/org/webrtc/AndroidVideoDecoder.java", + "sdk/android/src/java/org/webrtc/BaseBitrateAdjuster.java", + "sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java", + "sdk/android/src/java/org/webrtc/VideoCodecMimeType.java", + "sdk/android/src/java/org/webrtc/NativeAndroidVideoTrackSource.java", + "sdk/android/src/java/org/webrtc/VideoDecoderWrapper.java", + "sdk/android/src/java/org/webrtc/JNILogging.java", + "sdk/android/src/java/org/webrtc/CameraCapturer.java", + "sdk/android/src/java/org/webrtc/CameraSession.java", + "sdk/android/src/java/org/webrtc/H264Utils.java", + "sdk/android/src/java/org/webrtc/Empty.java", + "sdk/android/src/java/org/webrtc/DynamicBitrateAdjuster.java", + "sdk/android/src/java/org/webrtc/Camera1Session.java", + "sdk/android/src/java/org/webrtc/JniCommon.java", + "sdk/android/src/java/org/webrtc/NV12Buffer.java", + "sdk/android/src/java/org/webrtc/WrappedNativeI420Buffer.java", + "sdk/android/src/java/org/webrtc/GlGenericDrawer.java", + "sdk/android/src/java/org/webrtc/RefCountDelegate.java", + "sdk/android/src/java/org/webrtc/Camera2Session.java", + "sdk/android/src/java/org/webrtc/MediaCodecUtils.java", + "sdk/android/src/java/org/webrtc/CalledByNativeUnchecked.java", + "sdk/android/src/java/org/webrtc/VideoEncoderWrapper.java", + "sdk/android/src/java/org/webrtc/NV21Buffer.java", + "sdk/android/api/org/webrtc/RendererCommon.java", + "sdk/android/api/org/webrtc/YuvHelper.java", + "sdk/android/api/org/webrtc/LibvpxVp9Encoder.java", + "sdk/android/api/org/webrtc/Metrics.java", + "sdk/android/api/org/webrtc/CryptoOptions.java", + "sdk/android/api/org/webrtc/MediaConstraints.java", + "sdk/android/api/org/webrtc/YuvConverter.java", + "sdk/android/api/org/webrtc/JavaI420Buffer.java", + "sdk/android/api/org/webrtc/VideoDecoder.java", + "sdk/android/api/org/webrtc/WrappedNativeVideoDecoder.java", + "sdk/android/api/org/webrtc/Camera2Enumerator.java", + "sdk/android/api/org/webrtc/SurfaceTextureHelper.java", + "sdk/android/api/org/webrtc/EglBase10.java", + "sdk/android/api/org/webrtc/DataChannel.java", + "sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java", + "sdk/android/api/org/webrtc/audio/AudioDeviceModule.java", + "sdk/android/api/org/webrtc/audio/LegacyAudioDeviceModule.java", + "sdk/android/api/org/webrtc/SessionDescription.java", + "sdk/android/api/org/webrtc/GlUtil.java", + "sdk/android/api/org/webrtc/VideoSource.java", + "sdk/android/api/org/webrtc/AudioTrack.java", + "sdk/android/api/org/webrtc/EglRenderer.java", + "sdk/android/api/org/webrtc/VideoEncoder.java", + "sdk/android/api/org/webrtc/VideoCapturer.java", + "sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java", + "sdk/android/api/org/webrtc/AudioSource.java", + "sdk/android/api/org/webrtc/GlRectDrawer.java", + "sdk/android/api/org/webrtc/StatsReport.java", + "sdk/android/api/org/webrtc/CameraVideoCapturer.java", + "sdk/android/api/org/webrtc/NetEqFactoryFactory.java", + "sdk/android/api/org/webrtc/AudioProcessingFactory.java", + "sdk/android/api/org/webrtc/Camera2Capturer.java", + "sdk/android/api/org/webrtc/ScreenCapturerAndroid.java", + "sdk/android/api/org/webrtc/RefCounted.java", + "sdk/android/api/org/webrtc/VideoEncoderFallback.java", + "sdk/android/api/org/webrtc/AudioEncoderFactoryFactory.java", + "sdk/android/api/org/webrtc/EglBase14.java", + "sdk/android/api/org/webrtc/SoftwareVideoEncoderFactory.java", + "sdk/android/api/org/webrtc/VideoEncoderFactory.java", + "sdk/android/api/org/webrtc/StatsObserver.java", + "sdk/android/api/org/webrtc/PlatformSoftwareVideoDecoderFactory.java", + "sdk/android/api/org/webrtc/Camera1Capturer.java", + "sdk/android/api/org/webrtc/AddIceObserver.java", + "sdk/android/api/org/webrtc/SurfaceViewRenderer.java", + "sdk/android/api/org/webrtc/CameraEnumerator.java", + "sdk/android/api/org/webrtc/CameraEnumerationAndroid.java", + "sdk/android/api/org/webrtc/VideoDecoderFallback.java", + "sdk/android/api/org/webrtc/FileVideoCapturer.java", + "sdk/android/api/org/webrtc/NativeLibraryLoader.java", + "sdk/android/api/org/webrtc/Camera1Enumerator.java", + "sdk/android/api/org/webrtc/NativePeerConnectionFactory.java", + "sdk/android/api/org/webrtc/LibaomAv1Encoder.java", + "sdk/android/api/org/webrtc/BuiltinAudioEncoderFactoryFactory.java", + "sdk/android/api/org/webrtc/AudioDecoderFactoryFactory.java", + "sdk/android/api/org/webrtc/FecControllerFactoryFactoryInterface.java", + "sdk/android/api/org/webrtc/VideoFrameBufferType.java", + "sdk/android/api/org/webrtc/SdpObserver.java", + "sdk/android/api/org/webrtc/Predicate.java", + "sdk/android/api/org/webrtc/VideoFileRenderer.java", + "sdk/android/api/org/webrtc/WrappedNativeVideoEncoder.java", + "sdk/android/api/org/webrtc/LibvpxVp8Encoder.java", + "sdk/android/api/org/webrtc/DtmfSender.java", + "sdk/android/api/org/webrtc/VideoTrack.java", + "sdk/android/api/org/webrtc/LibvpxVp8Decoder.java", + "sdk/android/api/org/webrtc/GlShader.java", + "sdk/android/api/org/webrtc/FrameEncryptor.java", + "sdk/android/api/org/webrtc/EglBase.java", + "sdk/android/api/org/webrtc/VideoProcessor.java", + "sdk/android/api/org/webrtc/SSLCertificateVerifier.java", + "sdk/android/api/org/webrtc/VideoSink.java", + "sdk/android/api/org/webrtc/MediaSource.java", + "sdk/android/api/org/webrtc/DefaultVideoDecoderFactory.java", + "sdk/android/api/org/webrtc/VideoCodecInfo.java", + "sdk/android/api/org/webrtc/FrameDecryptor.java", + "sdk/android/api/org/webrtc/VideoDecoderFactory.java", + "sdk/android/api/org/webrtc/TextureBufferImpl.java", + "sdk/android/api/org/webrtc/VideoFrame.java", + "sdk/android/api/org/webrtc/IceCandidateErrorEvent.java", + "sdk/android/api/org/webrtc/CapturerObserver.java", + "sdk/android/api/org/webrtc/MediaStreamTrack.java", + "sdk/android/api/org/webrtc/GlTextureFrameBuffer.java", + "sdk/android/api/org/webrtc/TurnCustomizer.java", + "sdk/android/api/org/webrtc/TimestampAligner.java", + "sdk/android/api/org/webrtc/BuiltinAudioDecoderFactoryFactory.java", + "sdk/android/api/org/webrtc/LibvpxVp9Decoder.java", + "sdk/android/api/org/webrtc/SurfaceEglRenderer.java", + "sdk/android/api/org/webrtc/HardwareVideoDecoderFactory.java", + "sdk/android/api/org/webrtc/VideoCodecStatus.java", + "sdk/android/api/org/webrtc/Dav1dDecoder.java", + "sdk/android/api/org/webrtc/VideoFrameDrawer.java", + "sdk/android/api/org/webrtc/CallSessionFileRotatingLogSink.java", + "sdk/android/api/org/webrtc/EncodedImage.java", + ] + + +def make_github_url(repo, commit): + if not repo.endswith("/"): + repo += "/" + return repo + "archive/" + commit + ".tar.gz" + + +def make_googlesource_url(target, commit): + if target == "libwebrtc": + return "https://webrtc.googlesource.com/src.git/+archive/" + commit + ".tar.gz" + elif target == "build": + return ( + "https://chromium.googlesource.com/chromium/src/build/+archive/" + + commit + + ".tar.gz" + ) + elif target == "third_party": + return ( + "https://chromium.googlesource.com/chromium/src/third_party/+archive/" + + commit + + ".tar.gz" + ) + + +def fetch(target, url): + print("Fetching commit from {}".format(url)) + req = requests.get(url) + if req.status_code == 200: + with open(target + ".tar.gz", "wb") as f: + f.write(req.content) + else: + print( + "Hit status code {} fetching commit. Aborting.".format(req.status_code), + file=sys.stderr, + ) + sys.exit(1) + with open(os.path.join(LIBWEBRTC_DIR, "README.mozilla"), "a") as f: + # write the the command line used + f.write("# ./mach python {}\n".format(" ".join(sys.argv[0:]))) + f.write( + "{} updated from commit {} on {}.\n".format( + target, url, datetime.datetime.utcnow().isoformat() + ) + ) + + +def fetch_local(target, path, commit): + target_archive = target + ".tar.gz" + cp = subprocess.run(["git", "archive", "-o", target_archive, commit], cwd=path) + if cp.returncode != 0: + print( + "Hit return code {} fetching commit. Aborting.".format(cp.returncode), + file=sys.stderr, + ) + sys.exit(1) + + with open(os.path.join(LIBWEBRTC_DIR, "README.mozilla"), "a") as f: + # write the the command line used + f.write("# ./mach python {}\n".format(" ".join(sys.argv[0:]))) + f.write( + "{} updated from {} commit {} on {}.\n".format( + target, path, commit, datetime.datetime.utcnow().isoformat() + ) + ) + shutil.move(os.path.join(path, target_archive), target_archive) + + +def validate_tar_member(member, path): + def _is_within_directory(directory, target): + real_directory = os.path.realpath(directory) + real_target = os.path.realpath(target) + prefix = os.path.commonprefix([real_directory, real_target]) + return prefix == real_directory + + member_path = os.path.join(path, member.name) + if not _is_within_directory(path, member_path): + raise Exception("Attempted path traversal in tar file: " + member.name) + if member.issym(): + link_path = os.path.join(os.path.dirname(member_path), member.linkname) + if not _is_within_directory(path, link_path): + raise Exception("Attempted link path traversal in tar file: " + member.name) + if member.mode & (stat.S_ISUID | stat.S_ISGID): + raise Exception("Attempted setuid or setgid in tar file: " + member.name) + + +def safe_extract(tar, path=".", *, numeric_owner=False): + def _files(tar, path): + for member in tar: + validate_tar_member(member, path) + yield member + + tar.extractall(path, members=_files(tar, path), numeric_owner=numeric_owner) + + +def unpack(target): + target_archive = target + ".tar.gz" + target_path = "tmp-" + target + try: + shutil.rmtree(target_path) + except FileNotFoundError: + pass + with tarfile.open(target_archive) as t: + safe_extract(t, path=target_path) + + if target == "libwebrtc": + # use the top level directories from the tarfile and + # delete those directories in LIBWEBRTC_DIR + libwebrtc_used_in_firefox = os.listdir(target_path) + for path in libwebrtc_used_in_firefox: + try: + shutil.rmtree(os.path.join(LIBWEBRTC_DIR, path)) + except FileNotFoundError: + pass + except NotADirectoryError: + pass + + unused_libwebrtc_in_firefox = get_excluded_paths() + forced_used_in_firefox = get_included_path_overrides() + + # adjust target_path if GitHub packaging is involved + if not os.path.exists(os.path.join(target_path, libwebrtc_used_in_firefox[0])): + # GitHub packs everything inside a separate directory + target_path = os.path.join(target_path, os.listdir(target_path)[0]) + + # remove any entries found in unused_libwebrtc_in_firefox from the + # tarfile + for path in unused_libwebrtc_in_firefox: + if os.path.isdir(os.path.join(target_path, path)): + shutil.rmtree(os.path.join(target_path, path)) + else: + os.remove(os.path.join(target_path, path)) + + # move remaining top level entries from the tarfile to LIBWEBRTC_DIR + for path in os.listdir(target_path): + shutil.move( + os.path.join(target_path, path), os.path.join(LIBWEBRTC_DIR, path) + ) + + # An easy, but inefficient way to accomplish including specific + # files from directories otherwise removed. Re-extract the tar + # file, and only copy over the exact files requested. + shutil.rmtree(target_path) + with tarfile.open(target_archive) as t: + safe_extract(t, path=target_path) + + # Copy the force included files. Note: the instinctual action + # is to do this prior to removing the excluded paths to avoid + # reextracting the tar file. However, this causes errors due to + # pre-existing paths when other directories are moved out of the + # tar file in the "move all the top level entries from the + # tarfile" phase above. + for path in forced_used_in_firefox: + dest_path = os.path.join(LIBWEBRTC_DIR, path) + dir_path = os.path.dirname(dest_path) + if not os.path.exists(dir_path): + os.makedirs(dir_path) + shutil.move(os.path.join(target_path, path), dest_path) + elif target == "build": + try: + shutil.rmtree(os.path.join(LIBWEBRTC_DIR, "build")) + except FileNotFoundError: + pass + os.makedirs(os.path.join(LIBWEBRTC_DIR, "build")) + + if os.path.exists(os.path.join(target_path, "linux")): + for path in os.listdir(target_path): + shutil.move( + os.path.join(target_path, path), + os.path.join(LIBWEBRTC_DIR, "build", path), + ) + else: + # GitHub packs everything inside a separate directory + target_path = os.path.join(target_path, os.listdir(target_path)[0]) + for path in os.listdir(target_path): + shutil.move( + os.path.join(target_path, path), + os.path.join(LIBWEBRTC_DIR, "build", path), + ) + elif target == "third_party": + try: + shutil.rmtree(os.path.join(LIBWEBRTC_DIR, "third_party")) + except FileNotFoundError: + pass + except NotADirectoryError: + pass + + if os.path.exists(os.path.join(target_path, THIRDPARTY_USED_IN_FIREFOX[0])): + for path in THIRDPARTY_USED_IN_FIREFOX: + shutil.move( + os.path.join(target_path, path), + os.path.join(LIBWEBRTC_DIR, "third_party", path), + ) + else: + # GitHub packs everything inside a separate directory + target_path = os.path.join(target_path, os.listdir(target_path)[0]) + for path in THIRDPARTY_USED_IN_FIREFOX: + shutil.move( + os.path.join(target_path, path), + os.path.join(LIBWEBRTC_DIR, "third_party", path), + ) + + +def cleanup(target): + os.remove(target + ".tar.gz") + shutil.rmtree("tmp-" + target) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Update libwebrtc") + parser.add_argument("target", choices=("libwebrtc", "build", "third_party")) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--from-github", type=str) + group.add_argument("--from-googlesource", action="store_true", default=False) + group.add_argument("--from-local", type=str) + parser.add_argument("--commit", type=str, default="master") + parser.add_argument("--skip-fetch", action="store_true", default=False) + parser.add_argument("--skip-cleanup", action="store_true", default=False) + args = parser.parse_args() + + os.makedirs(LIBWEBRTC_DIR, exist_ok=True) + + if not args.skip_fetch: + if args.from_github: + fetch(args.target, make_github_url(args.from_github, args.commit)) + elif args.from_googlesource: + fetch(args.target, make_googlesource_url(args.target, args.commit)) + elif args.from_local: + fetch_local(args.target, args.from_local, args.commit) + unpack(args.target) + if not args.skip_cleanup: + cleanup(args.target) diff --git a/dom/media/webrtc/third_party_build/verify_vendoring.sh b/dom/media/webrtc/third_party_build/verify_vendoring.sh new file mode 100644 index 0000000000..3d2c161075 --- /dev/null +++ b/dom/media/webrtc/third_party_build/verify_vendoring.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +function show_error_msg() +{ + echo "*** ERROR *** $? line $1 $0 did not complete successfully!" + echo "$ERROR_HELP" +} +ERROR_HELP="" + +# Print an Error message if `set -eE` causes the script to exit due to a failed command +trap 'show_error_msg $LINENO' ERR + +source dom/media/webrtc/third_party_build/use_config_env.sh + +echo "MOZ_LIBWEBRTC_SRC: $MOZ_LIBWEBRTC_SRC" +echo "MOZ_LIBWEBRTC_BRANCH: $MOZ_LIBWEBRTC_BRANCH" +echo "MOZ_FASTFORWARD_BUG: $MOZ_FASTFORWARD_BUG" + +# After this point: +# * eE: All commands should succeed. +# * u: All variables should be defined before use. +# * o pipefail: All stages of all pipes should succeed. +set -eEuo pipefail + +./mach python $SCRIPT_DIR/vendor-libwebrtc.py \ + --from-local $MOZ_LIBWEBRTC_SRC \ + --commit $MOZ_LIBWEBRTC_BRANCH \ + libwebrtc + +hg revert -q \ + --include "third_party/libwebrtc/**moz.build" \ + --include "third_party/libwebrtc/README.mozilla" \ + third_party/libwebrtc + +FILE_CHANGE_CNT=`hg status third_party/libwebrtc | wc -l | tr -d " "` +if [ "x$FILE_CHANGE_CNT" != "x0" ]; then + echo "***" + echo "There are changes after vendoring - running extract-for-git.py" + echo "is recommended. First, find the mercurial commit after the" + echo "previous fast-forward landing. The commands you want will look" + echo "something like:" + echo " ./mach python $SCRIPT_DIR/extract-for-git.py {after-ff-commit}::{tip-of-central}" + echo " mv mailbox.patch $MOZ_LIBWEBRTC_SRC" + echo " (cd $MOZ_LIBWEBRTC_SRC && \\" + echo " git am mailbox.patch)" + echo "" + echo "After adding the new changes from moz-central to the moz-libwebrtc" + echo "patch stack, you may re-run this command to verify vendoring:" + echo " bash $0" + + exit 1 +fi + + +echo "Done - vendoring has been verified." diff --git a/dom/media/webrtc/third_party_build/webrtc.mozbuild b/dom/media/webrtc/third_party_build/webrtc.mozbuild new file mode 100644 index 0000000000..30169c36c2 --- /dev/null +++ b/dom/media/webrtc/third_party_build/webrtc.mozbuild @@ -0,0 +1,40 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file mimics the defines used by any headers in libwebrtc that get +# included in Mozilla code. +# We currently audit these by hand. Searching for the defines upstream is a good +# start: +# https://source.chromium.org/search?q=usage:%23if.*def.*%5B%5E_%5D$%20AND%20path:%5C.h%20AND%20path:%5Ethird_party%2Fwebrtc&ss=chromium + +if CONFIG['MOZ_WEBRTC']: + DEFINES['HAVE_UINT64_T'] = True + DEFINES['WEBRTC_MOZILLA_BUILD'] = True + DEFINES['RTC_ENABLE_VP9'] = True + + if CONFIG['OS_TARGET'] != 'WINNT': + DEFINES['WEBRTC_POSIX'] = True + DEFINES['WEBRTC_BUILD_LIBEVENT'] = True + + if CONFIG['OS_TARGET'] == 'Linux': + DEFINES['WEBRTC_LINUX'] = True + elif CONFIG['OS_TARGET'] == 'Darwin': + DEFINES['WEBRTC_MAC'] = True + elif CONFIG['OS_TARGET'] == 'WINNT': + DEFINES['WEBRTC_WIN'] = True + DEFINES['RTC_ENABLE_WIN_WGC'] = False + DEFINES['HAVE_WINSOCK2_H'] = True + elif CONFIG['OS_TARGET'] in ('DragonFly', 'FreeBSD', 'NetBSD', 'OpenBSD'): + DEFINES['WEBRTC_BSD'] = True + elif CONFIG['OS_TARGET'] == 'Android': + DEFINES['WEBRTC_LINUX'] = True + DEFINES['WEBRTC_ANDROID'] = True + + if CONFIG['MOZ_X11']: + DEFINES['WEBRTC_USE_X11'] = True + + if CONFIG["MOZ_WIDGET_TOOLKIT"] == "gtk": + DEFINES['WEBRTC_USE_PIPEWIRE'] = True diff --git a/dom/media/webrtc/third_party_build/write_default_config.py b/dom/media/webrtc/third_party_build/write_default_config.py new file mode 100644 index 0000000000..204789e052 --- /dev/null +++ b/dom/media/webrtc/third_party_build/write_default_config.py @@ -0,0 +1,104 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import argparse +import sys +from string import Template + +sys.path.insert(0, "./dom/media/webrtc/third_party_build") +import lookup_branch_head + +text = """#!/bin/bash + +# Edit {path-to} to match the location of your copy of Mozilla's +# fork of libwebrtc (at https://github.com/mozilla/libwebrtc). +export MOZ_LIBWEBRTC_SRC=$$STATE_DIR/moz-libwebrtc + +# Fast-forwarding each Chromium version of libwebrtc should be done +# under a separate bugzilla bug. This bug number is used when crafting +# the commit summary as each upstream commit is vendored into the +# mercurial repository. The bug used for the v106 fast-forward was +# 1800920. +export MOZ_FASTFORWARD_BUG="$bugnum" + +# MOZ_NEXT_LIBWEBRTC_MILESTONE and MOZ_NEXT_FIREFOX_REL_TARGET are +# not used during fast-forward processing, but facilitate generating this +# default config. To generate an default config for the next update, run +# bash dom/media/webrtc/third_party_build/update_default_config_env.sh +export MOZ_NEXT_LIBWEBRTC_MILESTONE=$m2 +export MOZ_NEXT_FIREFOX_REL_TARGET=$t2 + +# For Chromium release branches, see: +# https://chromiumdash.appspot.com/branches + +# Chromium's v$m1 release branch was $bh1. This is used to pre-stack +# the previous release branch's commits onto the appropriate base commit +# (the first common commit between trunk and the release branch). +export MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM="$bh1" + +# New target release branch for v$m2 is branch-heads/$bh2. This is used +# to calculate the next upstream commit. +export MOZ_TARGET_UPSTREAM_BRANCH_HEAD="branch-heads/$bh2" + +# For local development 'mozpatches' is fine for a branch name, but when +# pushing the patch stack to github, it should be named something like +# 'moz-mods-chr$m2-for-rel$t2'. +export MOZ_LIBWEBRTC_BRANCH="mozpatches" + +# After elm has been merged to mozilla-central, the patch stack in +# moz-libwebrtc should be pushed to github. The script +# push_official_branch.sh uses this branch name when pushing to the +# public repo. +export MOZ_LIBWEBRTC_OFFICIAL_BRANCH="moz-mods-chr$m2-for-rel$t2" +""" + + +def build_default_config_env(bug_number, milestone, target): + prior_branch_head = lookup_branch_head.get_branch_head(milestone) + if prior_branch_head is None: + sys.exit("error: chromium milestone '{}' is not found.".format(milestone)) + new_branch_head = lookup_branch_head.get_branch_head(milestone + 1) + if new_branch_head is None: + sys.exit( + "error: next chromium milestone '{}' is not found.".format(milestone + 1) + ) + + s = Template(text) + return s.substitute( + bugnum=bug_number, + m1=milestone, + m2=milestone + 1, + t2=target + 1, + bh1=prior_branch_head, + bh2=new_branch_head, + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Updates the default_config_env file for new release/milestone" + ) + parser.add_argument( + "--bug-number", + required=True, + type=int, + help="integer Bugzilla number (example: 1800920)", + ) + parser.add_argument( + "--milestone", + required=True, + type=int, + help="integer chromium milestone (example: 106)", + ) + parser.add_argument( + "--release-target", + required=True, + type=int, + help="integer firefox release (example: 110)", + ) + args = parser.parse_args() + + print( + build_default_config_env(args.bug_number, args.milestone, args.release_target), + end="", + ) |