path: root/dom/media/webrtc/third_party_build/cherry_pick_commit.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import argparse
import atexit
import os
import re
import shutil
import sys

from filter_git_changes import filter_git_changes
from restore_patch_stack import restore_patch_stack
from run_operations import (
    get_last_line,
    run_git,
    run_hg,
    run_shell,
    update_resume_state,
)
from vendor_and_commit import vendor_and_commit

# This script cherry-picks an upstream commit with the appropriate
# commit message, and adds the no-op commit tracking file for the when
# we vendor the upstream commit later.

error_help = None
script_name = os.path.basename(__file__)


def early_exit_handler():
    print(f"*** ERROR *** {script_name} did not complete successfully")
    if error_help is not None:
        print(error_help)


def write_commit_message_file(
    commit_message_filename,
    github_path,
    github_sha,
    bug_number,
    reviewers,
):
    print(f"commit_message_filename: {commit_message_filename}")
    print(f"github_path: {github_path}")
    print(f"github_sha: {github_sha}")
    print(f"bug_number: {bug_number}")

    cmd = "git show --format=%H --no-patch {}".format(github_sha)
    stdout_lines = run_git(cmd, github_path)
    github_long_sha = stdout_lines[0]
    print(f"github_long_sha: {github_long_sha}")

    cmd = "git show --format=%s%n%n%b --no-patch {}".format(github_sha)
    github_commit_msg_lines = run_git(cmd, github_path)

    with open(commit_message_filename, "w") as ofile:
        ofile.write(
            "Bug {} - Cherry-pick upstream libwebrtc commit {} r?{}".format(
                bug_number, github_sha, reviewers
            )
        )
        ofile.write("\n")
        ofile.write("\n")
        ofile.write(
            "Upstream commit: https://webrtc.googlesource.com/src/+/{}".format(
                github_long_sha
            )
        )
        ofile.write("\n")
        for line in github_commit_msg_lines:
            ofile.write("       {}".format(line))
            ofile.write("\n")


def cherry_pick_commit(
    commit_message_filename,
    github_path,
    github_sha,
):
    print(f"commit_message_filename: {commit_message_filename}")
    print(f"github_path: {github_path}")
    print(f"github_sha: {github_sha}")

    cmd = "git cherry-pick --no-commit {}".format(github_sha)
    run_git(cmd, github_path)

    cmd = "git commit --file {}".format(os.path.abspath(commit_message_filename))
    run_git(cmd, github_path)


def write_noop_tracking_file(
    github_sha,
    bug_number,
):
    noop_basename = "{}.no-op-cherry-pick-msg".format(github_sha)
    noop_filename = os.path.join(args.state_path, noop_basename)
    print(f"noop_filename: {noop_filename}")
    with open(noop_filename, "w") as ofile:
        ofile.write("We cherry-picked this in bug {}".format(bug_number))
        ofile.write("\n")
    shutil.copy(noop_filename, args.patch_path)
    cmd = "hg add {}".format(os.path.join(args.patch_path, noop_basename))
    run_hg(cmd)
    cmd = "hg amend {}".format(os.path.join(args.patch_path, noop_basename))
    run_hg(cmd)


if __name__ == "__main__":
    default_target_dir = "third_party/libwebrtc"
    default_state_dir = ".moz-fast-forward"
    default_log_dir = ".moz-fast-forward/logs"
    default_tmp_dir = ".moz-fast-forward/tmp"
    default_script_dir = "dom/media/webrtc/third_party_build"
    default_patch_dir = "third_party/libwebrtc/moz-patch-stack"
    default_repo_dir = ".moz-fast-forward/moz-libwebrtc"
    default_tar_name = "moz-libwebrtc.tar.gz"

    parser = argparse.ArgumentParser(
        description="Cherry-pick upstream libwebrtc commit"
    )
    parser.add_argument(
        "--target-path",
        default=default_target_dir,
        help="target path for vendoring (defaults to {})".format(default_target_dir),
    )
    parser.add_argument(
        "--state-path",
        default=default_state_dir,
        help="path to state directory (defaults to {})".format(default_state_dir),
    )
    parser.add_argument(
        "--log-path",
        default=default_log_dir,
        help="path to log directory (defaults to {})".format(default_log_dir),
    )
    parser.add_argument(
        "--tmp-path",
        default=default_tmp_dir,
        help="path to tmp directory (defaults to {})".format(default_tmp_dir),
    )
    parser.add_argument(
        "--script-path",
        default=default_script_dir,
        help="path to script directory (defaults to {})".format(default_script_dir),
    )
    parser.add_argument(
        "--repo-path",
        default=default_repo_dir,
        help="path to moz-libwebrtc repo (defaults to {})".format(default_repo_dir),
    )
    parser.add_argument(
        "--tar-name",
        default=default_tar_name,
        help="name of tar file (defaults to {})".format(default_tar_name),
    )
    parser.add_argument(
        "--commit-sha",
        required=True,
        help="sha of commit to examine",
    )
    parser.add_argument(
        "--branch",
        default="mozpatches",
        help="moz-libwebrtc branch (defaults to mozpatches)",
    )
    parser.add_argument(
        "--commit-bug-number",
        type=int,
        required=True,
        help="integer Bugzilla number (example: 1800920)",
    )
    parser.add_argument(
        "--patch-path",
        default=default_patch_dir,
        help="path to save patches (defaults to {})".format(default_patch_dir),
    )
    parser.add_argument(
        "--reviewers",
        required=True,
        help='reviewers for cherry-picked patch (like "ng,mjf")',
    )
    parser.add_argument(
        "--abort",
        action="store_true",
        default=False,
        help="abort an interrupted cherry-pick",
    )
    parser.add_argument(
        "--continue",
        dest="cont",  # because args.continue causes syntax errors
        action="store_true",
        default=False,
        help="continue an interrupted cherry-pick",
    )
    args = parser.parse_args()

    # register the exit handler after the arg parser completes so '--help' doesn't exit with
    # an error.
    atexit.register(early_exit_handler)

    # other scripts assume the short-sha is used for various comparisons, so
    # make sure the provided sha is in the short form.
    cmd = f"git show --format=%h --no-patch {args.commit_sha}"
    args.commit_sha = run_git(cmd, args.repo_path)[0]

    commit_message_filename = os.path.join(args.tmp_path, "cherry-pick-commit_msg.txt")

    resume_state_filename = os.path.join(args.state_path, "cherry_pick_commit.resume")
    resume_state = ""
    if os.path.exists(resume_state_filename):
        resume_state = get_last_line(resume_state_filename).strip()
    print(f"resume_state: '{resume_state}'")

    # don't allow abort/continue flags if not in resume state
    error_help = "--abort or --continue flags are not allowed when not in resume state"
    if len(resume_state) == 0 and (args.abort or args.cont):
        sys.exit(1)
    error_help = None

    # detect missing abort/continue flags if in resume state
    error_help = "cherry-pick in progress, use --abort or --continue"
    if len(resume_state) != 0 and not args.abort and not args.cont:
        sys.exit(1)
    error_help = None

    # handle aborting cherry-pick
    if args.abort:
        # If the resume_state is not resume2 or resume3 that means we may
        # have committed something to mercurial.  First we need to check
        # for our cherry-pick commit message, and if found, remove
        # that commit.
        if not (resume_state == "resume2" or resume_state == "resume3"):
            run_hg("hg revert --all")
            run_hg("hg purge {}".format(args.target_path))
            # check for committed mercurial patch and backout
            stdout_lines = run_hg("hg log --template {desc|firstline}\n -r .")
            # check for "Cherry-pick upstream libwebrtc commit"
            print("stdout_lines before filter: {}".format(stdout_lines))
            stdout_lines = [
                line
                for line in stdout_lines
                if re.findall("Cherry-pick upstream libwebrtc commit", line)
            ]
            print("looking for commit: {}".format(stdout_lines))
            if len(stdout_lines) > 0:
                cmd = "hg prune ."
                print("calling '{}'".format(cmd))
                run_hg(cmd)
        print("restoring patch stack")
        restore_patch_stack(
            args.repo_path,
            args.branch,
            os.path.abspath(args.patch_path),
            args.state_path,
            args.tar_name,
            "https",  # unused in this case
        )
        # reset the resume file
        print("reset resume file")
        update_resume_state("", resume_state_filename)
        print("after resetting resume file")
        atexit.unregister(early_exit_handler)
        sys.exit(0)

    # make sure the mercurial repo is clean before beginning
    error_help = (
        "There are modified or untracked files in the mercurial repo.\n"
        f"Please start with a clean repo before running {script_name}"
    )
    stdout_lines = run_hg("hg status")
    if len(stdout_lines) != 0:
        sys.exit(1)

    # make sure the github repo exists
    error_help = (
        f"No moz-libwebrtc github repo found at {args.repo_path}\n"
        f"Please run restore_patch_stack.py before running {script_name}"
    )
    if not os.path.exists(args.repo_path):
        sys.exit(1)
    error_help = None

    if len(resume_state) == 0:
        update_resume_state("resume2", resume_state_filename)
        print("-------")
        print(f"------- write commit message file {commit_message_filename}")
        print("-------")
        write_commit_message_file(
            commit_message_filename,
            args.repo_path,
            args.commit_sha,
            args.commit_bug_number,
            args.reviewers,
        )

    if len(resume_state) == 0 or resume_state == "resume2":
        resume_state = ""
        update_resume_state("resume3", resume_state_filename)
        print("-------")
        print(f"------- cherry-pick {args.commit_sha} into {args.repo_path}")
        print("-------")
        full_commit_message_filename = os.path.abspath(commit_message_filename)
        error_help = (
            f"The cherry-pick operation of {args.commit_sha} has failed.\n"
            "To fix this issue, you will need to jump to the github\n"
            f"repo at {args.repo_path} .\n"
            "Please resolve all the cherry-pick conflicts, and commit the changes\n"
            "using:\n"
            f"    git commit --file {full_commit_message_filename}\n"
            "\n"
            "When the github cherry-pick is complete, resume running this\n"
            f"script ({script_name})"
        )
        cherry_pick_commit(
            commit_message_filename,
            args.repo_path,
            args.commit_sha,
        )
        error_help = None

    if len(resume_state) == 0 or resume_state == "resume3":
        resume_state = ""
        update_resume_state("resume4", resume_state_filename)
        print("-------")
        print(f"------- vendor from {args.repo_path}")
        print("-------")
        error_help = (
            f"Vendoring the newly cherry-picked git commit ({args.commit_sha}) has failed.\n"
            "The mercurial repo is in an unknown state.  This failure is\n"
            "rare and thus makes it difficult to provide definitive guidance.\n"
            "In essence, the current failing command is:\n"
            f"./mach python {args.script_path}/vendor_and_commit.py \\\n"
            f"    --script-path {args.script_path} \\\n"
            f"    --repo-path {args.repo_path} \\\n"
            f"    --branch {args.branch} \\\n"
            f"    --commit-sha {args.commit_sha} \\\n"
            f"    --target-path {args.target_path} \\\n"
            f"    --state-path {args.state_path} \\\n"
            f"    --log-path {args.log_path} \\\n"
            f"    --commit-msg-path {commit_message_filename}\n"
            "\n"
            "Additional guidance may be in the terminal output above.  Resolve\n"
            "issues encountered by vendor_and_commit.py followed by re-running\n"
            "vendor_and_commit.py to resume/complete its processing.  After\n"
            "vendor_and_commit.py completes successfully, resume running\n"
            f"this script ({script_name})"
        )
        vendor_and_commit(
            args.script_path,
            args.repo_path,
            args.branch,
            args.commit_sha,
            args.target_path,  # os.path.abspath(args.target_path),
            args.state_path,
            args.log_path,
            commit_message_filename,
        )
        error_help = None

    if len(resume_state) == 0 or resume_state == "resume4":
        resume_state = ""
        update_resume_state("resume5", resume_state_filename)
        error_help = (
            "Reverting change to 'third_party/libwebrtc/README.mozilla'\n"
            "has failed.  The cherry-pick commit should not modify\n"
            "'third_party/libwebrtc/README.mozilla'.  If necessary\n"
            "manually revert changes to 'third_party/libwebrtc/README.mozilla'\n"
            f"and re-run {script_name}\n"
            "to complete the cherry-pick processing."
        )
        # The normal vendoring process updates README.mozilla with info
        # on what commit was vendored and the command line used to do
        # the vendoring.  Since we're only reusing the vendoring script
        # here, we don't want to update the README.mozilla file.
        cmd = "hg revert -r tip^ third_party/libwebrtc/README.mozilla"
        run_hg(cmd)
        cmd = "hg amend"
        run_hg(cmd)
        error_help = None

    if len(resume_state) == 0 or resume_state == "resume5":
        resume_state = ""
        update_resume_state("resume6", resume_state_filename)
        # get the files changed from the newly vendored cherry-pick
        # commit in mercurial
        cmd = "hg status --change tip --exclude '**/README.*'"
        stdout_lines = run_shell(cmd)  # run_shell to allow file wildcard
        print("Mercurial changes:\n{}".format(stdout_lines))
        hg_file_change_cnt = len(stdout_lines)

        # get the files changed from the original cherry-picked patch in
        # our github repo (moz-libwebrtc)
        git_paths_changed = filter_git_changes(args.repo_path, args.commit_sha, None)
        print("github changes:\n{}".format(git_paths_changed))
        git_file_change_cnt = len(git_paths_changed)

        error_help = (
            f"Vendoring the cherry-pick of commit {args.commit_sha} has failed due to mismatched\n"
            f"changed file counts between mercurial ({hg_file_change_cnt}) and git ({git_file_change_cnt}).\n"
            "This may be because the mozilla patch-stack was not verified after\n"
            "running restore_patch_stack.py.  After reconciling the changes in\n"
            f"the newly committed mercurial patch, please re-run {script_name} to complete\n"
            "the cherry-pick processing."
        )
        if hg_file_change_cnt != git_file_change_cnt:
            sys.exit(1)
        error_help = None

    if len(resume_state) == 0 or resume_state == "resume6":
        resume_state = ""
        update_resume_state("", resume_state_filename)
        print("-------")
        print("------- write the noop tracking file")
        print("-------")
        write_noop_tracking_file(args.commit_sha, args.commit_bug_number)

    # unregister the exit handler so the normal exit doesn't falsely
    # report as an error.
    atexit.unregister(early_exit_handler)