summaryrefslogtreecommitdiffstats
path: root/src/script/ceph-backport.sh
diff options
context:
space:
mode:
Diffstat (limited to 'src/script/ceph-backport.sh')
-rwxr-xr-xsrc/script/ceph-backport.sh1815
1 files changed, 1815 insertions, 0 deletions
diff --git a/src/script/ceph-backport.sh b/src/script/ceph-backport.sh
new file mode 100755
index 000000000..dbcac48d0
--- /dev/null
+++ b/src/script/ceph-backport.sh
@@ -0,0 +1,1815 @@
+#!/usr/bin/env bash
+set -e
+#
+# ceph-backport.sh - Ceph backporting script
+#
+# Credits: This script is based on work done by Loic Dachary
+#
+#
+# This script automates the process of staging a backport starting from a
+# Backport tracker issue.
+#
+# Setup:
+#
+# ceph-backport.sh --setup
+#
+# Usage and troubleshooting:
+#
+# ceph-backport.sh --help
+# ceph-backport.sh --usage | less
+# ceph-backport.sh --troubleshooting | less
+#
+
+full_path="$0"
+
+SCRIPT_VERSION="16.0.0.6848"
+active_milestones=""
+backport_pr_labels=""
+backport_pr_number=""
+backport_pr_title=""
+backport_pr_url=""
+deprecated_backport_common="$HOME/bin/backport_common.sh"
+existing_pr_milestone_number=""
+github_token=""
+github_token_file="$HOME/.github_token"
+github_user=""
+milestone=""
+non_interactive=""
+original_issue=""
+original_issue_url=""
+original_pr=""
+original_pr_url=""
+redmine_key=""
+redmine_key_file="$HOME/.redmine_key"
+redmine_login=""
+redmine_user_id=""
+setup_ok=""
+this_script=$(basename "$full_path")
+
+if [[ $* == *--debug* ]]; then
+ set -x
+fi
+
+# associative array keyed on "component" strings from PR titles, mapping them to
+# GitHub PR labels that make sense in backports
+declare -A comp_hash=(
+["auth"]="core"
+["bluestore"]="bluestore"
+["build/ops"]="build/ops"
+["ceph.spec"]="build/ops"
+["ceph-volume"]="ceph-volume"
+["cephfs"]="cephfs"
+["cmake"]="build/ops"
+["config"]="config"
+["client"]="cephfs"
+["common"]="common"
+["core"]="core"
+["dashboard"]="dashboard"
+["deb"]="build/ops"
+["doc"]="documentation"
+["grafana"]="monitoring"
+["mds"]="cephfs"
+["messenger"]="core"
+["mon"]="core"
+["msg"]="core"
+["mgr/dashboard"]="dashboard"
+["mgr/prometheus"]="monitoring"
+["mgr"]="core"
+["monitoring"]="monitoring"
+["orch"]="orchestrator"
+["osd"]="core"
+["perf"]="performance"
+["prometheus"]="monitoring"
+["pybind"]="pybind"
+["py3"]="python3"
+["python3"]="python3"
+["qa"]="tests"
+["rbd"]="rbd"
+["rgw"]="rgw"
+["rpm"]="build/ops"
+["tests"]="tests"
+["tool"]="tools"
+)
+
+declare -A flagged_pr_hash=()
+
+function abort_due_to_setup_problem {
+ error "problem detected in your setup"
+ info "Run \"${this_script} --setup\" to fix"
+ false
+}
+
+function assert_fail {
+ local message="$1"
+ error "(internal error) $message"
+ info "This could be reported as a bug!"
+ false
+}
+
+function backport_pr_needs_label {
+ local check_label="$1"
+ local label
+ local needs_label="yes"
+ while read -r label ; do
+ if [ "$label" = "$check_label" ] ; then
+ needs_label=""
+ fi
+ done <<< "$backport_pr_labels"
+ echo "$needs_label"
+}
+
+function backport_pr_needs_milestone {
+ if [ "$existing_pr_milestone_number" ] ; then
+ echo ""
+ else
+ echo "yes"
+ fi
+}
+
+function bail_out_github_api {
+ local api_said="$1"
+ local hint="$2"
+ info "GitHub API said:"
+ log bare "$api_said"
+ if [ "$hint" ] ; then
+ info "(hint) $hint"
+ fi
+ abort_due_to_setup_problem
+}
+
+function blindly_set_pr_metadata {
+ local pr_number="$1"
+ local json_blob="$2"
+ curl -u ${github_user}:${github_token} --silent --data-binary "$json_blob" "https://api.github.com/repos/ceph/ceph/issues/${pr_number}" >/dev/null 2>&1 || true
+}
+
+function check_milestones {
+ local milestones_to_check
+ milestones_to_check="$(echo "$1" | tr '\n' ' ' | xargs)"
+ info "Active milestones: $milestones_to_check"
+ for m in $milestones_to_check ; do
+ info "Examining all PRs targeting base branch \"$m\""
+ vet_prs_for_milestone "$m"
+ done
+ dump_flagged_prs
+}
+
+function check_tracker_status {
+ local -a ok_statuses=("new" "need more info")
+ local ts="$1"
+ local error_msg
+ local tslc="${ts,,}"
+ local tslc_is_ok=
+ for oks in "${ok_statuses[@]}"; do
+ if [ "$tslc" = "$oks" ] ; then
+ debug "Tracker status $ts is OK for backport to proceed"
+ tslc_is_ok="yes"
+ break
+ fi
+ done
+ if [ "$tslc_is_ok" ] ; then
+ true
+ else
+ if [ "$tslc" = "in progress" ] ; then
+ error_msg="backport $redmine_url is already in progress"
+ else
+ error_msg="backport $redmine_url is closed (status: ${ts})"
+ fi
+ if [ "$FORCE" ] || [ "$EXISTING_PR" ] ; then
+ warning "$error_msg"
+ else
+ error "$error_msg"
+ fi
+ fi
+ echo "$tslc_is_ok"
+}
+
+function cherry_pick_phase {
+ local base_branch
+ local default_val
+ local i
+ local merged
+ local number_of_commits
+ local offset
+ local sha1_to_cherry_pick
+ local singular_or_plural_commit
+ local yes_or_no_answer
+ populate_original_issue
+ if [ -z "$original_issue" ] ; then
+ error "Could not find original issue"
+ info "Does ${redmine_url} have a \"Copied from\" relation?"
+ false
+ fi
+ info "Parent issue: ${original_issue_url}"
+
+ populate_original_pr
+ if [ -z "$original_pr" ]; then
+ error "Could not find original PR"
+ info "Is the \"Pull request ID\" field of ${original_issue_url} populated?"
+ false
+ fi
+ info "Parent issue ostensibly fixed by: ${original_pr_url}"
+
+ verbose "Examining ${original_pr_url}"
+ remote_api_output=$(curl -u ${github_user}:${github_token} --silent "https://api.github.com/repos/ceph/ceph/pulls/${original_pr}")
+ base_branch=$(echo "${remote_api_output}" | jq -r '.base.label')
+ if [ "$base_branch" = "ceph:master" ] ; then
+ true
+ else
+ if [ "$FORCE" ] ; then
+ warning "base_branch ->$base_branch<- is something other than \"ceph:master\""
+ info "--force was given, so continuing anyway"
+ else
+ error "${original_pr_url} is targeting ${base_branch}: cowardly refusing to perform automated cherry-pick"
+ info "Out of an abundance of caution, the script only automates cherry-picking of commits from PRs targeting \"ceph:master\"."
+ info "You can still use the script to stage the backport, though. Just prepare the local branch \"${local_branch}\" manually and re-run the script."
+ false
+ fi
+ fi
+ merged=$(echo "${remote_api_output}" | jq -r '.merged')
+ if [ "$merged" = "true" ] ; then
+ true
+ else
+ error "${original_pr_url} is not merged yet"
+ info "Cowardly refusing to perform automated cherry-pick"
+ false
+ fi
+ number_of_commits=$(echo "${remote_api_output}" | jq '.commits')
+ if [ "$number_of_commits" -eq "$number_of_commits" ] 2>/dev/null ; then
+ # \$number_of_commits is set, and is an integer
+ if [ "$number_of_commits" -eq "1" ] ; then
+ singular_or_plural_commit="commit"
+ else
+ singular_or_plural_commit="commits"
+ fi
+ else
+ error "Could not determine the number of commits in ${original_pr_url}"
+ bail_out_github_api "$remote_api_output"
+ fi
+ info "Found $number_of_commits $singular_or_plural_commit in $original_pr_url"
+
+ set -x
+ git fetch "$upstream_remote"
+
+ if git show-ref --verify --quiet "refs/heads/$local_branch" ; then
+ if [ "$FORCE" ] ; then
+ if [ "$non_interactive" ] ; then
+ git checkout "$local_branch"
+ git reset --hard "${upstream_remote}/${milestone}"
+ else
+ echo
+ echo "A local branch $local_branch already exists and the --force option was given."
+ echo "If you continue, any local changes in $local_branch will be lost!"
+ echo
+ default_val="y"
+ echo -n "Do you really want to overwrite ${local_branch}? (default: ${default_val}) "
+ yes_or_no_answer="$(get_user_input "$default_val")"
+ [ "$yes_or_no_answer" ] && yes_or_no_answer="${yes_or_no_answer:0:1}"
+ if [ "$yes_or_no_answer" = "y" ] ; then
+ git checkout "$local_branch"
+ git reset --hard "${upstream_remote}/${milestone}"
+ else
+ info "OK, bailing out!"
+ false
+ fi
+ fi
+ else
+ set +x
+ maybe_restore_set_x
+ error "Cannot initialize $local_branch - local branch already exists"
+ false
+ fi
+ else
+ git checkout "${upstream_remote}/${milestone}" -b "$local_branch"
+ fi
+
+ git fetch "$upstream_remote" "pull/$original_pr/head:pr-$original_pr"
+
+ set +x
+ maybe_restore_set_x
+ info "Attempting to cherry pick $number_of_commits commits from ${original_pr_url} into local branch $local_branch"
+ offset="$((number_of_commits - 1))" || true
+ for ((i=offset; i>=0; i--)) ; do
+ info "Running \"git cherry-pick -x\" on $(git log --oneline --max-count=1 --no-decorate "pr-${original_pr}~${i}")"
+ sha1_to_cherry_pick=$(git rev-parse --verify "pr-${original_pr}~${i}")
+ set -x
+ if git cherry-pick -x "$sha1_to_cherry_pick" ; then
+ set +x
+ maybe_restore_set_x
+ else
+ set +x
+ maybe_restore_set_x
+ [ "$VERBOSE" ] && git status
+ error "Cherry pick failed"
+ info "Next, manually fix conflicts and complete the current cherry-pick"
+ if [ "$i" -gt "0" ] >/dev/null 2>&1 ; then
+ info "Then, cherry-pick the remaining commits from ${original_pr_url}, i.e.:"
+ for ((j=i-1; j>=0; j--)) ; do
+ info "-> missing commit: $(git log --oneline --max-count=1 --no-decorate "pr-${original_pr}~${j}")"
+ done
+ info "Finally, re-run the script"
+ else
+ info "Then re-run the script"
+ fi
+ false
+ fi
+ done
+ info "Cherry picking completed without conflicts"
+}
+
+function clear_line {
+ log overwrite " \r"
+}
+
+function clip_pr_body {
+ local pr_body="$*"
+ local clipped=""
+ local last_line_was_blank=""
+ local line=""
+ local pr_json_tempfile=$(mktemp)
+ echo "$pr_body" | sed -n '/<!--.*/q;p' > "$pr_json_tempfile"
+ while IFS= read -r line; do
+ if [ "$(trim_whitespace "$line")" ] ; then
+ last_line_was_blank=""
+ clipped="${clipped}${line}\n"
+ else
+ if [ "$last_line_was_blank" ] ; then
+ true
+ else
+ clipped="${clipped}\n"
+ fi
+ fi
+ done < "$pr_json_tempfile"
+ rm "$pr_json_tempfile"
+ echo "$clipped"
+}
+
+function debug {
+ log debug "$@"
+}
+
+function display_version_message_and_exit {
+ echo "$this_script: Ceph backporting script, version $SCRIPT_VERSION"
+ exit 0
+}
+
+function dump_flagged_prs {
+ local url=
+ clear_line
+ if [ "${#flagged_pr_hash[@]}" -eq "0" ] ; then
+ info "All backport PRs appear to have milestone set correctly"
+ else
+ warning "Some backport PRs had problematic milestone settings"
+ log bare "==========="
+ log bare "Flagged PRs"
+ log bare "-----------"
+ for url in "${!flagged_pr_hash[@]}" ; do
+ log bare "$url - ${flagged_pr_hash[$url]}"
+ done
+ log bare "==========="
+ fi
+}
+
+function eol {
+ local mtt="$1"
+ error "$mtt is EOL"
+ false
+}
+
+function error {
+ log error "$@"
+}
+
+function existing_pr_routine {
+ local base_branch
+ local clipped_pr_body
+ local new_pr_body
+ local new_pr_title
+ local pr_body
+ local pr_json_tempfile
+ local remote_api_output
+ local update_pr_body
+ remote_api_output="$(curl -u ${github_user}:${github_token} --silent "https://api.github.com/repos/ceph/ceph/pulls/${backport_pr_number}")"
+ backport_pr_title="$(echo "$remote_api_output" | jq -r '.title')"
+ if [ "$backport_pr_title" = "null" ] ; then
+ error "could not get PR title of existing PR ${backport_pr_number}"
+ bail_out_github_api "$remote_api_output"
+ fi
+ existing_pr_milestone_number="$(echo "$remote_api_output" | jq -r '.milestone.number')"
+ if [ "$existing_pr_milestone_number" = "null" ] ; then
+ existing_pr_milestone_number=""
+ fi
+ backport_pr_labels="$(echo "$remote_api_output" | jq -r '.labels[].name')"
+ pr_body="$(echo "$remote_api_output" | jq -r '.body')"
+ if [ "$pr_body" = "null" ] ; then
+ error "could not get PR body of existing PR ${backport_pr_number}"
+ bail_out_github_api "$remote_api_output"
+ fi
+ base_branch=$(echo "${remote_api_output}" | jq -r '.base.label')
+ base_branch="${base_branch#ceph:}"
+ if [ -z "$(is_active_milestone "$base_branch")" ] ; then
+ error "existing PR $backport_pr_url is targeting $base_branch which is not an active milestone"
+ info "Cowardly refusing to work on a backport to $base_branch"
+ false
+ fi
+ clipped_pr_body="$(clip_pr_body "$pr_body")"
+ verbose_en "Clipped body of existing PR ${backport_pr_number}:\n${clipped_pr_body}"
+ if [[ "$backport_pr_title" =~ ^${milestone}: ]] ; then
+ verbose "Existing backport PR ${backport_pr_number} title has ${milestone} prepended"
+ else
+ warning "Existing backport PR ${backport_pr_number} title does NOT have ${milestone} prepended"
+ new_pr_title="${milestone}: $backport_pr_title"
+ if [[ "$new_pr_title" =~ \" ]] ; then
+ new_pr_title="${new_pr_title//\"/\\\"}"
+ fi
+ verbose "New PR title: ${new_pr_title}"
+ fi
+ redmine_url_without_scheme="${redmine_url//http?:\/\//}"
+ verbose "Redmine URL without scheme: $redmine_url_without_scheme"
+ if [[ "$clipped_pr_body" =~ $redmine_url_without_scheme ]] ; then
+ info "Existing backport PR ${backport_pr_number} already mentions $redmine_url"
+ if [ "$FORCE" ] ; then
+ warning "--force was given, so updating the PR body anyway"
+ update_pr_body="yes"
+ fi
+ else
+ warning "Existing backport PR ${backport_pr_number} does NOT mention $redmine_url - adding it"
+ update_pr_body="yes"
+ fi
+ if [ "$update_pr_body" ] ; then
+ new_pr_body="backport tracker: ${redmine_url}"
+ if [ "${original_pr_url}" ] ; then
+ new_pr_body="${new_pr_body}
+possibly a backport of ${original_pr_url}"
+ fi
+ if [ "${original_issue_url}" ] ; then
+ new_pr_body="${new_pr_body}
+parent tracker: ${original_issue_url}"
+ fi
+ new_pr_body="${new_pr_body}
+
+---
+
+original PR body:
+
+$clipped_pr_body
+
+---
+
+updated using ceph-backport.sh version ${SCRIPT_VERSION}"
+ fi
+ maybe_update_pr_title_body "${new_pr_title}" "${new_pr_body}"
+}
+
+function failed_mandatory_var_check {
+ local varname="$1"
+ local error="$2"
+ verbose "$varname $error"
+ setup_ok=""
+}
+
+function flag_pr {
+ local pr_num="$1"
+ local pr_url="$2"
+ local flag_reason="$3"
+ warning "flagging PR#${pr_num} because $flag_reason"
+ flagged_pr_hash["${pr_url}"]="$flag_reason"
+}
+
+function from_file {
+ local what="$1"
+ xargs 2>/dev/null < "$HOME/.${what}" || true
+}
+
+function get_user_input {
+ local default_val="$1"
+ local user_input=
+ read -r user_input
+ if [ "$user_input" ] ; then
+ echo "$user_input"
+ else
+ echo "$default_val"
+ fi
+}
+
+# takes a string and a substring - returns position of substring within string,
+# or -1 if not found
+# NOTE: position of first character in string is 0
+function grep_for_substr {
+ local str="$1"
+ local look_for_in_str="$2"
+ str="${str,,}"
+ munged="${str%%${look_for_in_str}*}"
+ if [ "$munged" = "$str" ] ; then
+ echo "-1"
+ else
+ echo "${#munged}"
+ fi
+}
+
+# takes PR title, attempts to guess component
+function guess_component {
+ local comp=
+ local pos="0"
+ local pr_title="$1"
+ local winning_comp=
+ local winning_comp_pos="9999"
+ for comp in "${!comp_hash[@]}" ; do
+ pos=$(grep_for_substr "$pr_title" "$comp")
+ # echo "$comp: $pos"
+ [ "$pos" = "-1" ] && continue
+ if [ "$pos" -lt "$winning_comp_pos" ] ; then
+ winning_comp_pos="$pos"
+ winning_comp="$comp"
+ fi
+ done
+ [ "$winning_comp" ] && echo "${comp_hash["$winning_comp"]}" || echo ""
+}
+
+function info {
+ log info "$@"
+}
+
+function init_endpoints {
+ verbose "Initializing remote API endpoints"
+ redmine_endpoint="${redmine_endpoint:-"https://tracker.ceph.com"}"
+ github_endpoint="${github_endpoint:-"https://github.com/ceph/ceph"}"
+}
+
+function init_fork_remote {
+ [ "$github_user" ] || assert_fail "github_user not set"
+ [ "$EXPLICIT_FORK" ] && info "Using explicit fork ->$EXPLICIT_FORK<- instead of personal fork."
+ fork_remote="${fork_remote:-$(maybe_deduce_remote fork)}"
+}
+
+function init_github_token {
+ github_token="$(from_file github_token)"
+ if [ "$github_token" ] ; then
+ true
+ else
+ warning "$github_token_file not populated: initiating interactive setup routine"
+ INTERACTIVE_SETUP_ROUTINE="yes"
+ fi
+}
+
+function init_redmine_key {
+ redmine_key="$(from_file redmine_key)"
+ if [ "$redmine_key" ] ; then
+ true
+ else
+ warning "$redmine_key_file not populated: initiating interactive setup routine"
+ INTERACTIVE_SETUP_ROUTINE="yes"
+ fi
+}
+
+function init_upstream_remote {
+ upstream_remote="${upstream_remote:-$(maybe_deduce_remote upstream)}"
+}
+
+function interactive_setup_routine {
+ local default_val
+ local original_github_token
+ local original_redmine_key
+ local total_steps
+ local yes_or_no_answer
+ original_github_token="$github_token"
+ original_redmine_key="$redmine_key"
+ total_steps="4"
+ if [ -e "$deprecated_backport_common" ] ; then
+ github_token=""
+ redmine_key=""
+ # shellcheck disable=SC1090
+ source "$deprecated_backport_common" 2>/dev/null || true
+ total_steps="$((total_steps+1))"
+ fi
+ echo
+ echo "Welcome to the ${this_script} interactive setup routine!"
+ echo
+ echo "---------------------------------------------------------------------"
+ echo "Setup step 1 of $total_steps - GitHub token"
+ echo "---------------------------------------------------------------------"
+ echo "For information on how to generate a GitHub personal access token"
+ echo "to use with this script, go to https://github.com/settings/tokens"
+ echo "then click on \"Generate new token\" and make sure the token has"
+ echo "\"Full control of private repositories\" scope."
+ echo
+ echo "For more details, see:"
+ echo "https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line"
+ echo
+ echo -n "What is your GitHub token? "
+ default_val="$github_token"
+ [ "$github_token" ] && echo "(default: ${default_val})"
+ github_token="$(get_user_input "$default_val")"
+ if [ "$github_token" ] ; then
+ true
+ else
+ error "You must provide a valid GitHub personal access token"
+ abort_due_to_setup_problem
+ fi
+ [ "$github_token" ] || assert_fail "github_token not set, even after completing Step 1 of interactive setup"
+ echo
+ echo "---------------------------------------------------------------------"
+ echo "Setup step 2 of $total_steps - GitHub user"
+ echo "---------------------------------------------------------------------"
+ echo "The script will now attempt to determine your GitHub user (login)"
+ echo "from the GitHub token provided in the previous step. If this is"
+ echo "successful, there is a good chance that your GitHub token is OK."
+ echo
+ echo "Communicating with the GitHub API..."
+ set_github_user_from_github_token
+ [ "$github_user" ] || abort_due_to_setup_problem
+ echo
+ echo -n "Is the GitHub username (login) \"$github_user\" correct? "
+ default_val="y"
+ [ "$github_token" ] && echo "(default: ${default_val})"
+ yes_or_no_answer="$(get_user_input "$default_val")"
+ [ "$yes_or_no_answer" ] && yes_or_no_answer="${yes_or_no_answer:0:1}"
+ if [ "$yes_or_no_answer" = "y" ] ; then
+ if [ "$github_token" = "$original_github_token" ] ; then
+ true
+ else
+ debug "GitHub personal access token changed"
+ echo "$github_token" > "$github_token_file"
+ chmod 0600 "$github_token_file"
+ info "Wrote GitHub personal access token to $github_token_file"
+ fi
+ else
+ error "GitHub user does not look right"
+ abort_due_to_setup_problem
+ fi
+ [ "$github_token" ] || assert_fail "github_token not set, even after completing Steps 1 and 2 of interactive setup"
+ [ "$github_user" ] || assert_fail "github_user not set, even after completing Steps 1 and 2 of interactive setup"
+ echo
+ echo "---------------------------------------------------------------------"
+ echo "Setup step 3 of $total_steps - remote repos"
+ echo "---------------------------------------------------------------------"
+ echo "Searching \"git remote -v\" for remote repos"
+ echo
+ init_upstream_remote
+ init_fork_remote
+ vet_remotes
+ echo "Upstream remote is \"$upstream_remote\""
+ echo "Fork remote is \"$fork_remote\""
+ [ "$setup_ok" ] || abort_due_to_setup_problem
+ [ "$github_token" ] || assert_fail "github_token not set, even after completing Steps 1-3 of interactive setup"
+ [ "$github_user" ] || assert_fail "github_user not set, even after completing Steps 1-3 of interactive setup"
+ [ "$upstream_remote" ] || assert_fail "upstream_remote not set, even after completing Steps 1-3 of interactive setup"
+ [ "$fork_remote" ] || assert_fail "fork_remote not set, even after completing Steps 1-3 of interactive setup"
+ echo
+ echo "---------------------------------------------------------------------"
+ echo "Setup step 4 of $total_steps - Redmine key"
+ echo "---------------------------------------------------------------------"
+ echo "To generate a Redmine API access key, go to https://tracker.ceph.com"
+ echo "After signing in, click: \"My account\""
+ echo "Now, find \"API access key\"."
+ echo "Once you know the API access key, enter it below."
+ echo
+ echo -n "What is your Redmine key? "
+ default_val="$redmine_key"
+ [ "$redmine_key" ] && echo "(default: ${default_val})"
+ redmine_key="$(get_user_input "$default_val")"
+ if [ "$redmine_key" ] ; then
+ set_redmine_user_from_redmine_key
+ if [ "$setup_ok" ] ; then
+ true
+ else
+ info "You must provide a valid Redmine API access key"
+ abort_due_to_setup_problem
+ fi
+ if [ "$redmine_key" = "$original_redmine_key" ] ; then
+ true
+ else
+ debug "Redmine API access key changed"
+ echo "$redmine_key" > "$redmine_key_file"
+ chmod 0600 "$redmine_key_file"
+ info "Wrote Redmine API access key to $redmine_key_file"
+ fi
+ else
+ error "You must provide a valid Redmine API access key"
+ abort_due_to_setup_problem
+ fi
+ [ "$github_token" ] || assert_fail "github_token not set, even after completing Steps 1-4 of interactive setup"
+ [ "$github_user" ] || assert_fail "github_user not set, even after completing Steps 1-4 of interactive setup"
+ [ "$upstream_remote" ] || assert_fail "upstream_remote not set, even after completing Steps 1-4 of interactive setup"
+ [ "$fork_remote" ] || assert_fail "fork_remote not set, even after completing Steps 1-4 of interactive setup"
+ [ "$redmine_key" ] || assert_fail "redmine_key not set, even after completing Steps 1-4 of interactive setup"
+ [ "$redmine_user_id" ] || assert_fail "redmine_user_id not set, even after completing Steps 1-4 of interactive setup"
+ [ "$redmine_login" ] || assert_fail "redmine_login not set, even after completing Steps 1-4 of interactive setup"
+ if [ "$total_steps" -gt "4" ] ; then
+ echo
+ echo "---------------------------------------------------------------------"
+ echo "Step 5 of $total_steps - delete deprecated $deprecated_backport_common file"
+ echo "---------------------------------------------------------------------"
+ fi
+ maybe_delete_deprecated_backport_common
+ vet_setup --interactive
+}
+
+function is_active_milestone {
+ local is_active=
+ local milestone_under_test="$1"
+ for m in $active_milestones ; do
+ if [ "$milestone_under_test" = "$m" ] ; then
+ verbose "Milestone $m is active"
+ is_active="yes"
+ break
+ fi
+ done
+ echo "$is_active"
+}
+
+function log {
+ local level="$1"
+ local trailing_newline="yes"
+ local in_hex=""
+ shift
+ local msg="$*"
+ prefix="${this_script}: "
+ verbose_only=
+ case $level in
+ bare)
+ prefix=
+ ;;
+ debug)
+ prefix="${prefix}DEBUG: "
+ verbose_only="yes"
+ ;;
+ err*)
+ prefix="${prefix}ERROR: "
+ ;;
+ hex)
+ in_hex="yes"
+ ;;
+ info)
+ :
+ ;;
+ overwrite)
+ trailing_newline=
+ prefix=
+ ;;
+ verbose)
+ verbose_only="yes"
+ ;;
+ verbose_en)
+ verbose_only="yes"
+ trailing_newline=
+ ;;
+ warn|warning)
+ prefix="${prefix}WARNING: "
+ ;;
+ esac
+ if [ "$in_hex" ] ; then
+ print_in_hex "$msg"
+ elif [ "$verbose_only" ] && [ -z "$VERBOSE" ] ; then
+ true
+ else
+ msg="${prefix}${msg}"
+ if [ "$trailing_newline" ] ; then
+ echo "${msg}" >&2
+ else
+ echo -en "${msg}" >&2
+ fi
+ fi
+}
+
+function maybe_deduce_remote {
+ local remote_type="$1"
+ local remote=""
+ local url_component=""
+ if [ "$remote_type" = "upstream" ] ; then
+ url_component="ceph"
+ elif [ "$remote_type" = "fork" ] ; then
+ if [ "$EXPLICIT_FORK" ] ; then
+ url_component="$EXPLICIT_FORK"
+ else
+ url_component="$github_user"
+ fi
+ else
+ assert_fail "bad remote_type ->$remote_type<- in maybe_deduce_remote"
+ fi
+ remote=$(git remote -v | grep --extended-regexp --ignore-case '(://|@)github.com(/|:)'${url_component}'/ceph(\s|\.|\/)' | head -n1 | cut -f 1)
+ echo "$remote"
+}
+
+function maybe_delete_deprecated_backport_common {
+ local default_val
+ local user_inp
+ if [ -e "$deprecated_backport_common" ] ; then
+ echo "You still have a $deprecated_backport_common file,"
+ echo "which was used to store configuration parameters in version"
+ echo "15.0.0.6270 and earlier versions of ${this_script}."
+ echo
+ echo "Since $deprecated_backport_common has been deprecated in favor"
+ echo "of the interactive setup routine, which has been completed"
+ echo "successfully, the file should be deleted now."
+ echo
+ echo -n "Delete it now? (default: y) "
+ default_val="y"
+ user_inp="$(get_user_input "$default_val")"
+ user_inp="$(echo "$user_inp" | tr '[:upper:]' '[:lower:]' | xargs)"
+ if [ "$user_inp" ] ; then
+ user_inp="${user_inp:0:1}"
+ if [ "$user_inp" = "y" ] ; then
+ set -x
+ rm -f "$deprecated_backport_common"
+ set +x
+ maybe_restore_set_x
+ fi
+ fi
+ if [ -e "$deprecated_backport_common" ] ; then
+ error "$deprecated_backport_common still exists. Bailing out!"
+ false
+ fi
+ fi
+}
+
+function maybe_restore_set_x {
+ if [ "$DEBUG" ] ; then
+ set -x
+ fi
+}
+
+function maybe_update_pr_milestone_labels {
+ local component
+ local data_binary
+ local data_binary
+ local label
+ local needs_milestone
+ if [ "$EXPLICIT_COMPONENT" ] ; then
+ debug "Component given on command line: using it"
+ component="$EXPLICIT_COMPONENT"
+ else
+ debug "Attempting to guess component"
+ component=$(guess_component "$backport_pr_title")
+ fi
+ data_binary="{"
+ needs_milestone="$(backport_pr_needs_milestone)"
+ if [ "$needs_milestone" ] ; then
+ debug "Attempting to set ${milestone} milestone in ${backport_pr_url}"
+ data_binary="${data_binary}\"milestone\":${milestone_number}"
+ else
+ info "Backport PR ${backport_pr_url} already has ${milestone} milestone"
+ fi
+ if [ "$(backport_pr_needs_label "$component")" ] ; then
+ debug "Attempting to add ${component} label to ${backport_pr_url}"
+ if [ "$needs_milestone" ] ; then
+ data_binary="${data_binary},"
+ fi
+ data_binary="${data_binary}\"labels\":[\"${component}\""
+ while read -r label ; do
+ if [ "$label" ] ; then
+ data_binary="${data_binary},\"${label}\""
+ fi
+ done <<< "$backport_pr_labels"
+ data_binary="${data_binary}]}"
+ else
+ info "Backport PR ${backport_pr_url} already has label ${component}"
+ data_binary="${data_binary}}"
+ fi
+ if [ "$data_binary" = "{}" ] ; then
+ true
+ else
+ blindly_set_pr_metadata "$backport_pr_number" "$data_binary"
+ fi
+}
+
+function maybe_update_pr_title_body {
+ local new_title="$1"
+ local new_body="$2"
+ local data_binary
+ if [ "$new_title" ] && [ "$new_body" ] ; then
+ data_binary="{\"title\":\"${new_title}\", \"body\":\"$(munge_body "${new_body}")\"}"
+ elif [ "$new_title" ] ; then
+ data_binary="{\"title\":\"${new_title}\"}"
+ backport_pr_title="${new_title}"
+ elif [ "$new_body" ] ; then
+ data_binary="{\"body\":\"$(munge_body "${new_body}")\"}"
+ #log hex "${data_binary}"
+ #echo -n "${data_binary}"
+ fi
+ if [ "$data_binary" ] ; then
+ blindly_set_pr_metadata "${backport_pr_number}" "$data_binary"
+ fi
+}
+
+function milestone_number_from_remote_api {
+ local mtt="$1" # milestone to try
+ local mn="" # milestone number
+ local milestones
+ remote_api_output=$(curl -u ${github_user}:${github_token} --silent -X GET "https://api.github.com/repos/ceph/ceph/milestones")
+ mn=$(echo "$remote_api_output" | jq --arg milestone "$mtt" '.[] | select(.title==$milestone) | .number')
+ if [ "$mn" -gt "0" ] >/dev/null 2>&1 ; then
+ echo "$mn"
+ else
+ error "Could not determine milestone number of ->$milestone<-"
+ verbose_en "GitHub API said:\n${remote_api_output}\n"
+ remote_api_output=$(curl -u ${github_user}:${github_token} --silent -X GET "https://api.github.com/repos/ceph/ceph/milestones")
+ milestones=$(echo "$remote_api_output" | jq '.[].title')
+ info "Valid values are ${milestones}"
+ info "(This probably means the Release field of ${redmine_url} is populated with"
+ info "an unexpected value - i.e. it does not match any of the GitHub milestones.)"
+ false
+ fi
+}
+
+function munge_body {
+ echo "$new_body" | tr '\r' '\n' | sed 's/$/\\n/' | tr -d '\n'
+}
+
+function number_to_url {
+ local number_type="$1"
+ local number="$2"
+ if [ "$number_type" = "github" ] ; then
+ echo "${github_endpoint}/pull/${number}"
+ elif [ "$number_type" = "redmine" ] ; then
+ echo "${redmine_endpoint}/issues/${number}"
+ else
+ assert_fail "internal error in number_to_url: bad type ->$number_type<-"
+ fi
+}
+
+function populate_original_issue {
+ if [ -z "$original_issue" ] ; then
+ original_issue=$(curl --silent "${redmine_url}.json?include=relations" |
+ jq '.issue.relations[] | select(.relation_type | contains("copied_to")) | .issue_id')
+ original_issue_url="$(number_to_url "redmine" "${original_issue}")"
+ fi
+}
+
+function populate_original_pr {
+ if [ "$original_issue" ] ; then
+ if [ -z "$original_pr" ] ; then
+ original_pr=$(curl --silent "${original_issue_url}.json" |
+ jq -r '.issue.custom_fields[] | select(.id | contains(21)) | .value')
+ original_pr_url="$(number_to_url "github" "${original_pr}")"
+ fi
+ fi
+}
+
+function print_in_hex {
+ local str="$1"
+ local c
+
+ for (( i=0; i < ${#str}; i++ ))
+ do
+ c=${str:$i:1}
+ if [[ $c == ' ' ]]
+ then
+ printf "[%s] 0x%X\n" " " \'\ \' >&2
+ else
+ printf "[%s] 0x%X\n" "$c" \'"$c"\' >&2
+ fi
+ done
+}
+
+function set_github_user_from_github_token {
+ local quiet="$1"
+ local api_error
+ local curl_opts
+ setup_ok=""
+ [ "$github_token" ] || assert_fail "set_github_user_from_github_token: git_token not set"
+ curl_opts="--silent -u :${github_token} https://api.github.com/user"
+ [ "$quiet" ] || set -x
+ remote_api_output="$(curl $curl_opts)"
+ set +x
+ github_user=$(echo "${remote_api_output}" | jq -r .login 2>/dev/null | grep -v null || true)
+ api_error=$(echo "${remote_api_output}" | jq -r .message 2>/dev/null | grep -v null || true)
+ if [ "$api_error" ] ; then
+ info "GitHub API said: ->$api_error<-"
+ info "If you can't figure out what's wrong by examining the curl command and its output, above,"
+ info "please also study https://developer.github.com/v3/users/#get-the-authenticated-user"
+ github_user=""
+ else
+ [ "$github_user" ] || assert_fail "set_github_user_from_github_token: failed to set github_user"
+ info "my GitHub username is $github_user"
+ setup_ok="yes"
+ fi
+}
+
+function set_redmine_user_from_redmine_key {
+ [ "$redmine_key" ] || assert_fail "set_redmine_user_from_redmine_key was called, but redmine_key not set"
+ local api_key_from_api
+ remote_api_output="$(curl --silent "https://tracker.ceph.com/users/current.json?key=$redmine_key")"
+ redmine_login="$(echo "$remote_api_output" | jq -r '.user.login')"
+ redmine_user_id="$(echo "$remote_api_output" | jq -r '.user.id')"
+ api_key_from_api="$(echo "$remote_api_output" | jq -r '.user.api_key')"
+ if [ "$redmine_login" ] && [ "$redmine_user_id" ] && [ "$api_key_from_api" = "$redmine_key" ] ; then
+ [ "$redmine_user_id" ] || assert_fail "set_redmine_user_from_redmine_key: failed to set redmine_user_id"
+ [ "$redmine_login" ] || assert_fail "set_redmine_user_from_redmine_key: failed to set redmine_login"
+ info "my Redmine username is $redmine_login (ID $redmine_user_id)"
+ setup_ok="yes"
+ else
+ error "Redmine API access key $redmine_key is invalid"
+ redmine_login=""
+ redmine_user_id=""
+ setup_ok=""
+ fi
+}
+
+function tracker_component_is_in_desired_state {
+ local comp="$1"
+ local val_is="$2"
+ local val_should_be="$3"
+ local in_desired_state
+ if [ "$val_is" = "$val_should_be" ] ; then
+ debug "Tracker $comp is in the desired state"
+ in_desired_state="yes"
+ fi
+ echo "$in_desired_state"
+}
+
+function tracker_component_was_updated {
+ local comp="$1"
+ local val_old="$2"
+ local val_new="$3"
+ local was_updated
+ if [ "$val_old" = "$val_new" ] ; then
+ true
+ else
+ debug "Tracker $comp was updated!"
+ was_updated="yes"
+ fi
+ echo "$was_updated"
+}
+
+function trim_whitespace {
+ local var="$*"
+ # remove leading whitespace characters
+ var="${var#"${var%%[![:space:]]*}"}"
+ # remove trailing whitespace characters
+ var="${var%"${var##*[![:space:]]}"}"
+ echo -n "$var"
+}
+
+function troubleshooting_advice {
+ cat <<EOM
+Troubleshooting notes
+---------------------
+
+If the script inexplicably fails with:
+
+ error: a cherry-pick or revert is already in progress
+ hint: try "git cherry-pick (--continue | --quit | --abort)"
+ fatal: cherry-pick failed
+
+This is because HEAD is not where git expects it to be:
+
+ $ git cherry-pick --abort
+ warning: You seem to have moved HEAD. Not rewinding, check your HEAD!
+
+This can be fixed by issuing the command:
+
+ $ git cherry-pick --quit
+
+EOM
+}
+
+# to update known milestones, consult:
+# curl --verbose -X GET https://api.github.com/repos/ceph/ceph/milestones
+function try_known_milestones {
+ local mtt=$1 # milestone to try
+ local mn="" # milestone number
+ case $mtt in
+ cuttlefish) eol "$mtt" ;;
+ dumpling) eol "$mtt" ;;
+ emperor) eol "$mtt" ;;
+ firefly) eol "$mtt" ;;
+ giant) eol "$mtt" ;;
+ hammer) eol "$mtt" ;;
+ infernalis) eol "$mtt" ;;
+ jewel) mn="8" ;;
+ kraken) eol "$mtt" ;;
+ luminous) mn="10" ;;
+ mimic) mn="11" ;;
+ nautilus) mn="12" ;;
+ octopus) mn="13" ;;
+ pacific) mn="TBD" ;;
+ esac
+ echo "$mn"
+}
+
+function update_version_number_and_exit {
+ set -x
+ local raw_version
+ local munge_first_hyphen
+ # munge_first_hyphen will look like this: 15.0.0.5774-g4c2f2eda969
+ local script_version_number
+ raw_version="$(git describe --long --match 'v*' | sed 's/^v//')" # example: "15.0.0-5774-g4c2f2eda969"
+ munge_first_hyphen="${raw_version/-/.}" # example: "15.0.0.5774-g4c2f2eda969"
+ script_version_number="${munge_first_hyphen%-*}" # example: "15.0.0.5774"
+ sed -i -e "s/^SCRIPT_VERSION=.*/SCRIPT_VERSION=\"${script_version_number}\"/" "$full_path"
+ exit 0
+}
+
+function usage {
+ cat <<EOM >&2
+Setup:
+
+ ${this_script} --setup
+
+Documentation:
+
+ ${this_script} --help
+ ${this_script} --usage | less
+ ${this_script} --troubleshooting | less
+
+Usage:
+ ${this_script} BACKPORT_TRACKER_ISSUE_NUMBER
+
+Options (not needed in normal operation):
+ --cherry-pick-only (stop after cherry-pick phase)
+ --component/-c COMPONENT
+ (explicitly set the component label; if omitted, the
+ script will try to guess the component)
+ --debug (turns on "set -x")
+ --existing-pr BACKPORT_PR_ID
+ (use this when the backport PR is already open)
+ --force (exercise caution!)
+ --fork EXPLICIT_FORK (use EXPLICIT_FORK instead of personal GitHub fork)
+ --milestones (vet all backport PRs for correct milestone setting)
+ --setup/-s (run the interactive setup routine - NOTE: this can
+ be done any number of times)
+ --setup-report (check the setup and print a report)
+ --update-version (this option exists as a convenience for the script
+ maintainer only: not intended for day-to-day usage)
+ --verbose/-v (produce more output than normal)
+ --version (display version number and exit)
+
+Example:
+ ${this_script} 31459
+ (if cherry-pick conflicts are present, finish cherry-picking phase manually
+ and then run the script again with the same argument)
+
+CAVEAT: The script must be run from inside a local git clone.
+EOM
+}
+
+function usage_advice {
+ cat <<EOM
+Usage advice
+------------
+
+Once you have completed --setup, you can run the script with the ID of
+a Backport tracker issue. For example, to stage the backport
+https://tracker.ceph.com/issues/41502, run:
+
+ ${this_script} 41502
+
+Provided the commits in the corresponding master PR cherry-pick cleanly, the
+script will automatically perform all steps required to stage the backport:
+
+Cherry-pick phase:
+
+1. fetching the latest commits from the upstream remote
+2. creating a wip branch for the backport
+3. figuring out which upstream PR contains the commits to cherry-pick
+4. cherry-picking the commits
+
+PR phase:
+
+5. pushing the wip branch to your fork
+6. opening the backport PR with compliant title and description describing
+ the backport
+7. (optionally) setting the milestone and label in the PR
+8. updating the Backport tracker issue
+
+When run with --cherry-pick-only, the script will stop after the cherry-pick
+phase.
+
+If any of the commits do not cherry-pick cleanly, the script will abort in
+step 4. In this case, you can either finish the cherry-picking manually
+or abort the cherry-pick. In any case, when and if the local wip branch is
+ready (all commits cherry-picked), if you run the script again, like so:
+
+ ${this_script} 41502
+
+the script will detect that the wip branch already exists and skip over
+steps 1-4, starting from step 5 ("PR phase"). In other words, if the wip branch
+already exists for any reason, the script will assume that the cherry-pick
+phase (steps 1-4) is complete.
+
+As this implies, you can do steps 1-4 manually. Provided the wip branch name
+is in the format wip-\$TRACKER_ID-\$STABLE_RELEASE (e.g. "wip-41502-mimic"),
+the script will detect the wip branch and start from step 5.
+
+For details on all the options the script takes, run:
+
+ ${this_script} --help
+
+For more information on Ceph backporting, see:
+
+ https://github.com/ceph/ceph/tree/master/SubmittingPatches-backports.rst
+
+EOM
+}
+
+function verbose {
+ log verbose "$@"
+}
+
+function verbose_en {
+ log verbose_en "$@"
+}
+
+function vet_pr_milestone {
+ local pr_number="$1"
+ local pr_title="$2"
+ local pr_url="$3"
+ local milestone_stanza="$4"
+ local milestone_title_should_be="$5"
+ local milestone_number_should_be
+ local milestone_number_is=
+ local milestone_title_is=
+ milestone_number_should_be="$(try_known_milestones "$milestone_title_should_be")"
+ log overwrite "Vetting milestone of PR#${pr_number}\r"
+ if [ "$milestone_stanza" = "null" ] ; then
+ blindly_set_pr_metadata "$pr_number" "{\"milestone\": $milestone_number_should_be}"
+ warning "$pr_url: set milestone to \"$milestone_title_should_be\""
+ flag_pr "$pr_number" "$pr_url" "milestone not set"
+ else
+ milestone_title_is=$(echo "$milestone_stanza" | jq -r '.title')
+ milestone_number_is=$(echo "$milestone_stanza" | jq -r '.number')
+ if [ "$milestone_number_is" -eq "$milestone_number_should_be" ] ; then
+ true
+ else
+ blindly_set_pr_metadata "$pr_number" "{\"milestone\": $milestone_number_should_be}"
+ warning "$pr_url: changed milestone from \"$milestone_title_is\" to \"$milestone_title_should_be\""
+ flag_pr "$pr_number" "$pr_url" "milestone set to wrong value \"$milestone_title_is\""
+ fi
+ fi
+}
+
+function vet_prs_for_milestone {
+ local milestone_title="$1"
+ local pages_of_output=
+ local pr_number=
+ local pr_title=
+ local pr_url=
+ # determine last page (i.e., total number of pages)
+ remote_api_output="$(curl -u ${github_user}:${github_token} --silent --head "https://api.github.com/repos/ceph/ceph/pulls?base=${milestone_title}" | grep -E '^Link' || true)"
+ if [ "$remote_api_output" ] ; then
+ # Link: <https://api.github.com/repositories/2310495/pulls?base=luminous&page=2>; rel="next", <https://api.github.com/repositories/2310495/pulls?base=luminous&page=2>; rel="last"
+ # shellcheck disable=SC2001
+ pages_of_output="$(echo "$remote_api_output" | sed 's/^.*&page\=\([0-9]\+\)>; rel=\"last\".*$/\1/g')"
+ else
+ pages_of_output="1"
+ fi
+ verbose "GitHub has $pages_of_output pages of pull request data for \"base:${milestone_title}\""
+ for ((page=1; page<=pages_of_output; page++)) ; do
+ verbose "Fetching PRs (page $page of ${pages_of_output})"
+ remote_api_output="$(curl -u ${github_user}:${github_token} --silent -X GET "https://api.github.com/repos/ceph/ceph/pulls?base=${milestone_title}&page=${page}")"
+ prs_in_page="$(echo "$remote_api_output" | jq -r '. | length')"
+ verbose "Page $page of remote API output contains information on $prs_in_page PRs"
+ for ((i=0; i<prs_in_page; i++)) ; do
+ pr_number="$(echo "$remote_api_output" | jq -r ".[${i}].number")"
+ pr_title="$(echo "$remote_api_output" | jq -r ".[${i}].title")"
+ pr_url="$(number_to_url "github" "${pr_number}")"
+ milestone_stanza="$(echo "$remote_api_output" | jq -r ".[${i}].milestone")"
+ vet_pr_milestone "$pr_number" "$pr_title" "$pr_url" "$milestone_stanza" "$milestone_title"
+ done
+ clear_line
+ done
+}
+
+function vet_remotes {
+ if [ "$upstream_remote" ] ; then
+ verbose "Upstream remote is $upstream_remote"
+ else
+ error "Cannot auto-determine upstream remote"
+ "(Could not find any upstream remote in \"git remote -v\")"
+ false
+ fi
+ if [ "$fork_remote" ] ; then
+ verbose "Fork remote is $fork_remote"
+ else
+ error "Cannot auto-determine fork remote"
+ if [ "$EXPLICIT_FORK" ] ; then
+ info "(Could not find $EXPLICIT_FORK fork of ceph/ceph in \"git remote -v\")"
+ else
+ info "(Could not find GitHub user ${github_user}'s fork of ceph/ceph in \"git remote -v\")"
+ fi
+ setup_ok=""
+ fi
+}
+
+function vet_setup {
+ local argument="$1"
+ local not_set="!!! NOT SET !!!"
+ local invalid="!!! INVALID !!!"
+ local redmine_endpoint_display
+ local redmine_user_id_display
+ local github_endpoint_display
+ local github_user_display
+ local upstream_remote_display
+ local fork_remote_display
+ local redmine_key_display
+ local github_token_display
+ debug "Entering vet_setup with argument $argument"
+ if [ "$argument" = "--report" ] || [ "$argument" = "--normal-operation" ] ; then
+ [ "$github_token" ] && [ "$setup_ok" ] && set_github_user_from_github_token quiet
+ init_upstream_remote
+ [ "$github_token" ] && [ "$setup_ok" ] && init_fork_remote
+ vet_remotes
+ [ "$redmine_key" ] && set_redmine_user_from_redmine_key
+ fi
+ if [ "$github_token" ] ; then
+ if [ "$setup_ok" ] ; then
+ github_token_display="(OK; value not shown)"
+ else
+ github_token_display="$invalid"
+ fi
+ else
+ github_token_display="$not_set"
+ fi
+ if [ "$redmine_key" ] ; then
+ if [ "$setup_ok" ] ; then
+ redmine_key_display="(OK; value not shown)"
+ else
+ redmine_key_display="$invalid"
+ fi
+ else
+ redmine_key_display="$not_set"
+ fi
+ redmine_endpoint_display="${redmine_endpoint:-$not_set}"
+ redmine_user_id_display="${redmine_user_id:-$not_set}"
+ github_endpoint_display="${github_endpoint:-$not_set}"
+ github_user_display="${github_user:-$not_set}"
+ upstream_remote_display="${upstream_remote:-$not_set}"
+ fork_remote_display="${fork_remote:-$not_set}"
+ test "$redmine_endpoint" || failed_mandatory_var_check redmine_endpoint "not set"
+ test "$redmine_user_id" || failed_mandatory_var_check redmine_user_id "could not be determined"
+ test "$redmine_key" || failed_mandatory_var_check redmine_key "not set"
+ test "$github_endpoint" || failed_mandatory_var_check github_endpoint "not set"
+ test "$github_user" || failed_mandatory_var_check github_user "could not be determined"
+ test "$github_token" || failed_mandatory_var_check github_token "not set"
+ test "$upstream_remote" || failed_mandatory_var_check upstream_remote "could not be determined"
+ test "$fork_remote" || failed_mandatory_var_check fork_remote "could not be determined"
+ if [ "$argument" = "--report" ] || [ "$argument" == "--interactive" ] ; then
+ read -r -d '' setup_summary <<EOM || true > /dev/null 2>&1
+redmine_endpoint $redmine_endpoint
+redmine_user_id $redmine_user_id_display
+redmine_key $redmine_key_display
+github_endpoint $github_endpoint
+github_user $github_user_display
+github_token $github_token_display
+upstream_remote $upstream_remote_display
+fork_remote $fork_remote_display
+EOM
+ log bare
+ log bare "============================================="
+ log bare " ${this_script} setup report"
+ log bare "============================================="
+ log bare "variable name value"
+ log bare "---------------------------------------------"
+ log bare "$setup_summary"
+ log bare "---------------------------------------------"
+ else
+ verbose "redmine_endpoint $redmine_endpoint_display"
+ verbose "redmine_user_id $redmine_user_id_display"
+ verbose "redmine_key $redmine_key_display"
+ verbose "github_endpoint $github_endpoint_display"
+ verbose "github_user $github_user_display"
+ verbose "github_token $github_token_display"
+ verbose "upstream_remote $upstream_remote_display"
+ verbose "fork_remote $fork_remote_display"
+ fi
+ if [ "$argument" = "--report" ] || [ "$argument" = "--interactive" ] ; then
+ if [ "$setup_ok" ] ; then
+ info "setup is OK"
+ else
+ info "setup is NOT OK"
+ fi
+ log bare "=============================================="
+ log bare
+ fi
+}
+
+function warning {
+ log warning "$@"
+}
+
+
+#
+# are we in a local git clone?
+#
+
+if git status >/dev/null 2>&1 ; then
+ debug "In a local git clone. Good."
+else
+ error "This script must be run from inside a local git clone"
+ abort_due_to_setup_problem
+fi
+
+#
+# do we have jq available?
+#
+
+if type jq >/dev/null 2>&1 ; then
+ debug "jq is available. Good."
+else
+ error "This script uses jq, but it does not seem to be installed"
+ abort_due_to_setup_problem
+fi
+
+#
+# is jq available?
+#
+
+if command -v jq >/dev/null ; then
+ debug "jq is available. Good."
+else
+ error "This script needs \"jq\" in order to work, and it is not available"
+ abort_due_to_setup_problem
+fi
+
+
+#
+# process command-line arguments
+#
+
+munged_options=$(getopt -o c:dhsv --long "cherry-pick-only,component:,debug,existing-pr:,force,fork:,help,milestones,prepare,setup,setup-report,troubleshooting,update-version,usage,verbose,version" -n "$this_script" -- "$@")
+eval set -- "$munged_options"
+
+ADVICE=""
+CHECK_MILESTONES=""
+CHERRY_PICK_ONLY=""
+CHERRY_PICK_PHASE="yes"
+DEBUG=""
+EXISTING_PR=""
+EXPLICIT_COMPONENT=""
+EXPLICIT_FORK=""
+FORCE=""
+HELP=""
+INTERACTIVE_SETUP_ROUTINE=""
+ISSUE=""
+PR_PHASE="yes"
+SETUP_OPTION=""
+TRACKER_PHASE="yes"
+TROUBLESHOOTING_ADVICE=""
+USAGE_ADVICE=""
+VERBOSE=""
+while true ; do
+ case "$1" in
+ --cherry-pick-only) CHERRY_PICK_PHASE="yes" ; PR_PHASE="" ; TRACKER_PHASE="" ; shift ;;
+ --component|-c) shift ; EXPLICIT_COMPONENT="$1" ; shift ;;
+ --debug|-d) DEBUG="$1" ; shift ;;
+ --existing-pr) shift ; EXISTING_PR="$1" ; CHERRY_PICK_PHASE="" ; PR_PHASE="" ; shift ;;
+ --force) FORCE="$1" ; shift ;;
+ --fork) shift ; EXPLICIT_FORK="$1" ; shift ;;
+ --help|-h) ADVICE="1" ; HELP="$1" ; shift ;;
+ --milestones) CHECK_MILESTONES="$1" ; shift ;;
+ --prepare) CHERRY_PICK_PHASE="yes" ; PR_PHASE="" ; TRACKER_PHASE="" ; shift ;;
+ --setup*|-s) SETUP_OPTION="$1" ; shift ;;
+ --troubleshooting) ADVICE="$1" ; TROUBLESHOOTING_ADVICE="$1" ; shift ;;
+ --update-version) update_version_number_and_exit ;;
+ --usage) ADVICE="$1" ; USAGE_ADVICE="$1" ; shift ;;
+ --verbose|-v) VERBOSE="$1" ; shift ;;
+ --version) display_version_message_and_exit ;;
+ --) shift ; ISSUE="$1" ; break ;;
+ *) echo "Internal error" ; false ;;
+ esac
+done
+
+if [ "$ADVICE" ] ; then
+ [ "$HELP" ] && usage
+ [ "$USAGE_ADVICE" ] && usage_advice
+ [ "$TROUBLESHOOTING_ADVICE" ] && troubleshooting_advice
+ exit 0
+fi
+
+if [ "$SETUP_OPTION" ] || [ "$CHECK_MILESTONES" ] ; then
+ ISSUE="0"
+fi
+
+if [[ $ISSUE =~ ^[0-9]+$ ]] ; then
+ issue=$ISSUE
+else
+ error "Invalid or missing argument"
+ usage
+ false
+fi
+
+if [ "$DEBUG" ]; then
+ set -x
+ VERBOSE="--verbose"
+fi
+
+if [ "$VERBOSE" ]; then
+ info "Verbose mode ON"
+ VERBOSE="--verbose"
+fi
+
+
+#
+# make sure setup has been completed
+#
+
+init_endpoints
+init_github_token
+init_redmine_key
+setup_ok="OK"
+if [ "$SETUP_OPTION" ] ; then
+ vet_setup --report
+ maybe_delete_deprecated_backport_common
+ if [ "$setup_ok" ] ; then
+ exit 0
+ else
+ default_val="y"
+ echo -n "Run the interactive setup routine now? (default: ${default_val}) "
+ yes_or_no_answer="$(get_user_input "$default_val")"
+ [ "$yes_or_no_answer" ] && yes_or_no_answer="${yes_or_no_answer:0:1}"
+ if [ "$yes_or_no_answer" = "y" ] ; then
+ INTERACTIVE_SETUP_ROUTINE="yes"
+ else
+ if [ "$FORCE" ] ; then
+ warning "--force was given; proceeding with broken setup"
+ else
+ info "Bailing out!"
+ exit 1
+ fi
+ fi
+ fi
+fi
+if [ "$INTERACTIVE_SETUP_ROUTINE" ] ; then
+ interactive_setup_routine
+else
+ vet_setup --normal-operation
+ maybe_delete_deprecated_backport_common
+fi
+if [ "$INTERACTIVE_SETUP_ROUTINE" ] || [ "$SETUP_OPTION" ] ; then
+ echo
+ if [ "$setup_ok" ] ; then
+ if [ "$ISSUE" ] && [ "$ISSUE" != "0" ] ; then
+ true
+ else
+ exit 0
+ fi
+ else
+ exit 1
+ fi
+fi
+vet_remotes
+[ "$setup_ok" ] || abort_due_to_setup_problem
+
+#
+# query remote GitHub API for active milestones
+#
+
+verbose "Querying GitHub API for active milestones"
+remote_api_output="$(curl -u ${github_user}:${github_token} --silent -X GET "https://api.github.com/repos/ceph/ceph/milestones")"
+active_milestones="$(echo "$remote_api_output" | jq -r '.[] | .title')"
+if [ "$active_milestones" = "null" ] ; then
+ error "Could not determine the active milestones"
+ bail_out_github_api "$remote_api_output"
+fi
+
+if [ "$CHECK_MILESTONES" ] ; then
+ check_milestones "$active_milestones"
+ exit 0
+fi
+
+#
+# query remote Redmine API for information about the Backport tracker issue
+#
+
+redmine_url="$(number_to_url "redmine" "${issue}")"
+debug "Considering Redmine issue: $redmine_url - is it in the Backport tracker?"
+
+remote_api_output="$(curl --silent "${redmine_url}.json")"
+tracker="$(echo "$remote_api_output" | jq -r '.issue.tracker.name')"
+if [ "$tracker" = "Backport" ]; then
+ debug "Yes, $redmine_url is a Backport issue"
+else
+ error "Issue $redmine_url is not a Backport"
+ info "(This script only works with Backport tracker issues.)"
+ false
+fi
+
+debug "Looking up release/milestone of $redmine_url"
+milestone="$(echo "$remote_api_output" | jq -r '.issue.custom_fields[0].value')"
+if [ "$milestone" ] ; then
+ debug "Release/milestone: $milestone"
+else
+ error "could not obtain release/milestone from ${redmine_url}"
+ false
+fi
+
+debug "Looking up status of $redmine_url"
+tracker_status_id="$(echo "$remote_api_output" | jq -r '.issue.status.id')"
+tracker_status_name="$(echo "$remote_api_output" | jq -r '.issue.status.name')"
+if [ "$tracker_status_name" ] ; then
+ debug "Tracker status: $tracker_status_name"
+ if [ "$FORCE" ] || [ "$EXISTING_PR" ] ; then
+ test "$(check_tracker_status "$tracker_status_name")" || true
+ else
+ test "$(check_tracker_status "$tracker_status_name")"
+ fi
+else
+ error "could not obtain status from ${redmine_url}"
+ false
+fi
+
+tracker_title="$(echo "$remote_api_output" | jq -r '.issue.subject')"
+debug "Title of $redmine_url is ->$tracker_title<-"
+
+tracker_description="$(echo "$remote_api_output" | jq -r '.issue.description')"
+debug "Description of $redmine_url is ->$tracker_description<-"
+
+tracker_assignee_id="$(echo "$remote_api_output" | jq -r '.issue.assigned_to.id')"
+tracker_assignee_name="$(echo "$remote_api_output" | jq -r '.issue.assigned_to.name')"
+if [ "$tracker_assignee_id" = "null" ] || [ "$tracker_assignee_id" = "$redmine_user_id" ] ; then
+ true
+else
+ error_msg_1="$redmine_url is assigned to someone else: $tracker_assignee_name (ID $tracker_assignee_id)"
+ error_msg_2="(my ID is $redmine_user_id)"
+ if [ "$FORCE" ] || [ "$EXISTING_PR" ] ; then
+ warning "$error_msg_1"
+ info "$error_msg_2"
+ info "--force and/or --existing-pr given: continuing execution"
+ else
+ error "$error_msg_1"
+ info "$error_msg_2"
+ info "Cowardly refusing to continue"
+ false
+ fi
+fi
+
+if [ -z "$(is_active_milestone "$milestone")" ] ; then
+ error "$redmine_url is a backport to $milestone which is not an active milestone"
+ info "Cowardly refusing to work on a backport to an inactive release"
+ false
+fi
+
+milestone_number=$(try_known_milestones "$milestone")
+if [ "$milestone_number" -gt "0" ] >/dev/null 2>&1 ; then
+ debug "Milestone ->$milestone<- is known to have number ->$milestone_number<-: skipping remote API call"
+else
+ warning "Milestone ->$milestone<- is unknown to the script: falling back to GitHub API"
+ milestone_number=$(milestone_number_from_remote_api "$milestone")
+fi
+target_branch="$milestone"
+info "milestone/release is $milestone"
+debug "milestone number is $milestone_number"
+
+if [ "$CHERRY_PICK_PHASE" ] ; then
+ local_branch=wip-${issue}-${target_branch}
+ if git show-ref --verify --quiet "refs/heads/$local_branch" ; then
+ if [ "$FORCE" ] ; then
+ warning "local branch $local_branch already exists"
+ info "--force was given: will clobber $local_branch and attempt automated cherry-pick"
+ cherry_pick_phase
+ elif [ "$CHERRY_PICK_ONLY" ] ; then
+ error "local branch $local_branch already exists"
+ info "Cowardly refusing to clobber $local_branch as it might contain valuable data"
+ info "(hint) run with --force to clobber it and attempt the cherry-pick"
+ false
+ fi
+ if [ "$FORCE" ] || [ "$CHERRY_PICK_ONLY" ] ; then
+ true
+ else
+ info "local branch $local_branch already exists: skipping cherry-pick phase"
+ fi
+ else
+ info "$local_branch does not exist: will create it and attempt automated cherry-pick"
+ cherry_pick_phase
+ fi
+fi
+
+if [ "$PR_PHASE" ] ; then
+ current_branch=$(git rev-parse --abbrev-ref HEAD)
+ if [ "$current_branch" = "$local_branch" ] ; then
+ true
+ else
+ set -x
+ git checkout "$local_branch"
+ set +x
+ maybe_restore_set_x
+ fi
+
+ set -x
+ git push -u "$fork_remote" "$local_branch"
+ set +x
+ maybe_restore_set_x
+
+ original_issue=""
+ original_pr=""
+ original_pr_url=""
+
+ debug "Generating backport PR description"
+ populate_original_issue
+ populate_original_pr
+ desc="backport tracker: ${redmine_url}"
+ if [ "$original_pr" ] || [ "$original_issue" ] ; then
+ desc="${desc}\n\n---\n"
+ [ "$original_pr" ] && desc="${desc}\nbackport of $(number_to_url "github" "${original_pr}")"
+ [ "$original_issue" ] && desc="${desc}\nparent tracker: $(number_to_url "redmine" "${original_issue}")"
+ fi
+ desc="${desc}\n\nthis backport was staged using ceph-backport.sh version ${SCRIPT_VERSION}\nfind the latest version at ${github_endpoint}/blob/master/src/script/ceph-backport.sh"
+
+ debug "Generating backport PR title"
+ if [ "$original_pr" ] ; then
+ backport_pr_title="${milestone}: $(curl --silent https://api.github.com/repos/ceph/ceph/pulls/${original_pr} | jq -r '.title')"
+ else
+ if [[ $tracker_title =~ ^${milestone}: ]] ; then
+ backport_pr_title="${tracker_title}"
+ else
+ backport_pr_title="${milestone}: ${tracker_title}"
+ fi
+ fi
+ if [[ "$backport_pr_title" =~ \" ]] ; then
+ backport_pr_title="${backport_pr_title//\"/\\\"}"
+ fi
+
+ debug "Opening backport PR"
+ if [ "$EXPLICIT_FORK" ] ; then
+ source_repo="$EXPLICIT_FORK"
+ else
+ source_repo="$github_user"
+ fi
+ remote_api_output=$(curl -u ${github_user}:${github_token} --silent --data-binary "{\"title\":\"${backport_pr_title}\",\"head\":\"${source_repo}:${local_branch}\",\"base\":\"${target_branch}\",\"body\":\"${desc}\"}" "https://api.github.com/repos/ceph/ceph/pulls")
+ backport_pr_number=$(echo "$remote_api_output" | jq -r .number)
+ if [ -z "$backport_pr_number" ] || [ "$backport_pr_number" = "null" ] ; then
+ error "failed to open backport PR"
+ bail_out_github_api "$remote_api_output"
+ fi
+ backport_pr_url="$(number_to_url "github" "$backport_pr_number")"
+ info "Opened backport PR ${backport_pr_url}"
+fi
+
+if [ "$EXISTING_PR" ] ; then
+ populate_original_issue
+ populate_original_pr
+ backport_pr_number="$EXISTING_PR"
+ backport_pr_url="$(number_to_url "github" "$backport_pr_number")"
+ existing_pr_routine
+fi
+
+if [ "$PR_PHASE" ] || [ "$EXISTING_PR" ] ; then
+ maybe_update_pr_milestone_labels
+ pgrep firefox >/dev/null && firefox "${backport_pr_url}"
+fi
+
+if [ "$TRACKER_PHASE" ] ; then
+ debug "Considering Backport tracker issue ${redmine_url}"
+ status_should_be=2 # In Progress
+ desc_should_be="${backport_pr_url}"
+ assignee_should_be="${redmine_user_id}"
+ if [ "$EXISTING_PR" ] ; then
+ data_binary="{\"issue\":{\"description\":\"${desc_should_be}\",\"status_id\":${status_should_be}}}"
+ else
+ data_binary="{\"issue\":{\"description\":\"${desc_should_be}\",\"status_id\":${status_should_be},\"assigned_to_id\":${assignee_should_be}}}"
+ fi
+ remote_api_status_code="$(curl --write-out '%{http_code}' --output /dev/null --silent -X PUT --header "Content-type: application/json" --data-binary "${data_binary}" "${redmine_url}.json?key=$redmine_key")"
+ if [ "$FORCE" ] || [ "$EXISTING_PR" ] ; then
+ true
+ else
+ if [ "${remote_api_status_code:0:1}" = "2" ] ; then
+ true
+ elif [ "${remote_api_status_code:0:1}" = "4" ] ; then
+ warning "remote API ${redmine_endpoint} returned status ${remote_api_status_code}"
+ info "This merely indicates that you cannot modify issue fields at ${redmine_endpoint}"
+ info "and does not limit your ability to do backports."
+ else
+ error "Remote API ${redmine_endpoint} returned unexpected response code ${remote_api_status_code}"
+ fi
+ fi
+ # check if anything actually changed on the Redmine issue
+ remote_api_output=$(curl --silent "${redmine_url}.json?include=journals")
+ status_is="$(echo "$remote_api_output" | jq -r '.issue.status.id')"
+ desc_is="$(echo "$remote_api_output" | jq -r '.issue.description')"
+ assignee_is="$(echo "$remote_api_output" | jq -r '.issue.assigned_to.id')"
+ tracker_was_updated=""
+ tracker_is_in_desired_state="yes"
+ [ "$(tracker_component_was_updated "status" "$tracker_status_id" "$status_is")" ] && tracker_was_updated="yes"
+ [ "$(tracker_component_was_updated "desc" "$tracker_description" "$desc_is")" ] && tracker_was_updated="yes"
+ if [ "$EXISTING_PR" ] ; then
+ true
+ else
+ [ "$(tracker_component_was_updated "assignee" "$tracker_assignee_id" "$assignee_is")" ] && tracker_was_updated="yes"
+ fi
+ [ "$(tracker_component_is_in_desired_state "status" "$status_is" "$status_should_be")" ] || tracker_is_in_desired_state=""
+ [ "$(tracker_component_is_in_desired_state "desc" "$desc_is" "$desc_should_be")" ] || tracker_is_in_desired_state=""
+ if [ "$EXISTING_PR" ] ; then
+ true
+ else
+ [ "$(tracker_component_is_in_desired_state "assignee" "$assignee_is" "$assignee_should_be")" ] || tracker_is_in_desired_state=""
+ fi
+ if [ "$tracker_is_in_desired_state" ] ; then
+ [ "$tracker_was_updated" ] && info "Backport tracker ${redmine_url} was updated"
+ info "Backport tracker ${redmine_url} is in the desired state"
+ pgrep firefox >/dev/null && firefox "${redmine_url}"
+ exit 0
+ fi
+ if [ "$tracker_was_updated" ] ; then
+ warning "backport tracker ${redmine_url} was updated, but is not in the desired state. Please check it."
+ pgrep firefox >/dev/null && firefox "${redmine_url}"
+ exit 1
+ else
+ data_binary="{\"issue\":{\"notes\":\"please link this Backport tracker issue with GitHub PR ${desc_should_be}\nceph-backport.sh version ${SCRIPT_VERSION}\"}}"
+ remote_api_status_code=$(curl --write-out '%{http_code}' --output /dev/null --silent -X PUT --header "Content-type: application/json" --data-binary "${data_binary}" "${redmine_url}.json?key=$redmine_key")
+ if [ "${remote_api_status_code:0:1}" = "2" ] ; then
+ info "Comment added to ${redmine_url}"
+ fi
+ exit 0
+ fi
+fi