diff options
Diffstat (limited to 'src/script/ptl-tool.py')
| -rwxr-xr-x | src/script/ptl-tool.py | 404 |
1 files changed, 404 insertions, 0 deletions
diff --git a/src/script/ptl-tool.py b/src/script/ptl-tool.py new file mode 100755 index 000000000..17931313f --- /dev/null +++ b/src/script/ptl-tool.py @@ -0,0 +1,404 @@ +#!/usr/bin/python3 + +# README: +# +# This tool's purpose is to make it easier to merge PRs into test branches and +# into master. Make sure you generate a Personal access token in GitHub and +# add it your ~/.github.key. +# +# Because developers often have custom names for the ceph upstream remote +# (https://github.com/ceph/ceph.git), You will probably want to export the +# PTL_TOOL_BASE_PATH environment variable in your shell rc files before using +# this script: +# +# export PTL_TOOL_BASE_PATH=refs/remotes/<remotename>/ +# +# and PTL_TOOL_BASE_REMOTE as the name of your Ceph upstream remote (default: "upstream"): +# +# export PTL_TOOL_BASE_REMOTE=<remotename> +# +# +# ** Here are some basic exmples to get started: ** +# +# Merging all PRs labeled 'wip-pdonnell-testing' into a new test branch: +# +# $ src/script/ptl-tool.py --pr-label wip-pdonnell-testing +# Adding labeled PR #18805 to PR list +# Adding labeled PR #18774 to PR list +# Adding labeled PR #18600 to PR list +# Will merge PRs: [18805, 18774, 18600] +# Detaching HEAD onto base: master +# Merging PR #18805 +# Merging PR #18774 +# Merging PR #18600 +# Checked out new branch wip-pdonnell-testing-20171108.054517 +# Created tag testing/wip-pdonnell-testing-20171108.054517 +# +# +# Merging all PRs labeled 'wip-pdonnell-testing' into master: +# +# $ src/script/ptl-tool.py --pr-label wip-pdonnell-testing --branch master +# Adding labeled PR #18805 to PR list +# Adding labeled PR #18774 to PR list +# Adding labeled PR #18600 to PR list +# Will merge PRs: [18805, 18774, 18600] +# Detaching HEAD onto base: master +# Merging PR #18805 +# Merging PR #18774 +# Merging PR #18600 +# Checked out branch master +# +# Now push to master: +# $ git push upstream master +# ... +# +# +# Merging PR #1234567 and #2345678 into a new test branch with a testing label added to the PR: +# +# $ src/script/ptl-tool.py 1234567 2345678 --label wip-pdonnell-testing +# Detaching HEAD onto base: master +# Merging PR #1234567 +# Labeled PR #1234567 wip-pdonnell-testing +# Merging PR #2345678 +# Labeled PR #2345678 wip-pdonnell-testing +# Deleted old test branch wip-pdonnell-testing-20170928 +# Created branch wip-pdonnell-testing-20170928 +# Created tag testing/wip-pdonnell-testing-20170928_03 +# +# +# Merging PR #1234567 into master leaving a detached HEAD (i.e. do not update your repo's master branch) and do not label: +# +# $ src/script/ptl-tool.py --branch HEAD --merge-branch-name master 1234567 +# Detaching HEAD onto base: master +# Merging PR #1234567 +# Leaving HEAD detached; no branch anchors your commits +# +# Now push to master: +# $ git push upstream HEAD:master +# +# +# Merging PR #12345678 into luminous leaving a detached HEAD (i.e. do not update your repo's master branch) and do not label: +# +# $ src/script/ptl-tool.py --base luminous --branch HEAD --merge-branch-name luminous 12345678 +# Detaching HEAD onto base: luminous +# Merging PR #12345678 +# Leaving HEAD detached; no branch anchors your commits +# +# Now push to luminous: +# $ git push upstream HEAD:luminous +# +# +# Merging all PRs labelled 'wip-pdonnell-testing' into master leaving a detached HEAD: +# +# $ src/script/ptl-tool.py --base master --branch HEAD --merge-branch-name master --pr-label wip-pdonnell-testing +# Adding labeled PR #18192 to PR list +# Will merge PRs: [18192] +# Detaching HEAD onto base: master +# Merging PR #18192 +# Leaving HEAD detached; no branch anchors your commit + + +# TODO +# Look for check failures? +# redmine issue update: http://www.redmine.org/projects/redmine/wiki/Rest_Issues + +import argparse +import codecs +import datetime +import getpass +import git +import itertools +import json +import logging +import os +import re +import requests +import sys + +from os.path import expanduser + +log = logging.getLogger(__name__) +log.addHandler(logging.StreamHandler()) +log.setLevel(logging.INFO) + +BASE_PROJECT = os.getenv("PTL_TOOL_BASE_PROJECT", "ceph") +BASE_REPO = os.getenv("PTL_TOOL_BASE_REPO", "ceph") +BASE_REMOTE = os.getenv("PTL_TOOL_BASE_REMOTE", "upstream") +BASE_PATH = os.getenv("PTL_TOOL_BASE_PATH", "refs/remotes/upstream/") +GITDIR = os.getenv("PTL_TOOL_GITDIR", ".") +USER = os.getenv("PTL_TOOL_USER", getpass.getuser()) +with open(expanduser("~/.github.key")) as f: + PASSWORD = f.read().strip() +TEST_BRANCH = os.getenv("PTL_TOOL_TEST_BRANCH", "wip-{user}-testing-%Y%m%d.%H%M%S") + +SPECIAL_BRANCHES = ('master', 'luminous', 'jewel', 'HEAD') + +INDICATIONS = [ + re.compile("(Reviewed-by: .+ <[\w@.-]+>)", re.IGNORECASE), + re.compile("(Acked-by: .+ <[\w@.-]+>)", re.IGNORECASE), + re.compile("(Tested-by: .+ <[\w@.-]+>)", re.IGNORECASE), +] + +# find containing git dir +git_dir = GITDIR +max_levels = 6 +while not os.path.exists(git_dir + '/.git'): + git_dir += '/..' + max_levels -= 1 + if max_levels < 0: + break + +CONTRIBUTORS = {} +NEW_CONTRIBUTORS = {} +with codecs.open(git_dir + "/.githubmap", encoding='utf-8') as f: + comment = re.compile("\s*#") + patt = re.compile("([\w-]+)\s+(.*)") + for line in f: + if comment.match(line): + continue + m = patt.match(line) + CONTRIBUTORS[m.group(1)] = m.group(2) + +BZ_MATCH = re.compile("(.*https?://bugzilla.redhat.com/.*)") +TRACKER_MATCH = re.compile("(.*https?://tracker.ceph.com/.*)") + +def get(session, url, params=None, paging=True): + if params is None: + params = {} + params['per_page'] = 100 + + log.debug(f"Fetching {url}") + response = session.get(url, auth=(USER, PASSWORD), params=params) + log.debug(f"Response = {response}; links = {response.headers.get('link', '')}") + if response.status_code != 200: + log.error(f"Failed to fetch {url}: {response}") + sys.exit(1) + j = response.json() + yield j + if paging: + link = response.headers.get('link', None) + page = 2 + while link is not None and 'next' in link: + log.debug(f"Fetching {url}") + new_params = dict(params) + new_params.update({'page': page}) + response = session.get(url, auth=(USER, PASSWORD), params=new_params) + log.debug(f"Response = {response}; links = {response.headers.get('link', '')}") + if response.status_code != 200: + log.error(f"Failed to fetch {url}: {response}") + sys.exit(1) + yield response.json() + link = response.headers.get('link', None) + page += 1 + +def get_credits(session, pr, pr_req): + comments = [pr_req] + + log.debug(f"Getting comments for #{pr}") + endpoint = f"https://api.github.com/repos/{BASE_PROJECT}/{BASE_REPO}/issues/{pr}/comments" + for c in get(session, endpoint): + comments.extend(c) + + log.debug(f"Getting reviews for #{pr}") + endpoint = f"https://api.github.com/repos/{BASE_PROJECT}/{BASE_REPO}/pulls/{pr}/reviews" + reviews = [] + for c in get(session, endpoint): + comments.extend(c) + reviews.extend(c) + + log.debug(f"Getting review comments for #{pr}") + endpoint = f"https://api.github.com/repos/{BASE_PROJECT}/{BASE_REPO}/pulls/{pr}/comments" + for c in get(session, endpoint): + comments.extend(c) + + credits = set() + for comment in comments: + body = comment["body"] + if body: + url = comment["html_url"] + for m in BZ_MATCH.finditer(body): + log.info("[ {url} ] BZ cited: {cite}".format(url=url, cite=m.group(1))) + for m in TRACKER_MATCH.finditer(body): + log.info("[ {url} ] Ceph tracker cited: {cite}".format(url=url, cite=m.group(1))) + for indication in INDICATIONS: + for cap in indication.findall(comment["body"]): + credits.add(cap) + + new_new_contributors = {} + for review in reviews: + if review["state"] == "APPROVED": + user = review["user"]["login"] + try: + credits.add("Reviewed-by: "+CONTRIBUTORS[user]) + except KeyError as e: + try: + credits.add("Reviewed-by: "+NEW_CONTRIBUTORS[user]) + except KeyError as e: + try: + name = input("Need name for contributor \"%s\" (use ^D to skip); Reviewed-by: " % user) + name = name.strip() + if len(name) == 0: + continue + NEW_CONTRIBUTORS[user] = name + new_new_contributors[user] = name + credits.add("Reviewed-by: "+name) + except EOFError as e: + continue + + return "\n".join(credits), new_new_contributors + +def build_branch(args): + base = args.base + branch = datetime.datetime.utcnow().strftime(args.branch).format(user=USER) + label = args.label + merge_branch_name = args.merge_branch_name + if merge_branch_name is False: + merge_branch_name = branch + + session = requests.Session() + + if label: + # Check the label format + if re.search(r'\bwip-(.*?)-testing\b', label) is None: + log.error("Unknown Label '{lblname}'. Label Format: wip-<name>-testing".format(lblname=label)) + sys.exit(1) + + # Check if the Label exist in the repo + endpoint = f"https://api.github.com/repos/{BASE_PROJECT}/{BASE_REPO}/labels/{label}" + get(session, endpoint, paging=False) + + G = git.Repo(args.git) + + # First get the latest base branch and PRs from BASE_REMOTE + remote = getattr(G.remotes, BASE_REMOTE) + remote.fetch() + + prs = args.prs + if args.pr_label is not None: + if args.pr_label == '' or args.pr_label.isspace(): + log.error("--pr-label must have a non-space value") + sys.exit(1) + payload = {'labels': args.pr_label, 'sort': 'created', 'direction': 'desc'} + endpoint = f"https://api.github.com/repos/{BASE_PROJECT}/{BASE_REPO}/issues" + labeled_prs = [] + for l in get(session, endpoint, params=payload): + labeled_prs.extend(l) + if len(labeled_prs) == 0: + log.error("Search for PRs matching label '{}' returned no results!".format(args.pr_label)) + sys.exit(1) + for pr in labeled_prs: + if pr['pull_request']: + n = pr['number'] + log.info("Adding labeled PR #{} to PR list".format(n)) + prs.append(n) + log.info("Will merge PRs: {}".format(prs)) + + if base == 'HEAD': + log.info("Branch base is HEAD; not checking out!") + else: + log.info("Detaching HEAD onto base: {}".format(base)) + try: + base_path = args.base_path + base + base = next(ref for ref in G.refs if ref.path == base_path) + except StopIteration: + log.error("Branch " + base + " does not exist!") + sys.exit(1) + + # So we know that we're not on an old test branch, detach HEAD onto ref: + base.checkout() + + for pr in prs: + pr = int(pr) + log.info("Merging PR #{pr}".format(pr=pr)) + + remote_ref = "refs/pull/{pr}/head".format(pr=pr) + fi = remote.fetch(remote_ref) + if len(fi) != 1: + log.error("PR {pr} does not exist?".format(pr=pr)) + sys.exit(1) + tip = fi[0].ref.commit + + endpoint = f"https://api.github.com/repos/{BASE_PROJECT}/{BASE_REPO}/pulls/{pr}" + response = next(get(session, endpoint, paging=False)) + + message = "Merge PR #%d into %s\n\n* %s:\n" % (pr, merge_branch_name, remote_ref) + + for commit in G.iter_commits(rev="HEAD.."+str(tip)): + message = message + ("\t%s\n" % commit.message.split('\n', 1)[0]) + # Get tracker issues / bzs cited so the PTL can do updates + short = commit.hexsha[:8] + for m in BZ_MATCH.finditer(commit.message): + log.info("[ {sha1} ] BZ cited: {cite}".format(sha1=short, cite=m.group(1))) + for m in TRACKER_MATCH.finditer(commit.message): + log.info("[ {sha1} ] Ceph tracker cited: {cite}".format(sha1=short, cite=m.group(1))) + + message = message + "\n" + if args.credits: + (addendum, new_contributors) = get_credits(session, pr, response) + message += addendum + else: + new_contributors = [] + + G.git.merge(tip.hexsha, '--no-ff', m=message) + + if new_contributors: + # Check out the PR, add a commit adding to .githubmap + log.info("adding new contributors to githubmap in merge commit") + with open(git_dir + "/.githubmap", "a") as f: + for c in new_contributors: + f.write("%s %s\n" % (c, new_contributors[c])) + G.index.add([".githubmap"]) + G.git.commit("--amend", "--no-edit") + + if label: + req = session.post("https://api.github.com/repos/{project}/{repo}/issues/{pr}/labels".format(pr=pr, project=BASE_PROJECT, repo=BASE_REPO), data=json.dumps([label]), auth=(USER, PASSWORD)) + if req.status_code != 200: + log.error("PR #%d could not be labeled %s: %s" % (pr, label, req)) + sys.exit(1) + log.info("Labeled PR #{pr} {label}".format(pr=pr, label=label)) + + # If the branch is 'HEAD', leave HEAD detached (but use "master" for commit message) + if branch == 'HEAD': + log.info("Leaving HEAD detached; no branch anchors your commits") + else: + created_branch = False + try: + G.head.reference = G.create_head(branch) + log.info("Checked out new branch {branch}".format(branch=branch)) + created_branch = True + except: + G.head.reference = G.create_head(branch, force=True) + log.info("Checked out branch {branch}".format(branch=branch)) + + if created_branch: + # tag it for future reference. + tag = "testing/%s" % branch + git.refs.tag.Tag.create(G, tag) + log.info("Created tag %s" % tag) + +def main(): + parser = argparse.ArgumentParser(description="Ceph PTL tool") + default_base = 'master' + default_branch = TEST_BRANCH + default_label = '' + if len(sys.argv) > 1 and sys.argv[1] in SPECIAL_BRANCHES: + argv = sys.argv[2:] + default_branch = 'HEAD' # Leave HEAD detached + default_base = default_branch + default_label = False + else: + argv = sys.argv[1:] + parser.add_argument('--branch', dest='branch', action='store', default=default_branch, help='branch to create ("HEAD" leaves HEAD detached; i.e. no branch is made)') + parser.add_argument('--merge-branch-name', dest='merge_branch_name', action='store', default=False, help='name of the branch for merge messages') + parser.add_argument('--base', dest='base', action='store', default=default_base, help='base for branch') + parser.add_argument('--base-path', dest='base_path', action='store', default=BASE_PATH, help='base for branch') + parser.add_argument('--git-dir', dest='git', action='store', default=git_dir, help='git directory') + parser.add_argument('--label', dest='label', action='store', default=default_label, help='label PRs for testing') + parser.add_argument('--pr-label', dest='pr_label', action='store', help='label PRs for testing') + parser.add_argument('--no-credits', dest='credits', action='store_false', help='skip indication search (Reviewed-by, etc.)') + parser.add_argument('prs', metavar="PR", type=int, nargs='*', help='Pull Requests to merge') + args = parser.parse_args(argv) + return build_branch(args) + +if __name__ == "__main__": + main() |