summaryrefslogtreecommitdiffstats
path: root/src/script/ptl-tool.py
diff options
context:
space:
mode:
Diffstat (limited to 'src/script/ptl-tool.py')
-rwxr-xr-xsrc/script/ptl-tool.py368
1 files changed, 368 insertions, 0 deletions
diff --git a/src/script/ptl-tool.py b/src/script/ptl-tool.py
new file mode 100755
index 00000000..6d374674
--- /dev/null
+++ b/src/script/ptl-tool.py
@@ -0,0 +1,368 @@
+#!/usr/bin/env python2
+
+# README:
+#
+# This tool's purpose is to make it easier to merge PRs into test branches and
+# into master. Make sure you generate a Personal access token in GitHub and
+# add it your ~/.github.key.
+#
+# Because developers often have custom names for the ceph upstream remote
+# (https://github.com/ceph/ceph.git), You will probably want to export the
+# PTL_TOOL_BASE_PATH environment variable in your shell rc files before using
+# this script:
+#
+# export PTL_TOOL_BASE_PATH=refs/remotes/<remotename>/
+#
+# and PTL_TOOL_BASE_REMOTE as the name of your Ceph upstream remote (default: "upstream"):
+#
+# export PTL_TOOL_BASE_REMOTE=<remotename>
+#
+#
+# ** Here are some basic exmples to get started: **
+#
+# Merging all PRs labeled 'wip-pdonnell-testing' into a new test branch:
+#
+# $ src/script/ptl-tool.py --pr-label wip-pdonnell-testing
+# Adding labeled PR #18805 to PR list
+# Adding labeled PR #18774 to PR list
+# Adding labeled PR #18600 to PR list
+# Will merge PRs: [18805, 18774, 18600]
+# Detaching HEAD onto base: master
+# Merging PR #18805
+# Merging PR #18774
+# Merging PR #18600
+# Checked out new branch wip-pdonnell-testing-20171108.054517
+# Created tag testing/wip-pdonnell-testing-20171108.054517
+#
+#
+# Merging all PRs labeled 'wip-pdonnell-testing' into master:
+#
+# $ src/script/ptl-tool.py --pr-label wip-pdonnell-testing --branch master
+# Adding labeled PR #18805 to PR list
+# Adding labeled PR #18774 to PR list
+# Adding labeled PR #18600 to PR list
+# Will merge PRs: [18805, 18774, 18600]
+# Detaching HEAD onto base: master
+# Merging PR #18805
+# Merging PR #18774
+# Merging PR #18600
+# Checked out branch master
+#
+# Now push to master:
+# $ git push upstream master
+# ...
+#
+#
+# Merging PR #1234567 and #2345678 into a new test branch with a testing label added to the PR:
+#
+# $ src/script/ptl-tool.py 1234567 2345678 --label wip-pdonnell-testing
+# Detaching HEAD onto base: master
+# Merging PR #1234567
+# Labeled PR #1234567 wip-pdonnell-testing
+# Merging PR #2345678
+# Labeled PR #2345678 wip-pdonnell-testing
+# Deleted old test branch wip-pdonnell-testing-20170928
+# Created branch wip-pdonnell-testing-20170928
+# Created tag testing/wip-pdonnell-testing-20170928_03
+#
+#
+# Merging PR #1234567 into master leaving a detached HEAD (i.e. do not update your repo's master branch) and do not label:
+#
+# $ src/script/ptl-tool.py --branch HEAD --merge-branch-name master 1234567
+# Detaching HEAD onto base: master
+# Merging PR #1234567
+# Leaving HEAD detached; no branch anchors your commits
+#
+# Now push to master:
+# $ git push upstream HEAD:master
+#
+#
+# Merging PR #12345678 into luminous leaving a detached HEAD (i.e. do not update your repo's master branch) and do not label:
+#
+# $ src/script/ptl-tool.py --base luminous --branch HEAD --merge-branch-name luminous 12345678
+# Detaching HEAD onto base: luminous
+# Merging PR #12345678
+# Leaving HEAD detached; no branch anchors your commits
+#
+# Now push to luminous:
+# $ git push upstream HEAD:luminous
+#
+#
+# Merging all PRs labelled 'wip-pdonnell-testing' into master leaving a detached HEAD:
+#
+# $ src/script/ptl-tool.py --base master --branch HEAD --merge-branch-name master --pr-label wip-pdonnell-testing
+# Adding labeled PR #18192 to PR list
+# Will merge PRs: [18192]
+# Detaching HEAD onto base: master
+# Merging PR #18192
+# Leaving HEAD detached; no branch anchors your commit
+
+
+# TODO
+# Look for check failures?
+# redmine issue update: http://www.redmine.org/projects/redmine/wiki/Rest_Issues
+
+import argparse
+import codecs
+import datetime
+import getpass
+import git
+import itertools
+import json
+import logging
+import os
+import re
+import requests
+import sys
+
+from os.path import expanduser
+
+log = logging.getLogger(__name__)
+log.addHandler(logging.StreamHandler())
+log.setLevel(logging.INFO)
+
+BASE_PROJECT = os.getenv("PTL_TOOL_BASE_PROJECT", "ceph")
+BASE_REPO = os.getenv("PTL_TOOL_BASE_REPO", "ceph")
+BASE_REMOTE = os.getenv("PTL_TOOL_BASE_REMOTE", "upstream")
+BASE_PATH = os.getenv("PTL_TOOL_BASE_PATH", "refs/remotes/upstream/")
+GITDIR = os.getenv("PTL_TOOL_GITDIR", ".")
+USER = os.getenv("PTL_TOOL_USER", getpass.getuser())
+with open(expanduser("~/.github.key")) as f:
+ PASSWORD = f.read().strip()
+TEST_BRANCH = os.getenv("PTL_TOOL_TEST_BRANCH", "wip-{user}-testing-%Y%m%d.%H%M%S")
+
+SPECIAL_BRANCHES = ('master', 'luminous', 'jewel', 'HEAD')
+
+INDICATIONS = [
+ re.compile("(Reviewed-by: .+ <[\w@.-]+>)", re.IGNORECASE),
+ re.compile("(Acked-by: .+ <[\w@.-]+>)", re.IGNORECASE),
+ re.compile("(Tested-by: .+ <[\w@.-]+>)", re.IGNORECASE),
+]
+
+# find containing git dir
+git_dir = GITDIR
+max_levels = 6
+while not os.path.exists(git_dir + '/.git'):
+ git_dir += '/..'
+ max_levels -= 1
+ if max_levels < 0:
+ break
+
+CONTRIBUTORS = {}
+NEW_CONTRIBUTORS = {}
+with codecs.open(git_dir + "/.githubmap", encoding='utf-8') as f:
+ comment = re.compile("\s*#")
+ patt = re.compile("([\w-]+)\s+(.*)")
+ for line in f:
+ if comment.match(line):
+ continue
+ m = patt.match(line)
+ CONTRIBUTORS[m.group(1)] = m.group(2)
+
+BZ_MATCH = re.compile("(.*https?://bugzilla.redhat.com/.*)")
+TRACKER_MATCH = re.compile("(.*https?://tracker.ceph.com/.*)")
+
+def build_branch(args):
+ base = args.base
+ branch = datetime.datetime.utcnow().strftime(args.branch).format(user=USER)
+ label = args.label
+ merge_branch_name = args.merge_branch_name
+ if merge_branch_name is False:
+ merge_branch_name = branch
+
+ if label:
+ #Check the label format
+ if re.search(r'\bwip-(.*?)-testing\b', label) is None:
+ log.error("Unknown Label '{lblname}'. Label Format: wip-<name>-testing".format(lblname=label))
+ sys.exit(1)
+
+ #Check if the Label exist in the repo
+ res = requests.get("https://api.github.com/repos/{project}/{repo}/labels/{lblname}".format(lblname=label, project=BASE_PROJECT, repo=BASE_REPO), auth=(USER, PASSWORD))
+ if res.status_code != 200:
+ log.error("Label '{lblname}' not found in the repo".format(lblname=label))
+ sys.exit(1)
+
+ G = git.Repo(args.git)
+
+ # First get the latest base branch and PRs from BASE_REMOTE
+ remote = getattr(G.remotes, BASE_REMOTE)
+ remote.fetch()
+
+ prs = args.prs
+ if args.pr_label is not None:
+ if args.pr_label == '' or args.pr_label.isspace():
+ log.error("--pr-label must have a non-space value")
+ sys.exit(1)
+ payload = {'labels': args.pr_label, 'sort': 'created', 'direction': 'desc'}
+ labeled_prs = requests.get("https://api.github.com/repos/{project}/{repo}/issues".format(project=BASE_PROJECT, repo=BASE_REPO), auth=(USER, PASSWORD), params=payload)
+ if labeled_prs.status_code != 200:
+ log.error("Failed to load labeled PRs: {}".format(labeled_prs))
+ sys.exit(1)
+ labeled_prs = labeled_prs.json()
+ if len(labeled_prs) == 0:
+ log.error("Search for PRs matching label '{}' returned no results!".format(args.pr_label))
+ sys.exit(1)
+ for pr in labeled_prs:
+ if pr['pull_request']:
+ n = pr['number']
+ log.info("Adding labeled PR #{} to PR list".format(n))
+ prs.append(n)
+ log.info("Will merge PRs: {}".format(prs))
+
+ if base == 'HEAD':
+ log.info("Branch base is HEAD; not checking out!")
+ else:
+ log.info("Detaching HEAD onto base: {}".format(base))
+ try:
+ base_path = args.base_path + base
+ base = filter(lambda r: r.path == base_path, G.refs)[0]
+ except IndexError:
+ log.error("Branch " + base + " does not exist!")
+ sys.exit(1)
+
+ # So we know that we're not on an old test branch, detach HEAD onto ref:
+ base.checkout()
+
+ for pr in prs:
+ pr = int(pr)
+ log.info("Merging PR #{pr}".format(pr=pr))
+
+ remote_ref = "refs/pull/{pr}/head".format(pr=pr)
+ fi = remote.fetch(remote_ref)
+ if len(fi) != 1:
+ log.error("PR {pr} does not exist?".format(pr=pr))
+ sys.exit(1)
+ tip = fi[0].ref.commit
+
+ pr_req = requests.get("https://api.github.com/repos/ceph/ceph/pulls/{pr}".format(pr=pr), auth=(USER, PASSWORD))
+ if pr_req.status_code != 200:
+ log.error("PR '{pr}' not found: {c}".format(pr=pr,c=pr_req))
+ sys.exit(1)
+
+ message = "Merge PR #%d into %s\n\n* %s:\n" % (pr, merge_branch_name, remote_ref)
+
+ for commit in G.iter_commits(rev="HEAD.."+str(tip)):
+ message = message + ("\t%s\n" % commit.message.split('\n', 1)[0])
+ # Get tracker issues / bzs cited so the PTL can do updates
+ short = commit.hexsha[:8]
+ for m in BZ_MATCH.finditer(commit.message):
+ log.info("[ {sha1} ] BZ cited: {cite}".format(sha1=short, cite=m.group(1)))
+ for m in TRACKER_MATCH.finditer(commit.message):
+ log.info("[ {sha1} ] Ceph tracker cited: {cite}".format(sha1=short, cite=m.group(1)))
+
+ message = message + "\n"
+
+ comments = requests.get("https://api.github.com/repos/{project}/{repo}/issues/{pr}/comments".format(pr=pr, project=BASE_PROJECT, repo=BASE_REPO), auth=(USER, PASSWORD))
+ if comments.status_code != 200:
+ log.error("PR '{pr}' not found: {c}".format(pr=pr,c=comments))
+ sys.exit(1)
+
+ reviews = requests.get("https://api.github.com/repos/{project}/{repo}/pulls/{pr}/reviews".format(pr=pr, project=BASE_PROJECT, repo=BASE_REPO), auth=(USER, PASSWORD))
+ if reviews.status_code != 200:
+ log.error("PR '{pr}' not found: {c}".format(pr=pr,c=comments))
+ sys.exit(1)
+
+ review_comments = requests.get("https://api.github.com/repos/{project}/{repo}/pulls/{pr}/comments".format(pr=pr, project=BASE_PROJECT, repo=BASE_REPO), auth=(USER, PASSWORD))
+ if review_comments.status_code != 200:
+ log.error("PR '{pr}' not found: {c}".format(pr=pr,c=comments))
+ sys.exit(1)
+
+ indications = set()
+ for comment in [pr_req.json()]+comments.json()+reviews.json()+review_comments.json():
+ body = comment["body"]
+ if body:
+ url = comment["html_url"]
+ for m in BZ_MATCH.finditer(body):
+ log.info("[ {url} ] BZ cited: {cite}".format(url=url, cite=m.group(1)))
+ for m in TRACKER_MATCH.finditer(body):
+ log.info("[ {url} ] Ceph tracker cited: {cite}".format(url=url, cite=m.group(1)))
+ for indication in INDICATIONS:
+ for cap in indication.findall(comment["body"]):
+ indications.add(cap)
+
+ new_new_contributors = {}
+ for review in reviews.json():
+ if review["state"] == "APPROVED":
+ user = review["user"]["login"]
+ try:
+ indications.add("Reviewed-by: "+CONTRIBUTORS[user])
+ except KeyError as e:
+ try:
+ indications.add("Reviewed-by: "+NEW_CONTRIBUTORS[user])
+ except KeyError as e:
+ try:
+ name = raw_input("Need name for contributor \"%s\" (use ^D to skip); Reviewed-by: " % user)
+ name = name.strip()
+ if len(name) == 0:
+ continue
+ NEW_CONTRIBUTORS[user] = name
+ new_new_contributors[user] = name
+ indications.add("Reviewed-by: "+name)
+ except EOFError as e:
+ continue
+
+ for indication in indications:
+ message = message + indication + "\n"
+
+ G.git.merge(tip.hexsha, '--no-ff', m=message)
+
+ if new_new_contributors:
+ # Check out the PR, add a commit adding to .githubmap
+ log.info("adding new contributors to githubmap in merge commit")
+ with open(git_dir + "/.githubmap", "a") as f:
+ for c in new_new_contributors:
+ f.write("%s %s\n" % (c, new_new_contributors[c]))
+ G.index.add([".githubmap"])
+ G.git.commit("--amend", "--no-edit")
+
+ if label:
+ req = requests.post("https://api.github.com/repos/{project}/{repo}/issues/{pr}/labels".format(pr=pr, project=BASE_PROJECT, repo=BASE_REPO), data=json.dumps([label]), auth=(USER, PASSWORD))
+ if req.status_code != 200:
+ log.error("PR #%d could not be labeled %s: %s" % (pr, label, req))
+ sys.exit(1)
+ log.info("Labeled PR #{pr} {label}".format(pr=pr, label=label))
+
+ # If the branch is 'HEAD', leave HEAD detached (but use "master" for commit message)
+ if branch == 'HEAD':
+ log.info("Leaving HEAD detached; no branch anchors your commits")
+ else:
+ created_branch = False
+ try:
+ G.head.reference = G.create_head(branch)
+ log.info("Checked out new branch {branch}".format(branch=branch))
+ created_branch = True
+ except:
+ G.head.reference = G.create_head(branch, force=True)
+ log.info("Checked out branch {branch}".format(branch=branch))
+
+ if created_branch:
+ # tag it for future reference.
+ tag = "testing/%s" % branch
+ git.refs.tag.Tag.create(G, tag)
+ log.info("Created tag %s" % tag)
+
+def main():
+ parser = argparse.ArgumentParser(description="Ceph PTL tool")
+ default_base = 'master'
+ default_branch = TEST_BRANCH
+ default_label = ''
+ if len(sys.argv) > 1 and sys.argv[1] in SPECIAL_BRANCHES:
+ argv = sys.argv[2:]
+ default_branch = 'HEAD' # Leave HEAD detached
+ default_base = default_branch
+ default_label = False
+ else:
+ argv = sys.argv[1:]
+ parser.add_argument('--branch', dest='branch', action='store', default=default_branch, help='branch to create ("HEAD" leaves HEAD detached; i.e. no branch is made)')
+ parser.add_argument('--merge-branch-name', dest='merge_branch_name', action='store', default=False, help='name of the branch for merge messages')
+ parser.add_argument('--base', dest='base', action='store', default=default_base, help='base for branch')
+ parser.add_argument('--base-path', dest='base_path', action='store', default=BASE_PATH, help='base for branch')
+ parser.add_argument('--git-dir', dest='git', action='store', default=git_dir, help='git directory')
+ parser.add_argument('--label', dest='label', action='store', default=default_label, help='label PRs for testing')
+ parser.add_argument('--pr-label', dest='pr_label', action='store', help='label PRs for testing')
+ parser.add_argument('prs', metavar="PR", type=int, nargs='*', help='Pull Requests to merge')
+ args = parser.parse_args(argv)
+ return build_branch(args)
+
+if __name__ == "__main__":
+ main()