Adding upstream version 14.2.21.upstream/14.2.21upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 368 insertions, 0 deletions

diff --git a/src/script/ptl-tool.py b/src/script/ptl-tool.py
new file mode 100755
index 00000000..6d374674
--- /dev/null
+++ b/src/script/ptl-tool.py
@@ -0,0 +1,368 @@
+#!/usr/bin/env python2
+
+# README:
+#
+# This tool's purpose is to make it easier to merge PRs into test branches and
+# into master. Make sure you generate a Personal access token in GitHub and
+# add it your ~/.github.key.
+#
+# Because developers often have custom names for the ceph upstream remote
+# (https://github.com/ceph/ceph.git), You will probably want to export the
+# PTL_TOOL_BASE_PATH environment variable in your shell rc files before using
+# this script:
+#
+#     export PTL_TOOL_BASE_PATH=refs/remotes/<remotename>/
+#
+# and PTL_TOOL_BASE_REMOTE as the name of your Ceph upstream remote (default: "upstream"):
+#
+#     export PTL_TOOL_BASE_REMOTE=<remotename>
+#
+#
+# ** Here are some basic exmples to get started: **
+#
+# Merging all PRs labeled 'wip-pdonnell-testing' into a new test branch:
+#
+# $ src/script/ptl-tool.py --pr-label wip-pdonnell-testing
+# Adding labeled PR #18805 to PR list
+# Adding labeled PR #18774 to PR list
+# Adding labeled PR #18600 to PR list
+# Will merge PRs: [18805, 18774, 18600]
+# Detaching HEAD onto base: master
+# Merging PR #18805
+# Merging PR #18774
+# Merging PR #18600
+# Checked out new branch wip-pdonnell-testing-20171108.054517
+# Created tag testing/wip-pdonnell-testing-20171108.054517
+#
+#
+# Merging all PRs labeled 'wip-pdonnell-testing' into master:
+#
+# $ src/script/ptl-tool.py --pr-label wip-pdonnell-testing --branch master
+# Adding labeled PR #18805 to PR list
+# Adding labeled PR #18774 to PR list
+# Adding labeled PR #18600 to PR list
+# Will merge PRs: [18805, 18774, 18600]
+# Detaching HEAD onto base: master
+# Merging PR #18805
+# Merging PR #18774
+# Merging PR #18600
+# Checked out branch master
+#
+# Now push to master:
+# $ git push upstream master
+# ...
+#
+#
+# Merging PR #1234567 and #2345678 into a new test branch with a testing label added to the PR:
+#
+# $ src/script/ptl-tool.py 1234567 2345678 --label wip-pdonnell-testing
+# Detaching HEAD onto base: master
+# Merging PR #1234567
+# Labeled PR #1234567 wip-pdonnell-testing
+# Merging PR #2345678
+# Labeled PR #2345678 wip-pdonnell-testing
+# Deleted old test branch wip-pdonnell-testing-20170928
+# Created branch wip-pdonnell-testing-20170928
+# Created tag testing/wip-pdonnell-testing-20170928_03
+#
+#
+# Merging PR #1234567 into master leaving a detached HEAD (i.e. do not update your repo's master branch) and do not label:
+#
+# $ src/script/ptl-tool.py --branch HEAD --merge-branch-name master 1234567
+# Detaching HEAD onto base: master
+# Merging PR #1234567
+# Leaving HEAD detached; no branch anchors your commits
+#
+# Now push to master:
+# $ git push upstream HEAD:master
+#
+#
+# Merging PR #12345678 into luminous leaving a detached HEAD (i.e. do not update your repo's master branch) and do not label:
+#
+# $ src/script/ptl-tool.py --base luminous --branch HEAD --merge-branch-name luminous 12345678
+# Detaching HEAD onto base: luminous
+# Merging PR #12345678
+# Leaving HEAD detached; no branch anchors your commits
+#
+# Now push to luminous:
+# $ git push upstream HEAD:luminous
+#
+#
+# Merging all PRs labelled 'wip-pdonnell-testing' into master leaving a detached HEAD:
+#
+# $ src/script/ptl-tool.py --base master --branch HEAD --merge-branch-name master --pr-label wip-pdonnell-testing
+# Adding labeled PR #18192 to PR list
+# Will merge PRs: [18192]
+# Detaching HEAD onto base: master
+# Merging PR #18192
+# Leaving HEAD detached; no branch anchors your commit
+
+
+# TODO
+# Look for check failures?
+# redmine issue update: http://www.redmine.org/projects/redmine/wiki/Rest_Issues
+
+import argparse
+import codecs
+import datetime
+import getpass
+import git
+import itertools
+import json
+import logging
+import os
+import re
+import requests
+import sys
+
+from os.path import expanduser
+
+log = logging.getLogger(__name__)
+log.addHandler(logging.StreamHandler())
+log.setLevel(logging.INFO)
+
+BASE_PROJECT = os.getenv("PTL_TOOL_BASE_PROJECT", "ceph")
+BASE_REPO = os.getenv("PTL_TOOL_BASE_REPO", "ceph")
+BASE_REMOTE = os.getenv("PTL_TOOL_BASE_REMOTE", "upstream")
+BASE_PATH = os.getenv("PTL_TOOL_BASE_PATH", "refs/remotes/upstream/")
+GITDIR = os.getenv("PTL_TOOL_GITDIR", ".")
+USER = os.getenv("PTL_TOOL_USER", getpass.getuser())
+with open(expanduser("~/.github.key")) as f:
+    PASSWORD = f.read().strip()
+TEST_BRANCH = os.getenv("PTL_TOOL_TEST_BRANCH", "wip-{user}-testing-%Y%m%d.%H%M%S")
+
+SPECIAL_BRANCHES = ('master', 'luminous', 'jewel', 'HEAD')
+
+INDICATIONS = [
+    re.compile("(Reviewed-by: .+ <[\w@.-]+>)", re.IGNORECASE),
+    re.compile("(Acked-by: .+ <[\w@.-]+>)", re.IGNORECASE),
+    re.compile("(Tested-by: .+ <[\w@.-]+>)", re.IGNORECASE),
+]
+
+# find containing git dir
+git_dir = GITDIR
+max_levels = 6
+while not os.path.exists(git_dir + '/.git'):
+    git_dir += '/..'
+    max_levels -= 1
+    if max_levels < 0:
+        break
+
+CONTRIBUTORS = {}
+NEW_CONTRIBUTORS = {}
+with codecs.open(git_dir + "/.githubmap", encoding='utf-8') as f:
+    comment = re.compile("\s*#")
+    patt = re.compile("([\w-]+)\s+(.*)")
+    for line in f:
+        if comment.match(line):
+            continue
+        m = patt.match(line)
+        CONTRIBUTORS[m.group(1)] = m.group(2)
+
+BZ_MATCH = re.compile("(.*https?://bugzilla.redhat.com/.*)")
+TRACKER_MATCH = re.compile("(.*https?://tracker.ceph.com/.*)")
+
+def build_branch(args):
+    base = args.base
+    branch = datetime.datetime.utcnow().strftime(args.branch).format(user=USER)
+    label = args.label
+    merge_branch_name = args.merge_branch_name
+    if merge_branch_name is False:
+        merge_branch_name = branch
+
+    if label:
+        #Check the label format
+        if re.search(r'\bwip-(.*?)-testing\b', label) is None:
+            log.error("Unknown Label '{lblname}'. Label Format: wip-<name>-testing".format(lblname=label))
+            sys.exit(1)
+
+        #Check if the Label exist in the repo
+        res = requests.get("https://api.github.com/repos/{project}/{repo}/labels/{lblname}".format(lblname=label, project=BASE_PROJECT, repo=BASE_REPO), auth=(USER, PASSWORD))
+        if res.status_code != 200:
+            log.error("Label '{lblname}' not found in the repo".format(lblname=label))
+            sys.exit(1)
+
+    G = git.Repo(args.git)
+
+    # First get the latest base branch and PRs from BASE_REMOTE
+    remote = getattr(G.remotes, BASE_REMOTE)
+    remote.fetch()
+
+    prs = args.prs
+    if args.pr_label is not None:
+        if args.pr_label == '' or args.pr_label.isspace():
+            log.error("--pr-label must have a non-space value")
+            sys.exit(1)
+        payload = {'labels': args.pr_label, 'sort': 'created', 'direction': 'desc'}
+        labeled_prs = requests.get("https://api.github.com/repos/{project}/{repo}/issues".format(project=BASE_PROJECT, repo=BASE_REPO), auth=(USER, PASSWORD), params=payload)
+        if labeled_prs.status_code != 200:
+            log.error("Failed to load labeled PRs: {}".format(labeled_prs))
+            sys.exit(1)
+        labeled_prs = labeled_prs.json()
+        if len(labeled_prs) == 0:
+            log.error("Search for PRs matching label '{}' returned no results!".format(args.pr_label))
+            sys.exit(1)
+        for pr in labeled_prs:
+            if pr['pull_request']:
+                n = pr['number']
+                log.info("Adding labeled PR #{} to PR list".format(n))
+                prs.append(n)
+    log.info("Will merge PRs: {}".format(prs))
+
+    if base == 'HEAD':
+        log.info("Branch base is HEAD; not checking out!")
+    else:
+        log.info("Detaching HEAD onto base: {}".format(base))
+        try:
+            base_path = args.base_path + base
+            base = filter(lambda r: r.path == base_path, G.refs)[0]
+        except IndexError:
+            log.error("Branch " + base + " does not exist!")
+            sys.exit(1)
+
+        # So we know that we're not on an old test branch, detach HEAD onto ref:
+        base.checkout()
+
+    for pr in prs:
+        pr = int(pr)
+        log.info("Merging PR #{pr}".format(pr=pr))
+
+        remote_ref = "refs/pull/{pr}/head".format(pr=pr)
+        fi = remote.fetch(remote_ref)
+        if len(fi) != 1:
+            log.error("PR {pr} does not exist?".format(pr=pr))
+            sys.exit(1)
+        tip = fi[0].ref.commit
+
+        pr_req = requests.get("https://api.github.com/repos/ceph/ceph/pulls/{pr}".format(pr=pr), auth=(USER, PASSWORD))
+        if pr_req.status_code != 200:
+            log.error("PR '{pr}' not found: {c}".format(pr=pr,c=pr_req))
+            sys.exit(1)
+
+        message = "Merge PR #%d into %s\n\n* %s:\n" % (pr, merge_branch_name, remote_ref)
+
+        for commit in G.iter_commits(rev="HEAD.."+str(tip)):
+            message = message + ("\t%s\n" % commit.message.split('\n', 1)[0])
+            # Get tracker issues / bzs cited so the PTL can do updates
+            short = commit.hexsha[:8]
+            for m in BZ_MATCH.finditer(commit.message):
+                log.info("[ {sha1} ] BZ cited: {cite}".format(sha1=short, cite=m.group(1)))
+            for m in TRACKER_MATCH.finditer(commit.message):
+                log.info("[ {sha1} ] Ceph tracker cited: {cite}".format(sha1=short, cite=m.group(1)))
+
+        message = message + "\n"
+
+        comments = requests.get("https://api.github.com/repos/{project}/{repo}/issues/{pr}/comments".format(pr=pr, project=BASE_PROJECT, repo=BASE_REPO), auth=(USER, PASSWORD))
+        if comments.status_code != 200:
+            log.error("PR '{pr}' not found: {c}".format(pr=pr,c=comments))
+            sys.exit(1)
+
+        reviews = requests.get("https://api.github.com/repos/{project}/{repo}/pulls/{pr}/reviews".format(pr=pr, project=BASE_PROJECT, repo=BASE_REPO), auth=(USER, PASSWORD))
+        if reviews.status_code != 200:
+            log.error("PR '{pr}' not found: {c}".format(pr=pr,c=comments))
+            sys.exit(1)
+
+        review_comments = requests.get("https://api.github.com/repos/{project}/{repo}/pulls/{pr}/comments".format(pr=pr, project=BASE_PROJECT, repo=BASE_REPO), auth=(USER, PASSWORD))
+        if review_comments.status_code != 200:
+            log.error("PR '{pr}' not found: {c}".format(pr=pr,c=comments))
+            sys.exit(1)
+
+        indications = set()
+        for comment in [pr_req.json()]+comments.json()+reviews.json()+review_comments.json():
+            body = comment["body"]
+            if body:
+                url = comment["html_url"]
+                for m in BZ_MATCH.finditer(body):
+                    log.info("[ {url} ] BZ cited: {cite}".format(url=url, cite=m.group(1)))
+                for m in TRACKER_MATCH.finditer(body):
+                    log.info("[ {url} ] Ceph tracker cited: {cite}".format(url=url, cite=m.group(1)))
+                for indication in INDICATIONS:
+                    for cap in indication.findall(comment["body"]):
+                        indications.add(cap)
+
+        new_new_contributors = {}
+        for review in reviews.json():
+            if review["state"] == "APPROVED":
+                user = review["user"]["login"]
+                try:
+                    indications.add("Reviewed-by: "+CONTRIBUTORS[user])
+                except KeyError as e:
+                    try:
+                        indications.add("Reviewed-by: "+NEW_CONTRIBUTORS[user])
+                    except KeyError as e:
+                        try:
+                            name = raw_input("Need name for contributor \"%s\" (use ^D to skip); Reviewed-by: " % user)
+                            name = name.strip()
+                            if len(name) == 0:
+                                continue
+                            NEW_CONTRIBUTORS[user] = name
+                            new_new_contributors[user] = name
+                            indications.add("Reviewed-by: "+name)
+                        except EOFError as e:
+                            continue
+
+        for indication in indications:
+            message = message + indication + "\n"
+
+        G.git.merge(tip.hexsha, '--no-ff', m=message)
+
+        if new_new_contributors:
+            # Check out the PR, add a commit adding to .githubmap
+            log.info("adding new contributors to githubmap in merge commit")
+            with open(git_dir + "/.githubmap", "a") as f:
+                for c in new_new_contributors:
+                    f.write("%s %s\n" % (c, new_new_contributors[c]))
+            G.index.add([".githubmap"])
+            G.git.commit("--amend", "--no-edit")
+
+        if label:
+            req = requests.post("https://api.github.com/repos/{project}/{repo}/issues/{pr}/labels".format(pr=pr, project=BASE_PROJECT, repo=BASE_REPO), data=json.dumps([label]), auth=(USER, PASSWORD))
+            if req.status_code != 200:
+                log.error("PR #%d could not be labeled %s: %s" % (pr, label, req))
+                sys.exit(1)
+            log.info("Labeled PR #{pr} {label}".format(pr=pr, label=label))
+
+    # If the branch is 'HEAD', leave HEAD detached (but use "master" for commit message)
+    if branch == 'HEAD':
+        log.info("Leaving HEAD detached; no branch anchors your commits")
+    else:
+        created_branch = False
+        try:
+            G.head.reference = G.create_head(branch)
+            log.info("Checked out new branch {branch}".format(branch=branch))
+            created_branch = True
+        except:
+            G.head.reference = G.create_head(branch, force=True)
+            log.info("Checked out branch {branch}".format(branch=branch))
+
+        if created_branch:
+            # tag it for future reference.
+            tag = "testing/%s" % branch
+            git.refs.tag.Tag.create(G, tag)
+            log.info("Created tag %s" % tag)
+
+def main():
+    parser = argparse.ArgumentParser(description="Ceph PTL tool")
+    default_base = 'master'
+    default_branch = TEST_BRANCH
+    default_label = ''
+    if len(sys.argv) > 1 and sys.argv[1] in SPECIAL_BRANCHES:
+        argv = sys.argv[2:]
+        default_branch = 'HEAD' # Leave HEAD detached
+        default_base = default_branch
+        default_label = False
+    else:
+        argv = sys.argv[1:]
+    parser.add_argument('--branch', dest='branch', action='store', default=default_branch, help='branch to create ("HEAD" leaves HEAD detached; i.e. no branch is made)')
+    parser.add_argument('--merge-branch-name', dest='merge_branch_name', action='store', default=False, help='name of the branch for merge messages')
+    parser.add_argument('--base', dest='base', action='store', default=default_base, help='base for branch')
+    parser.add_argument('--base-path', dest='base_path', action='store', default=BASE_PATH, help='base for branch')
+    parser.add_argument('--git-dir', dest='git', action='store', default=git_dir, help='git directory')
+    parser.add_argument('--label', dest='label', action='store', default=default_label, help='label PRs for testing')
+    parser.add_argument('--pr-label', dest='pr_label', action='store', help='label PRs for testing')
+    parser.add_argument('prs', metavar="PR", type=int, nargs='*', help='Pull Requests to merge')
+    args = parser.parse_args(argv)
+    return build_branch(args)
+
+if __name__ == "__main__":
+    main()