diff options
Diffstat (limited to '')
| -rwxr-xr-x | scripts/process-rt | 667 |
1 files changed, 667 insertions, 0 deletions
diff --git a/scripts/process-rt b/scripts/process-rt new file mode 100755 index 0000000..87d5285 --- /dev/null +++ b/scripts/process-rt @@ -0,0 +1,667 @@ +#!/usr/bin/python3 + +# Copyright (c) 2017-2018 Jonathan McDowell <noodles@earth.li> +# GNU GPL; v2 or later +# +# Process RT tickets for keyring-maint@debian as raised by the +# nm.debian.org web interface + +# Semi-helpful gpgme/Python examples: +# https://pypkg.com/pypi/gpg/f/examples/ + +import datetime +import gpg +import io +import os +import pprint +import re +import requests +import subprocess +import sys +from urllib.parse import urlencode + +debug = False +RT_BASE_URL = 'https://rt.debian.org/REST/1.0/' +KEYSERVERS = ['keyserver.ubuntu.com:11371', 'the.earth.li:11371', + 'pool.sks-keyservers.net:11371' ] +DAM = ['enrico', 'joerg', 'jmw'] +FD = DAM + ['noodles', 'mattia', 'peb', 'santiago', 'tobi', 'hartmans', + 'stuart', 'olasd'] + +SIG_DAYS_WARN = 45 +DATE_FORMAT = "%d/%m/%Y %H:%M:%S" + +# Try to find the keyring base directory, assuming we live in /scripts/ +basedir = os.path.dirname(os.path.dirname(os.path.realpath(sys.argv[0]))) + '/' +if os.path.exists(basedir + 'debian-keyring-gpg'): + KEYRING_BASE_DIR = basedir +else: + print("Can't find keyring directory.") + sys.exit(-1) +GNUPG_HOME = KEYRING_BASE_DIR + 'gpghome/' + +# Maps roles to keyring directories +role2keyring = { + 'DM': 'debian-maintainers-gpg', + 'DD': 'debian-keyring-gpg', + 'DN': 'debian-nonupload-gpg', + 'emeritus': 'emeritus-keyring-gpg', +} + +# The keys must match nm2:backend/const.py:ALL_STATUS +desc2role = { + 'Debian Developer, uploading': 'DD', + 'Debian Developer, non-uploading': 'DN', + 'Debian Maintainer': 'DM', + 'Debian Maintainer, with guest account': 'DM', + 'Debian Contributor': 'DC', + 'Debian Contributor, with guest account': 'DC', + 'Debian Developer, emeritus': 'emeritus', + 'Debian Developer, removed': 'removed', +} + +fp_regex = ("[0-9A-F]{4} [0-9A-F]{4} [0-9A-F]{4} [0-9A-F]{4} [0-9A-F]{4} " + + "[0-9A-F]{4} [0-9A-F]{4} [0-9A-F]{4} [0-9A-F]{4} [0-9A-F]{4}") +fp_regex_nospc = "[0-9A-F]{40}" + +# Global keyid to username/name dict +keyids = {} + + +def get_gpg_ctx(do_import=False): + """ Setup (if necessary) and return a GnuPG context + + Checks if the private GnuPG home directory already exists. If not, + creates it and imports the DD + DN keyrings into it. Also configures + GnuPG to do clean imports (i.e. only include signatures that can be + verified). + """ + if not os.path.isdir(GNUPG_HOME): + os.makedirs(GNUPG_HOME) + do_import = True + + c = gpg.Context() + + c.set_engine_info(gpg.constants.protocol.OpenPGP, + home_dir=GNUPG_HOME) + + if do_import: + for keyring in ['debian-keyring', 'debian-nonupload']: + keyfile = KEYRING_BASE_DIR + 'output/keyrings/' + keyring + '.gpg' + if not os.path.exists(keyfile): + raise RuntimeError(keyfile + " does not exist. " + + "Need to run 'make'?") + keys = gpg.Data(file=keyfile) + c.op_import(keys) + + with open(GNUPG_HOME + 'gpg.conf', 'w') as f: + f.write('import-options import-clean\n') + + return c + + +def fetch_key(ctx, fpr): + """Fetches the supplied fingerprint from a public keyserver + + Does an HKP lookup for the supplied fingerprint, then imports it into + the current keyring. Then does an export (picking up any cleaning done + by the import) and returns the binary key data. + + Note this function does not remove the key from the GnuPG keyring. If + the key is destined for the DM keyring, or subsequently not to be added, + it must be removed by the caller using delete_key(). + """ + gotkey = False + for keyserver in KEYSERVERS: + url = "http://{server}/pks/lookup?{query}".format( + server=keyserver, + query=urlencode({ + "op": "get", + "search": "0x" + fpr, + "exact": "on", + })) + res = requests.get(url) + keytext = [] + for line in res.text.splitlines(): + if line == "-----BEGIN PGP PUBLIC KEY BLOCK-----": + gotkey = True + if gotkey: + keytext.append(line) + if line == "-----END PGP PUBLIC KEY BLOCK-----": + break + if gotkey: + break + + if not gotkey: + raise RuntimeError('Failed to fetch key') + + key = gpg.Data(string="\n".join(keytext)) + ctx.op_import(key) + + key = gpg.Data() + ctx.op_export(fpr, 0, key) + key.seek(0, os.SEEK_SET) + keydata = key.read() + + return keydata + + +def delete_key(ctx, fpr): + """Delete the key matching fpr from the GnuPG keyring""" + keys = list(ctx.keylist(fpr)) + for k in keys: + ctx.op_delete(k, True) + + +def get_keyinfo(ctx, fpr, needsigs=2): + ctx.set_keylist_mode(gpg.constants.keylist.mode.SIGS) + key = ctx.get_key(fpr) + for subkey in key.subkeys: + if subkey.fpr == fpr: + keytype = str(subkey.length) + if subkey.pubkey_algo == gpg.constants.pk.RSA: + keytype += 'R' + elif subkey.pubkey_algo == gpg.constants.pk.DSA: + keytype += 'D' + elif subkey.pubkey_algo == gpg.constants.pk.ECC: + keytype += 'E' + elif subkey.pubkey_algo == gpg.constants.pk.EDDSA: + keytype += 'E' + elif subkey.pubkey_algo == gpg.constants.pk.ELG: + keytype += 'g' + else: + keytype += '?' + + sigs = {} + for uid in key.uids: + # print(uid.name, uid.email) + if not uid.revoked: + for sig in uid.signatures: + if sig.keyid in keyids: + sigs[keyids[sig.keyid]['username']] = 1 + # else: + # print("Skipping unknown ID " + sig.keyid) + + if len(sigs) < needsigs: + print('\nWARNING: Insufficient key signatures, check endorsements\n') + + certs = None + for sig in sorted(sigs.keys()): + if certs: + certs += ', ' + sig + else: + certs = sig + + return (keytype, certs) + + +def read_keyids(): + """Read the keyids file into a dict allowing a username/name mapping""" + with open(KEYRING_BASE_DIR + 'keyids', 'r') as f: + dds = f.readlines() + for dd in dds: + keyid = dd[2:18] + name = dd[19:dd.find('<') - 1] + username = dd[dd.find('<') + 1:] + username = username[:username.find('>')] + keyids[keyid] = { + 'name': name, + 'username': username, + } + + +def write_keyids(): + """Write the sorted keyids username/name dict out to the keyids file""" + with open(KEYRING_BASE_DIR + 'keyids', 'w') as f: + for key in sorted(keyids.keys()): + f.write("0x{} {} <{}>\n".format(key, + keyids[key]['name'], + keyids[key]['username'])) + + +def get_rt_auth(): + """Attempt to locate a valid set of RT login details + + Look for, and return, a set of RT login details. Uses RT_USER/RT_PASS + from the environment, failing back to ~/.rtrc if either is not set. + """ + rt_user = None + rt_pass = None + + if 'RT_USER' in os.environ: + rt_user = os.environ['RT_USER'] + if 'RT_PASS' in os.environ: + rt_pass = os.environ['RT_PASS'] + + if not rt_user or not rt_pass: + with open(os.environ['HOME'] + '/.rtrc', 'r') as f: + for line in f: + if not rt_user and line.startswith('user '): + rt_user = line[5:].strip() + elif not rt_pass and line.startswith('passwd '): + rt_pass = line[7:].strip() + + return (rt_user, rt_pass) + + +def fetch_ticket(rtid): + (rt_user, rt_pass) = get_rt_auth() + + args = { + 'params': { + 'user': rt_user, + 'pass': rt_pass, + } + } + + print("Fetching ticket " + str(rtid)) + + # res = requests.post(RT_BASE_URL + 'ticket/' + str(rtid) + '/show', + # **args) + # Look for "Owner: Nobody" or "Owner: noodles" + # res = requests.post(RT_BASE_URL + 'ticket/' + str(rtid) + '/edit', + # **args) + # "content" variable = "Owner: noodles" + + res = requests.post(RT_BASE_URL + 'ticket/' + str(rtid) + '/attachments', + **args) + + # Validate the RT result + res_lines = res.text.splitlines() + ver, status, text = res_lines[0].split(None, 2) + + if int(status) != 200: + print("RT status code is not 200", res_lines) + return + + attachments = [] + inattachments = False + text = None + signature = None + + for line in res_lines[2:]: + if inattachments: + m = re.match(' +(\d+):', line) + if m: + print('Attachment found, ' + m.group(1)) + attachments.append(int(m.group(1))) + else: + inattachments = False + else: + m = re.match('Attachments: (\d+):', line) + if m: + print('Attachment found, ' + m.group(1)) + attachments.append(int(m.group(1))) + inattachments = True + + for attachment in attachments: + res = requests.post(RT_BASE_URL + 'ticket/' + str(rtid) + + '/attachments/' + str(attachment), + **args) + + # Validate the RT result + res_lines = res.text.splitlines() + ver, status, text = res_lines[0].split(None, 2) + + if int(status) != 200: + print("RT status code is not 200", res_lines) + + incontent = False + message = [] + for line in res_lines[2:]: + if line.startswith('Content: '): + incontent = True + message.append(line[9:]) + elif incontent and line.startswith(' '): + message.append(line[9:]) + elif incontent: + incontent = False + + with get_gpg_ctx() as c: + sig = None + try: + text, result = c.verify(io.BytesIO( + "\n".join(message).encode('utf-8'))) + sig = result.signatures[0] + except gpg.errors.GPGMEError: + text = None + continue + except gpg.errors.BadSignatures as e: + # If a request is signed by multiple keys (such as old + new + # for a replacement) the important thing is one of the + # signatures is valid. + for s in e.results[1].signatures: + if s.status == 0: + text = e.results[0] + sig = s + break + else: + print("Bad signature from", s.fpr) + if sig: + key = c.get_key(sig.fpr) + # print(result.signatures[0].__str__()) + for subkey in key.subkeys: + if subkey.fpr[24:] in keyids: + signature = keyids[subkey.fpr[24:]]['username'] + print("Good signature from " + signature + " (" + + sig.fpr + ")") + + sig_date = datetime.datetime.fromtimestamp(sig.timestamp) + now = datetime.datetime.now() + if (now - sig_date) >= datetime.timedelta(days=SIG_DAYS_WARN): + print(( + "WARNING: Old GPG signature ({formatted_sig_date}, " + "today is {formatted_now}), please verify validity." + ).format( + formatted_sig_date=sig_date.strftime(DATE_FORMAT), + formatted_now=now.strftime(DATE_FORMAT), + )) + break + else: + print("Couldn't verify message.") + + return (signature, text) + + +def parse_ticket(text): + state = {} + + for line in text.decode().split('\n'): + if line.startswith(' Key fingerprint: '): + if line[20:] != 'None': + state['keyid'] = line[20:] + elif line.startswith(' Username: '): + state['username'] = line[20:] + elif line.startswith(' uid: '): + state['username'] = line[20:] + elif line.startswith(' Details: '): + state['details'] = line[20:] + elif line.startswith(' First name: '): + state['first'] = line[20:] + elif line.startswith(' Middle name: '): + state['middle'] = line[20:] + elif line.startswith(' Last name: '): + state['last'] = line[20:] + elif line.startswith(' Current status: '): + if line[20:] not in desc2role: + print('Unknown current status: ' + line[20:]) + else: + state['current'] = desc2role[line[20:]] + elif line.startswith(' Target keyring: '): + if line[20:] not in desc2role: + print('Unknown destination status: ' + line[20:]) + else: + state['dest'] = desc2role[line[20:]] + elif 'details' not in state: + # Try to see if we have a fingerprint on the line and if + # it might be a replacement request + m = re.search(fp_regex, line) + if not m: + m = re.search(fp_regex_nospc, line) + if m: + fp = m.group(0).replace(' ', '') + if fp[24:] in keyids: + state['oldkeyid'] = fp + else: + state['keyid'] = fp + + + # Based on the current + target statuses work out if this is an add or + # remove. + if 'dest' not in state: + # Assume it's a replacement. + for role in role2keyring: + if os.path.exists(role2keyring[role] + '/0x' + + state['oldkeyid'][24:]): + state['role'] = role + break + if 'role' not in state: + state['role'] = 'DD' + state['action'] = 'replace' + state['subject'] = keyids[state['oldkeyid'][24:]]['name'] + state['username'] = keyids[state['oldkeyid'][24:]]['username'] + elif state['dest'] in ['DD', 'DM', 'DN']: + state['action'] = 'add' + state['role'] = state['dest'] + elif state['dest'] in ['DC', 'emeritus', 'removed']: + state['action'] = 'remove' + if 'current' in state: + state['role'] = state['current'] + else: + # Assume DD -> removed as a fall back + state['role'] = 'DD' + + # Collapse first/middle/last to a single name field + if 'first' in state and state['first'] != '-': + state['subject'] = state['first'] + if 'middle' in state and state['middle'] != '-': + if 'subject' in state: + state['subject'] += ' ' + state['middle'] + else: + state['subject'] = state['middle'] + if 'last' in state and state['last'] != '-': + if 'subject' in state: + state['subject'] += ' ' + state['last'] + else: + state['subject'] = state['last'] + + for field in ['details', 'username']: + if field in state and state[field] == '-': + del state[field] + + # Get the key length + type, plus signatures from other DDs + if 'keyid' in state: + with get_gpg_ctx() as c: + if state['action'] in ('add', 'replace') or state['role'] == 'DM': + state['keydata'] = fetch_key(c, state['keyid']) + # If it's a removal we don't need to check signature count. For + # DM we relax the number of signatures, otherwise we use the + # default. + if state['action'] == 'remove': + keyinfo = get_keyinfo(c, state['keyid'], 0) + elif state['role'] == 'DM': + keyinfo = get_keyinfo(c, state['keyid'], 1) + else: + keyinfo = get_keyinfo(c, state['keyid']) + state['keytype'] = keyinfo[0] + state['certs'] = keyinfo[1] + + if 'oldkeyid' in state: + with get_gpg_ctx() as c: + fetch_key(c, state['oldkeyid']) + keyinfo = get_keyinfo(c, state['oldkeyid'], 0) + state['oldkeytype'] = keyinfo[0] + + return state + + +def do_action(state): + if state['action'] == 'remove': + if 'keyid' not in state: + # When inactive DDs with no valid keys are retired, + # there's nothing for us to do + print('-!- Nothing for keyring-maint to do, please reasign ticket') + print('-!- to DSA for account removal.') + sys.exit(-1) + if 'dest' in state and state['dest'] == 'emeritus': + subprocess.call(['git', 'mv', role2keyring[state['role']] + '/0x' + + state['keyid'][24:], + 'emeritus-keyring-gpg/']) + state['logmsg'] = ('Move 0x' + state['keyid'][24:] + + ' (' + state['subject'] + ') to ' + + 'emeritus (RT #' + state['rtid'] + ')') + else: + subprocess.call(['git', 'rm', role2keyring[state['role']] + '/0x' + + state['keyid'][24:]]) + state['logmsg'] = ('Remove 0x' + state['keyid'][24:] + + ' (' + state['subject'] + ')' + + ' (RT #' + state['rtid'] + ')') + if state['role'] in ['DD', 'DN']: + with get_gpg_ctx() as c: + delete_key(c, state['keyid']) + elif state['action'] == 'add': + state['logmsg'] = ('Add new ' + state['dest'] + ' key 0x' + + state['keyid'][24:] + ' (' + state['subject'] + + ') (RT #' + state['rtid'] + ')') + # See if it's just a move from a different keyring + if state['current'] in role2keyring: + subprocess.call(['git', 'mv', + role2keyring[state['current']] + '/0x' + + state['keyid'][24:], + role2keyring[state['dest']]]) + state['notes'] = 'Move from ' + state['current'] + ' keyring' + else: + keyfile = role2keyring[state['dest']] + '/0x' + state['keyid'][24:] + with open(keyfile, 'wb') as f: + f.write(state['keydata']) + subprocess.call(['git', 'add', keyfile]) + + # We don't keep DM keys in the our working keyring + if state['dest'] == 'DM': + with get_gpg_ctx() as c: + delete_key(c, state['keyid']) + keyids[state['keyid'][24:]] = { + 'name': state['subject'], + 'username': state['username'], + } + write_keyids() + subprocess.call(['git', 'add', 'keyids']) + elif state['action'] == 'replace': + state['logmsg'] = ('Replace 0x' + state['oldkeyid'][24:] + ' with 0x' + + state['keyid'][24:] + ' (' + state['subject'] + + ') (RT #' + state['rtid'] + ')') + + keyfile = role2keyring[state['role']] + '/0x' + state['keyid'][24:] + with open(keyfile, 'wb') as f: + f.write(state['keydata']) + subprocess.call(['git', 'add', keyfile]) + subprocess.call(['git', 'rm', role2keyring[state['role']] + '/0x' + + state['oldkeyid'][24:]]) + + # Remove the replaced key + with get_gpg_ctx() as c: + delete_key(c, state['oldkeyid']) + + keyids[state['keyid'][24:]] = { + 'name': state['subject'], + 'username': state['username'], + } + write_keyids() + subprocess.call(['git', 'add', 'keyids']) + else: + print("Don't know how to handle action: " + state['action']) + + +def do_dch(state): + release = "unknown" + with open('debian/changelog', 'r') as f: + line = f.readline() + m = re.match("debian-keyring \((.*)\) (.*); urgency=", line) + version = m.group(1) + release = m.group(2) + if release == "UNRELEASED": + if debug: + print('dch --multimaint-merge -D UNRELEASED -a "' + + state['logmsg'] + '"') + else: + subprocess.call(['dch', '--multimaint-merge', '-D', 'UNRELEASED', + '-a', state['logmsg']]) + elif release == "unstable": + newver = datetime.date.today().strftime("%Y.%m.xx") + if newver == version: + print(' * Warning: New version and previous released version are ') + print(' the same: ' + newver + '. This should not be so!') + print(' Check debian/changelog') + if debug: + print('dch -D UNRELEASED -v ' + newver + ' "' + state['logmsg'] + + '"') + else: + subprocess.call(['dch', '-D', 'UNRELEASED', '-v', newver, + state['logmsg']]) + else: + print("Unknown changelog release: " + release) + + if not debug: + subprocess.call(['git', 'add', 'debian/changelog']) + + +def do_git_template(state): + with open('git-commit-template', 'w') as f: + f.write(state['logmsg']) + f.write('\n\n') + f.write("Action: " + state['action'] + "\n") + f.write("Subject: " + state['subject'] + "\n") + if 'username' in state: + f.write("Username: " + state['username'] + "\n") + f.write("Role: " + state['role'] + "\n") + if state['action'] == 'replace': + f.write("Old-key: " + state['oldkeyid'] + "\n") + f.write("Old-key-type: " + state['oldkeytype'] + "\n") + f.write("New-key: " + state['keyid'] + "\n") + f.write("New-key-type: " + state['keytype'] + "\n") + else: + f.write("Key: " + state['keyid'] + "\n") + f.write("Key-type: " + state['keytype'] + "\n") + f.write("RT-Ticket: " + state['rtid'] + "\n") + f.write("Request-signed-by: " + state['requester'] + "\n") + if state['action'] != 'remove' and state['certs'] != None: + prefix = 'Key-certified-by: ' + prefixlen = len(prefix) + certs = state['certs'] + while (len(certs) + prefixlen) > 72: + last = certs.rfind(',', 0, 72 - prefixlen) + 1 + f.write(prefix + certs[:last] + "\n") + certs = certs[last:] + prefix = ' ' + prefixlen = 1 + f.write(prefix + certs + "\n") + if 'details' in state: + f.write("Details: " + state['details'] + "\n") + if state['role'] == 'DM' and 'agreement' in state: + f.write("Advocates:\n") + for a in state['advocates']: + f.write(" " + a + "\n") + f.write("Agreement: " + state['agreement'] + "\n") + f.write("BTS: " + state['bts'] + "\n") + if 'notes' in state: + f.write('Notes: ' + state['notes'] + '\n') + + +if len(sys.argv) < 2: + print('Must supply RT ticket to process.') + sys.exit(-1) + +# Change to the keyring dir so that git etc work +os.chdir(KEYRING_BASE_DIR) + +read_keyids() +(requester, ticket) = fetch_ticket(sys.argv[1]) +if ticket is None: + print('No signature on ticket.') + sys.exit(-1) +if requester is None: + print('Signature from unknown key.') + sys.exit(-1) +state = parse_ticket(ticket) +state['rtid'] = sys.argv[1] +state['requester'] = requester +if state['action'] == 'add': + if requester not in FD: + print('Signature for add must come from Front Desk.') + sys.exit(-1) +# Output our state for confirmation, without the keydata +pp = pprint.PrettyPrinter() +s = dict(state) +if 'keydata' in s: + del s['keydata'] +pp.pprint(s) +proceed = input("Do you wish to proceed? [y/n]: ") +if proceed.lower() not in ['y', 'yes']: + print("Aborting.") + # If this was an add we need to make sure the key isn't in our keyring + if state['action'] in ['add', 'replace']: + delete_key(get_gpg_ctx(), state['keyid']) + sys.exit(-1) +do_action(state) +do_dch(state) +do_git_template(state) |