summaryrefslogtreecommitdiffstats
path: root/scripts/update-keyrings
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/update-keyrings')
-rwxr-xr-xscripts/update-keyrings405
1 files changed, 405 insertions, 0 deletions
diff --git a/scripts/update-keyrings b/scripts/update-keyrings
new file mode 100755
index 0000000..8e5ed6f
--- /dev/null
+++ b/scripts/update-keyrings
@@ -0,0 +1,405 @@
+#!/usr/bin/python3
+
+# Authors: Daniel Kahn Gillmor <dkg@fifthhorseman.net>,
+# Gunnar Wolf <gwolf@debian.org>,
+# Jonathan McDowell <noodles@earth.li>
+# License: Parts from dkg are GPLv3+
+
+import os
+from os import path
+import socket
+from subprocess import run, Popen, PIPE
+from shutil import chown, copy, copyfile, rmtree
+from distutils.dir_util import copy_tree
+import sys
+import tempfile
+import hashlib
+import codecs
+from multiprocessing.pool import ThreadPool
+from typing import List, Tuple, Optional
+import datetime
+from email.utils import parseaddr
+
+
+def check_environ(should_run_on: str = 'kaufmann.debian.org') -> None:
+ '''Make sure that we are running where we expect to run
+
+ The expectation is to run on kaufmann.debian.org, but this can be
+ bypassed for testing with the RUNANYWAY environment variable.
+ '''
+ if not (os.environ.get('RUNANYWAY', False) or
+ socket.getfqdn(socket.gethostname()) == should_run_on):
+ raise Exception('''
+This script is meant to be run in %s
+You can still run it if you are sure by setting
+$RUNANYWAY to a nonempty value.
+ ''' % (should_run_on))
+
+
+def wkd_localpart(incoming: bytes) -> str:
+ '''Z-base32 the localpart of an e-mail address
+
+ https://tools.ietf.org/html/draft-koch-openpgp-webkey-service-08#section-3.1
+ describes why this is needed.
+
+ See https://tools.ietf.org/html/rfc6189#section-5.1.6 for a
+ description of the z-base32 scheme.
+ '''
+ zb32 = "ybndrfg8ejkmcpqxot1uwisza345h769"
+
+ b = hashlib.sha1(incoming).digest()
+ ret = ""
+ assert(len(b) * 8 == 160)
+ for i in range(0, 160, 5):
+ byte = i // 8
+ offset = i - byte * 8
+ # offset | bits remaining in k+1 | right-shift k+1
+ # 3 | 0 | x
+ # 4 | 1 | 7
+ # 5 | 2 | 6
+ # 6 | 3 | 5
+ # 7 | 4 | 4
+ if offset < 4:
+ n = (b[byte] >> (3 - offset))
+ else:
+ n = (b[byte] << (offset - 3)) + (b[byte + 1] >> (11 - offset))
+
+ ret += zb32[n & 0b11111]
+ return ret
+
+
+def getdomainlocalpart(line: bytes, domain: bytes) -> Optional[bytes]:
+ 'Get the localpart of the e-mail address of a GnuPG user ID line matching DOMAIN'
+ if line.startswith(b'uid:'):
+ uid = line.split(b':')[9]
+ uid = uid.decode()
+ _name, addr = parseaddr(uid) # if parsing fails, this returns ('', '')
+ localpart, _at, thisdomain = addr.rpartition('@')
+ if thisdomain == domain.decode():
+ return localpart.lower().encode()
+ return None
+
+
+def gpgbase(keyrings: List[str]) -> List[str]:
+ 'Return the standard set of options to invoke gpg in an automated way'
+ return ['gpg', '--batch', '--no-options', '--with-colons',
+ '--no-default-keyring',
+ '--homedir=/dev/null', '--trust-model=always',
+ '--fixed-list-mode'] + list(map(lambda k: '--keyring=' + k, keyrings))
+
+
+def emit_wkd_and_return_dane(localpart: bytes, domain: str, keyrings: List[str]) -> bytes:
+ '''For a given address, emit the WKD file, and return the DANE OPENPGKEY record
+
+ These are handled differently because we want to generate a
+ single, reproducible zonefile for the DNS, while we are generating
+ a tree of files for WKD.
+
+ The caller will assemble all of the OPENPGPKEY records into a
+ single zonefile.
+ '''
+ wkdstr = wkd_localpart(localpart)
+ # what do we do if this local part is not a proper encoding?
+ addr = codecs.decode(localpart) + '@' + domain
+ cmd = gpgbase(keyrings) + ['--output',
+ path.join('openpgpkey', domain, 'hu', wkdstr),
+ '--export-options',
+ 'export-clean',
+ '--export-filter',
+ 'keep-uid=mbox=' + addr,
+ '--export',
+ '<' + addr + '>']
+ run(cmd, check=True)
+ cmd = gpgbase(keyrings) + ['--export-options', 'export-dane,export-clean',
+ '--export-filter', 'keep-uid=mbox=' + addr,
+ '--export', '<' + addr + '>']
+ dane = run(cmd, stdout=PIPE, check=True)
+ return dane.stdout
+
+
+def build_wkd_and_dane(domain: str, keyrings: List[str]) -> None:
+ 'Publish WKD and DANE OPENPGPKEY for all domain-relevant OpenPGP certificates'
+ if not path.isdir('openpgpkey'):
+ os.mkdir('openpgpkey')
+ os.mkdir(path.join('openpgpkey', domain))
+ os.mkdir(path.join('openpgpkey', domain, 'hu'))
+
+ # FIXME: deal with IDN:
+ bytedomain = codecs.encode(domain)
+
+ lister = Popen(gpgbase(keyrings) +
+ ['--list-keys', '@' + domain], stdout=PIPE)
+
+ localparts = set(
+ map(lambda x: getdomainlocalpart(x, bytedomain), lister.stdout))
+ localparts.discard(None)
+
+ dane_map = {}
+
+ def runner(x: bytes) -> Tuple[bytes, bytes]:
+ return (x, emit_wkd_and_return_dane(x, domain, keyrings))
+
+ def add_to_zone(res: Tuple[bytes, bytes]) -> None:
+ dane_map[res[0]] = res[1]
+
+ pool = ThreadPool(None)
+ for localpart in localparts:
+ pool.apply_async(runner, (localpart,), {}, add_to_zone)
+
+ pool.close()
+ pool.join()
+ # make the policy file:
+ policyfile = open(path.join('openpgpkey', domain, 'policy'), 'wb')
+ del policyfile
+ # write out the zonefile all at once, ordered by the localpart
+ with open(path.join('_openpgpkey.' + domain + '.zone'), 'wb') as zonefile:
+ when = datetime.datetime.now()
+ # FIXME: inspect serial number from existing zonefile --
+ # update serial number if it was from the same day
+ serial = 0
+ zonefile.write(openpgpkey_zonefile_header(when, serial))
+ for local in sorted(dane_map.keys()):
+ zonefile.write(dane_map[local])
+
+
+def fix_perms(path: str) -> None:
+ '''Fix the permissions of a given directory
+
+ Ensures all files/directories are owned and writeable by the keyring group.
+ Additionally they must be readable by all and directories executable.
+ '''
+ try:
+ chown(path, group="keyring")
+ os.chmod(path, 0o775)
+ except:
+ pass
+ for root, dirs, files in os.walk(path):
+ for cur in dirs:
+ try:
+ chown(os.path.join(root, cur), group="keyring")
+ except:
+ pass
+ try:
+ os.chmod(os.path.join(root, cur), 0o775)
+ except:
+ pass
+ for cur in files:
+ try:
+ chown(os.path.join(root, cur), group="keyring")
+ except:
+ pass
+ try:
+ os.chmod(os.path.join(root, cur), 0o664)
+ except:
+ pass
+
+
+def publish(srcdir: str,
+ where: str = None) -> None:
+ '''Verify new keyrings in srcdir; if ok, then publish at where.
+
+ Verification ensures that the new keyrings are signed-off by a
+ member of debian's keyring-maint team.
+
+ Publication consists of verifying the keyrings, placing them where
+ onak can find them, produce a zonefile for OPENPGPKEY DANE
+ records, and a tree of files for WKD.
+
+ '''
+ if where is None:
+ prefix = os.environ.get('PREFIX', '/srv/keyring.debian.org')
+ else:
+ prefix = where
+ pendingdir = path.join(prefix, 'pending-updates')
+ hkpdir = path.join(prefix, 'keyrings-new')
+ outputdir = path.join(prefix, 'pub')
+ for direc in [srcdir, pendingdir, hkpdir, outputdir]:
+ if not path.isdir(direc):
+ raise Exception("%s is not a directory" % (direc))
+ srcdir = path.realpath(srcdir)
+ sha512fname = path.join(srcdir, 'sha512sums.txt')
+ if not path.exists(sha512fname):
+ raise Exception('sha512sums.txt not found in %s' % (srcdir))
+ placeholder = path.join(srcdir, 'keyrings', '.placeholder')
+ if path.exists(placeholder):
+ os.unlink(placeholder)
+ # gpgv needs the keyring in the filesystem, not just a file
+ # descriptor (https://dev.gnupg.org/T4608)
+ with tempfile.NamedTemporaryFile() as maint_keyring:
+ maint_keyring.write(keyring_maint_keys())
+ gpgvcall = [
+ 'gpgv',
+ '--enable-special-filenames',
+ '--keyring',
+ maint_keyring.name,
+ '--output',
+ '-',
+ sha512fname]
+ gpgvout = run(gpgvcall, stderr=PIPE, stdout=PIPE)
+ if gpgvout.returncode != 0:
+ raise Exception("gpg verification failed:\n%s" %
+ (codecs.decode(gpgvout.stderr)))
+ os.chdir(srcdir)
+ files_to_check = set(
+ path.join('keyrings', x + '.gpg') for x in [
+ 'debian-keyring',
+ 'debian-maintainers',
+ 'debian-nonupload',
+ 'debian-role-keys',
+ 'emeritus-keyring'])
+ unexpected_files = set()
+ for line in filter(lambda x: x, codecs.decode(gpgvout.stdout).split('\n')):
+ (indigest, fname) = line.split()
+ with open(fname, 'rb') as f:
+ data = f.read()
+ digest = hashlib.new('sha512', data=data).hexdigest()
+ if digest != indigest:
+ raise Exception(
+ 'mismatched digest for %s.\nWanted: %s\nGot: %s' %
+ (fname, indigest, digest))
+ if fname in files_to_check:
+ files_to_check.remove(fname)
+ else:
+ unexpected_files.add(fname)
+ if files_to_check:
+ raise Exception('No sha512 digest found for: %s' % (files_to_check))
+ if unexpected_files:
+ print(
+ 'unexpected files (maybe add them to files_to_check):',
+ unexpected_files)
+
+ keyrings = ['keyring', 'maintainers', 'nonupload']
+ for kname in keyrings:
+ kfile = path.join(pendingdir, 'debian-%s.gpg' % (kname))
+ if path.exists(kfile):
+ raise Exception(
+ 'Unhandled pending updates.\nKeyrings in %s should be dealt with and removed' %
+ (pendingdir))
+
+ for kname in keyrings:
+ kfile = path.join(hkpdir, 'debian-%s.gpg' % (kname))
+ copy(kfile, pendingdir)
+
+ print('Updating active keyrings.')
+ copy_tree(srcdir, outputdir)
+ fix_perms(outputdir)
+ print('Updating HKP keyrings.')
+ for kname in keyrings:
+ kfile = path.join(srcdir, 'keyrings', 'debian-%s.gpg' % (kname))
+ dst = os.path.join(hkpdir, os.path.basename(kfile))
+ copyfile(kfile, dst)
+ print('Publishing WKD and DANE data (may take a few minutes).')
+ with tempfile.TemporaryDirectory(prefix='pub_staging_', dir=prefix) as wkd_staging:
+ os.chdir(wkd_staging)
+
+ def dkeyring(name: str) -> str:
+ return path.join(srcdir, 'keyrings', 'debian-' + name + '.gpg')
+ build_wkd_and_dane('debian.org',
+ [dkeyring(x) for x in [
+ 'nonupload',
+ 'keyring',
+ 'role-keys']])
+ wkd_deploy_path = path.join(prefix, 'openpgpkey')
+ # not quite an atomic move:
+ if path.isdir(wkd_deploy_path):
+ os.rename(wkd_deploy_path, 'openpgpkey.old')
+ os.rename('openpgpkey', wkd_deploy_path)
+ fix_perms(wkd_deploy_path)
+ os.rename(
+ '_openpgpkey.debian.org.zone',
+ path.join(prefix, '_openpgpkey.debian.org.zone'))
+ fix_perms(path.join(prefix, '_openpgpkey.debian.org.zone'))
+ os.chdir(srcdir)
+ run(['static-update-component', 'openpgpkey.debian.org'], check=True)
+ run(['sudo', 'service', 'bind9', 'reload'], check=True)
+
+
+def openpgpkey_zonefile_header(timestamp: datetime.datetime = None, sequence: int = 0) -> bytes:
+ '''Return static DNS RRs for _openpgpkey.debian.org
+
+ These records were suggested by the Debian System Administration
+ (DSA) team.
+ '''
+ if timestamp is None:
+ timestamp = datetime.datetime.now()
+ return b'''; zonefile for OPENPGPKEY records (RFC 7929) for debian.org
+_openpgpkey.debian.org. 3600 IN SOA kaufmann.debian.org. hostmaster.debian.org. (
+ %d%02d ; serial
+ 1800 ; refresh (30 minutes)
+ 600 ; retry (10 minutes)
+ 1814400 ; expire (3 weeks)
+ 600 ; minimum (10 minutes)
+ )
+_openpgpkey.debian.org. 28800 IN NS sec2.rcode0.net.
+_openpgpkey.debian.org. 28800 IN NS nsp.dnsnode.net.
+_openpgpkey.debian.org. 28800 IN NS sec1.rcode0.net.
+
+''' % (int(timestamp.strftime('%Y%m%d')), sequence)
+
+
+def keyring_maint_keys() -> bytes:
+ '''Extract keyring-maint keys from the local system keyrings.
+
+On DSA-managed hosts, /srv/keyring.debian.org/keyrings is more recent
+and up-to-date so we prefer it. On other hosts that have the
+debian-keyring package installed, we can fall back to it.
+ '''
+ keyring_locations = [
+ '/srv/keyring.debian.org/keyrings',
+ '/usr/share/keyrings']
+ keyrings = ['debian-keyring.gpg', 'debian-nonupload.gpg']
+ keyring_maint_uids = ['Jonathan McDowell <noodles@earth.li>',
+ 'William John Sullivan <johns@debian.org>',
+ 'Gunnar Wolf <gwolf@debian.org>',
+ 'Daniel Kahn Gillmor <dkg@debian.org>']
+ keyring_files = None
+ for loc in keyring_locations:
+ possible_keyrings = [path.join(loc, k) for k in keyrings]
+ if path.isdir(loc) and all(
+ map(lambda k: path.exists(k), possible_keyrings)):
+ keyring_files = possible_keyrings
+ break
+
+ if keyring_files is None:
+ raise Exception(
+ "Could not find keyrings to extract keyring-maint keys")
+
+ gpgcmd = ['gpg',
+ '--batch',
+ '--homedir',
+ '/dev/null',
+ '--no-options',
+ '--no-default-keyring',
+ '--export-options',
+ 'export-minimal']
+ for k in keyring_files:
+ gpgcmd += ['--keyring', k]
+ gpgcmd += ['--export']
+ gpgcmd += ['=' + u for u in keyring_maint_uids]
+
+ return run(gpgcmd, stdout=PIPE, check=True).stdout
+
+
+if __name__ == '__main__':
+ if len(sys.argv) < 2:
+ raise Exception('Must provide directory containing new keyrings.')
+ elif len(sys.argv) > 2:
+ sys.argv.pop(0)
+ subcommand = sys.argv.pop(0)
+ if subcommand != 'build-wkd':
+ raise Exception("do not know this subcommand: %s" % (subcommand))
+ if len(sys.argv):
+ domain = sys.argv.pop(0)
+ else:
+ domain = 'debian.org'
+ if len(sys.argv):
+ keys = sys.argv
+ else:
+ keys = ['/usr/share/keyrings/debian-nonupload.gpg',
+ '/usr/share/keyrings/debian-keyring.gpg',
+ '/usr/share/keyrings/debian-role-keys.gpg']
+ build_wkd_and_dane(domain, keys)
+ else:
+ # standard update-keyrings
+ check_environ()
+ publish(sys.argv[1])