Adding upstream version 2022.12.24.upstream/2022.12.24upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 405 insertions, 0 deletions

diff --git a/scripts/update-keyrings b/scripts/update-keyrings
new file mode 100755
index 0000000..8e5ed6f
--- /dev/null
+++ b/scripts/update-keyrings
@@ -0,0 +1,405 @@
+#!/usr/bin/python3
+
+# Authors: Daniel Kahn Gillmor <dkg@fifthhorseman.net>,
+#          Gunnar Wolf <gwolf@debian.org>,
+#          Jonathan McDowell <noodles@earth.li>
+# License: Parts from dkg are GPLv3+
+
+import os
+from os import path
+import socket
+from subprocess import run, Popen, PIPE
+from shutil import chown, copy, copyfile, rmtree
+from distutils.dir_util import copy_tree
+import sys
+import tempfile
+import hashlib
+import codecs
+from multiprocessing.pool import ThreadPool
+from typing import List, Tuple, Optional
+import datetime
+from email.utils import parseaddr
+
+
+def check_environ(should_run_on: str = 'kaufmann.debian.org') -> None:
+    '''Make sure that we are running where we expect to run
+
+    The expectation is to run on kaufmann.debian.org, but this can be
+    bypassed for testing with the RUNANYWAY environment variable.
+    '''
+    if not (os.environ.get('RUNANYWAY', False) or
+            socket.getfqdn(socket.gethostname()) == should_run_on):
+        raise Exception('''
+This script is meant to be run in %s
+You can still run it if you are sure by setting
+$RUNANYWAY to a nonempty value.
+        ''' % (should_run_on))
+
+
+def wkd_localpart(incoming: bytes) -> str:
+    '''Z-base32 the localpart of an e-mail address
+
+    https://tools.ietf.org/html/draft-koch-openpgp-webkey-service-08#section-3.1
+    describes why this is needed.
+
+    See https://tools.ietf.org/html/rfc6189#section-5.1.6 for a
+    description of the z-base32 scheme.
+    '''
+    zb32 = "ybndrfg8ejkmcpqxot1uwisza345h769"
+
+    b = hashlib.sha1(incoming).digest()
+    ret = ""
+    assert(len(b) * 8 == 160)
+    for i in range(0, 160, 5):
+        byte = i // 8
+        offset = i - byte * 8
+        # offset | bits remaining in k+1 | right-shift k+1
+        # 3 | 0 | x
+        # 4 | 1 | 7
+        # 5 | 2 | 6
+        # 6 | 3 | 5
+        # 7 | 4 | 4
+        if offset < 4:
+            n = (b[byte] >> (3 - offset))
+        else:
+            n = (b[byte] << (offset - 3)) + (b[byte + 1] >> (11 - offset))
+
+        ret += zb32[n & 0b11111]
+    return ret
+
+
+def getdomainlocalpart(line: bytes, domain: bytes) -> Optional[bytes]:
+    'Get the localpart of the e-mail address of a GnuPG user ID line matching DOMAIN'
+    if line.startswith(b'uid:'):
+        uid = line.split(b':')[9]
+        uid = uid.decode()
+        _name, addr = parseaddr(uid)  # if parsing fails, this returns ('', '')
+        localpart, _at, thisdomain = addr.rpartition('@')
+        if thisdomain == domain.decode():
+            return localpart.lower().encode()
+    return None
+
+
+def gpgbase(keyrings: List[str]) -> List[str]:
+    'Return the standard set of options to invoke gpg in an automated way'
+    return ['gpg', '--batch', '--no-options', '--with-colons',
+            '--no-default-keyring',
+            '--homedir=/dev/null', '--trust-model=always',
+            '--fixed-list-mode'] + list(map(lambda k: '--keyring=' + k, keyrings))
+
+
+def emit_wkd_and_return_dane(localpart: bytes, domain: str, keyrings: List[str]) -> bytes:
+    '''For a given address, emit the WKD file, and return the DANE OPENPGKEY record
+
+    These are handled differently because we want to generate a
+    single, reproducible zonefile for the DNS, while we are generating
+    a tree of files for WKD.
+
+    The caller will assemble all of the OPENPGPKEY records into a
+    single zonefile.
+    '''
+    wkdstr = wkd_localpart(localpart)
+    # what do we do if this local part is not a proper encoding?
+    addr = codecs.decode(localpart) + '@' + domain
+    cmd = gpgbase(keyrings) + ['--output',
+                               path.join('openpgpkey', domain, 'hu', wkdstr),
+                               '--export-options',
+                               'export-clean',
+                               '--export-filter',
+                               'keep-uid=mbox=' + addr,
+                               '--export',
+                               '<' + addr + '>']
+    run(cmd, check=True)
+    cmd = gpgbase(keyrings) + ['--export-options', 'export-dane,export-clean',
+                               '--export-filter', 'keep-uid=mbox=' + addr,
+                               '--export', '<' + addr + '>']
+    dane = run(cmd, stdout=PIPE, check=True)
+    return dane.stdout
+
+
+def build_wkd_and_dane(domain: str, keyrings: List[str]) -> None:
+    'Publish WKD and DANE OPENPGPKEY for all domain-relevant OpenPGP certificates'
+    if not path.isdir('openpgpkey'):
+        os.mkdir('openpgpkey')
+    os.mkdir(path.join('openpgpkey', domain))
+    os.mkdir(path.join('openpgpkey', domain, 'hu'))
+
+    # FIXME: deal with IDN:
+    bytedomain = codecs.encode(domain)
+
+    lister = Popen(gpgbase(keyrings) +
+                   ['--list-keys', '@' + domain], stdout=PIPE)
+
+    localparts = set(
+        map(lambda x: getdomainlocalpart(x, bytedomain), lister.stdout))
+    localparts.discard(None)
+
+    dane_map = {}
+
+    def runner(x: bytes) -> Tuple[bytes, bytes]:
+        return (x, emit_wkd_and_return_dane(x, domain, keyrings))
+
+    def add_to_zone(res: Tuple[bytes, bytes]) -> None:
+        dane_map[res[0]] = res[1]
+
+    pool = ThreadPool(None)
+    for localpart in localparts:
+        pool.apply_async(runner, (localpart,), {}, add_to_zone)
+
+    pool.close()
+    pool.join()
+    # make the policy file:
+    policyfile = open(path.join('openpgpkey', domain, 'policy'), 'wb')
+    del policyfile
+    # write out the zonefile all at once, ordered by the localpart
+    with open(path.join('_openpgpkey.' + domain + '.zone'), 'wb') as zonefile:
+        when = datetime.datetime.now()
+        # FIXME: inspect serial number from existing zonefile --
+        # update serial number if it was from the same day
+        serial = 0
+        zonefile.write(openpgpkey_zonefile_header(when, serial))
+        for local in sorted(dane_map.keys()):
+            zonefile.write(dane_map[local])
+
+
+def fix_perms(path: str) -> None:
+    '''Fix the permissions of a given directory
+
+    Ensures all files/directories are owned and writeable by the keyring group.
+    Additionally they must be readable by all and directories executable.
+    '''
+    try:
+        chown(path, group="keyring")
+        os.chmod(path, 0o775)
+    except:
+        pass
+    for root, dirs, files in os.walk(path):
+        for cur in dirs:
+            try:
+                chown(os.path.join(root, cur), group="keyring")
+            except:
+                pass
+            try:
+                os.chmod(os.path.join(root, cur), 0o775)
+            except:
+                pass
+        for cur in files:
+            try:
+                chown(os.path.join(root, cur), group="keyring")
+            except:
+                pass
+            try:
+                os.chmod(os.path.join(root, cur), 0o664)
+            except:
+                pass
+
+
+def publish(srcdir: str,
+            where: str = None) -> None:
+    '''Verify new keyrings in srcdir; if ok, then publish at where.
+
+    Verification ensures that the new keyrings are signed-off by a
+    member of debian's keyring-maint team.
+
+    Publication consists of verifying the keyrings, placing them where
+    onak can find them, produce a zonefile for OPENPGPKEY DANE
+    records, and a tree of files for WKD.
+
+    '''
+    if where is None:
+        prefix = os.environ.get('PREFIX', '/srv/keyring.debian.org')
+    else:
+        prefix = where
+    pendingdir = path.join(prefix, 'pending-updates')
+    hkpdir = path.join(prefix, 'keyrings-new')
+    outputdir = path.join(prefix, 'pub')
+    for direc in [srcdir, pendingdir, hkpdir, outputdir]:
+        if not path.isdir(direc):
+            raise Exception("%s is not a directory" % (direc))
+    srcdir = path.realpath(srcdir)
+    sha512fname = path.join(srcdir, 'sha512sums.txt')
+    if not path.exists(sha512fname):
+        raise Exception('sha512sums.txt not found in %s' % (srcdir))
+    placeholder = path.join(srcdir, 'keyrings', '.placeholder')
+    if path.exists(placeholder):
+        os.unlink(placeholder)
+    # gpgv needs the keyring in the filesystem, not just a file
+    # descriptor (https://dev.gnupg.org/T4608)
+    with tempfile.NamedTemporaryFile() as maint_keyring:
+        maint_keyring.write(keyring_maint_keys())
+        gpgvcall = [
+            'gpgv',
+            '--enable-special-filenames',
+            '--keyring',
+            maint_keyring.name,
+            '--output',
+            '-',
+            sha512fname]
+        gpgvout = run(gpgvcall, stderr=PIPE, stdout=PIPE)
+        if gpgvout.returncode != 0:
+            raise Exception("gpg verification failed:\n%s" %
+                            (codecs.decode(gpgvout.stderr)))
+    os.chdir(srcdir)
+    files_to_check = set(
+        path.join('keyrings', x + '.gpg') for x in [
+            'debian-keyring',
+            'debian-maintainers',
+            'debian-nonupload',
+            'debian-role-keys',
+            'emeritus-keyring'])
+    unexpected_files = set()
+    for line in filter(lambda x: x, codecs.decode(gpgvout.stdout).split('\n')):
+        (indigest, fname) = line.split()
+        with open(fname, 'rb') as f:
+            data = f.read()
+        digest = hashlib.new('sha512', data=data).hexdigest()
+        if digest != indigest:
+            raise Exception(
+                'mismatched digest for %s.\nWanted: %s\nGot: %s' %
+                (fname, indigest, digest))
+        if fname in files_to_check:
+            files_to_check.remove(fname)
+        else:
+            unexpected_files.add(fname)
+    if files_to_check:
+        raise Exception('No sha512 digest found for: %s' % (files_to_check))
+    if unexpected_files:
+        print(
+            'unexpected files (maybe add them to files_to_check):',
+            unexpected_files)
+
+    keyrings = ['keyring', 'maintainers', 'nonupload']
+    for kname in keyrings:
+        kfile = path.join(pendingdir, 'debian-%s.gpg' % (kname))
+        if path.exists(kfile):
+            raise Exception(
+                'Unhandled pending updates.\nKeyrings in %s should be dealt with and removed' %
+                (pendingdir))
+
+    for kname in keyrings:
+        kfile = path.join(hkpdir, 'debian-%s.gpg' % (kname))
+        copy(kfile, pendingdir)
+
+    print('Updating active keyrings.')
+    copy_tree(srcdir, outputdir)
+    fix_perms(outputdir)
+    print('Updating HKP keyrings.')
+    for kname in keyrings:
+        kfile = path.join(srcdir, 'keyrings', 'debian-%s.gpg' % (kname))
+        dst = os.path.join(hkpdir, os.path.basename(kfile))
+        copyfile(kfile, dst)
+    print('Publishing WKD and DANE data (may take a few minutes).')
+    with tempfile.TemporaryDirectory(prefix='pub_staging_', dir=prefix) as wkd_staging:
+        os.chdir(wkd_staging)
+
+        def dkeyring(name: str) -> str:
+            return path.join(srcdir, 'keyrings', 'debian-' + name + '.gpg')
+        build_wkd_and_dane('debian.org',
+                           [dkeyring(x) for x in [
+                               'nonupload',
+                               'keyring',
+                               'role-keys']])
+        wkd_deploy_path = path.join(prefix, 'openpgpkey')
+        # not quite an atomic move:
+        if path.isdir(wkd_deploy_path):
+            os.rename(wkd_deploy_path, 'openpgpkey.old')
+        os.rename('openpgpkey', wkd_deploy_path)
+        fix_perms(wkd_deploy_path)
+        os.rename(
+            '_openpgpkey.debian.org.zone',
+            path.join(prefix, '_openpgpkey.debian.org.zone'))
+        fix_perms(path.join(prefix, '_openpgpkey.debian.org.zone'))
+        os.chdir(srcdir)
+    run(['static-update-component', 'openpgpkey.debian.org'], check=True)
+    run(['sudo', 'service', 'bind9', 'reload'], check=True)
+
+
+def openpgpkey_zonefile_header(timestamp: datetime.datetime = None, sequence: int = 0) -> bytes:
+    '''Return static DNS RRs for _openpgpkey.debian.org
+
+    These records were suggested by the Debian System Administration
+    (DSA) team.
+    '''
+    if timestamp is None:
+        timestamp = datetime.datetime.now()
+    return b'''; zonefile for OPENPGPKEY records (RFC 7929) for debian.org
+_openpgpkey.debian.org.  3600 IN SOA kaufmann.debian.org. hostmaster.debian.org. (
+                                 %d%02d ; serial
+                                 1800       ; refresh (30 minutes)
+                                 600        ; retry (10 minutes)
+                                 1814400    ; expire (3 weeks)
+                                 600        ; minimum (10 minutes)
+                                 )
+_openpgpkey.debian.org. 28800 IN NS sec2.rcode0.net.
+_openpgpkey.debian.org. 28800 IN NS nsp.dnsnode.net.
+_openpgpkey.debian.org. 28800 IN NS sec1.rcode0.net.
+
+''' % (int(timestamp.strftime('%Y%m%d')), sequence)
+
+
+def keyring_maint_keys() -> bytes:
+    '''Extract keyring-maint keys from the local system keyrings.
+
+On DSA-managed hosts, /srv/keyring.debian.org/keyrings is more recent
+and up-to-date so we prefer it.  On other hosts that have the
+debian-keyring package installed, we can fall back to it.
+    '''
+    keyring_locations = [
+        '/srv/keyring.debian.org/keyrings',
+        '/usr/share/keyrings']
+    keyrings = ['debian-keyring.gpg', 'debian-nonupload.gpg']
+    keyring_maint_uids = ['Jonathan McDowell <noodles@earth.li>',
+                          'William John Sullivan <johns@debian.org>',
+                          'Gunnar Wolf <gwolf@debian.org>',
+                          'Daniel Kahn Gillmor <dkg@debian.org>']
+    keyring_files = None
+    for loc in keyring_locations:
+        possible_keyrings = [path.join(loc, k) for k in keyrings]
+        if path.isdir(loc) and all(
+                map(lambda k: path.exists(k), possible_keyrings)):
+            keyring_files = possible_keyrings
+            break
+
+    if keyring_files is None:
+        raise Exception(
+            "Could not find keyrings to extract keyring-maint keys")
+
+    gpgcmd = ['gpg',
+              '--batch',
+              '--homedir',
+              '/dev/null',
+              '--no-options',
+              '--no-default-keyring',
+              '--export-options',
+              'export-minimal']
+    for k in keyring_files:
+        gpgcmd += ['--keyring', k]
+    gpgcmd += ['--export']
+    gpgcmd += ['=' + u for u in keyring_maint_uids]
+
+    return run(gpgcmd, stdout=PIPE, check=True).stdout
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        raise Exception('Must provide directory containing new keyrings.')
+    elif len(sys.argv) > 2:
+        sys.argv.pop(0)
+        subcommand = sys.argv.pop(0)
+        if subcommand != 'build-wkd':
+            raise Exception("do not know this subcommand: %s" % (subcommand))
+        if len(sys.argv):
+            domain = sys.argv.pop(0)
+        else:
+            domain = 'debian.org'
+        if len(sys.argv):
+            keys = sys.argv
+        else:
+            keys = ['/usr/share/keyrings/debian-nonupload.gpg',
+                    '/usr/share/keyrings/debian-keyring.gpg',
+                    '/usr/share/keyrings/debian-role-keys.gpg']
+        build_wkd_and_dane(domain, keys)
+    else:
+        # standard update-keyrings
+        check_environ()
+        publish(sys.argv[1])