From 26a029d407be480d791972afb5975cf62c9360a6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 02:47:55 +0200 Subject: Adding upstream version 124.0.1. Signed-off-by: Daniel Baumann --- taskcluster/docker/debian-raw/Dockerfile | 70 +++++++++++++++ taskcluster/docker/debian-raw/gpgvnoexpkeysig | 52 +++++++++++ taskcluster/docker/debian-raw/snapshot-hack.py | 105 ++++++++++++++++++++++ taskcluster/docker/debian-raw/taskcluster-hack.sh | 7 ++ 4 files changed, 234 insertions(+) create mode 100644 taskcluster/docker/debian-raw/Dockerfile create mode 100755 taskcluster/docker/debian-raw/gpgvnoexpkeysig create mode 100755 taskcluster/docker/debian-raw/snapshot-hack.py create mode 100755 taskcluster/docker/debian-raw/taskcluster-hack.sh (limited to 'taskcluster/docker/debian-raw') diff --git a/taskcluster/docker/debian-raw/Dockerfile b/taskcluster/docker/debian-raw/Dockerfile new file mode 100644 index 0000000000..7dce7ccf21 --- /dev/null +++ b/taskcluster/docker/debian-raw/Dockerfile @@ -0,0 +1,70 @@ +ARG BASE_IMAGE +FROM $BASE_IMAGE +MAINTAINER Mike Hommey + +ENV DEBIAN_FRONTEND=noninteractive + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] + +# %include taskcluster/docker/recipes/setup_packages.sh +COPY topsrcdir/taskcluster/docker/recipes/setup_packages.sh /usr/local/sbin/ +# %include taskcluster/docker/recipes/clean_packages.sh +COPY topsrcdir/taskcluster/docker/recipes/clean_packages.sh /usr/local/sbin/ + +COPY taskcluster-hack.sh /usr/local/sbin +COPY snapshot-hack.py /usr/local/sbin +COPY gpgvnoexpkeysig /usr/local/sbin + +ARG DIST +ARG SNAPSHOT +ARG TASKCLUSTER_ROOT_URL +# Set apt sources list to a snapshot. +# Note: the use of gpgvnoexpkeysig is because the Debian Jessie GPG key expired. +RUN if [ -n "$DIST" ]; then for s in debian_$DIST debian_$DIST-updates debian_$DIST-backports debian-security_$DIST-security debian-debug_$DIST-debug debian-debug_$DIST-proposed-updates-debug debian-debug_$DIST-backports-debug; do \ + case "$s" in \ + debian-debug_jessie*|debian_jessie-updates) \ + : No debian-debug/updates archive for Jessie; \ + ;; \ + debian-security_jessie-security) \ + echo "deb http://archive.debian.org/debian-security/ jessie/updates main"; \ + ;; \ + debian*_jessie*) \ + echo "deb http://archive.debian.org/${s%_*}/ ${s#*_} main"; \ + ;; \ + debian-security_buster-security) \ + d=${s#*_}; \ + echo "deb http://snapshot.debian.org/archive/${s%_*}/$SNAPSHOT/ ${d%-security}/updates main"; \ + ;; \ + *) \ + echo "deb http://snapshot.debian.org/archive/${s%_*}/$SNAPSHOT/ ${s#*_} main"; \ + ;; \ + esac; \ + done > /etc/apt/sources.list ; fi && \ + rm -f /etc/apt/sources.list.d/* && \ + ( echo 'quiet "true";'; \ + echo 'APT::Get::Assume-Yes "true";'; \ + echo 'APT::Install-Recommends "false";'; \ + echo 'Acquire::Check-Valid-Until "false";'; \ + echo 'Acquire::Retries "5";'; \ + if dpkg --compare-versions $(apt --version | awk '{print $2}') ge 2.1.15; then \ + echo 'dir::bin::methods::https "/usr/local/sbin/taskcluster-hack.sh";'; \ + fi; \ + if [ "$DIST" = "jessie" ]; then \ + echo 'Dir::Bin::gpg "/usr/local/sbin/gpgvnoexpkeysig";'; \ + fi; \ + ) > /etc/apt/apt.conf.d/99taskcluster && \ + ( echo 'Package: *'; \ + echo 'Pin: origin "'$TASKCLUSTER_ROOT_URL'"' | sed 's,https://,,'; \ + echo 'Pin-Priority: 1001'; \ + ) > /etc/apt/preferences.d/99taskcluster + +RUN apt-get update && \ + if grep -q snapshot.debian.org /etc/apt/sources.list; then \ + apt-get install python3-minimal libpython3-stdlib; \ + echo 'dir::bin::methods::http "/usr/local/sbin/snapshot-hack.py";' >> /etc/apt/apt.conf.d/99taskcluster; \ + fi && \ + apt-get dist-upgrade && \ + apt-get install \ + apt-transport-https \ + ca-certificates diff --git a/taskcluster/docker/debian-raw/gpgvnoexpkeysig b/taskcluster/docker/debian-raw/gpgvnoexpkeysig new file mode 100755 index 0000000000..fbbfd7a658 --- /dev/null +++ b/taskcluster/docker/debian-raw/gpgvnoexpkeysig @@ -0,0 +1,52 @@ +#!/bin/sh +# +# Downloaded from https://gitlab.mister-muffin.de/josch/mmdebstrap/raw/branch/main/gpgvnoexpkeysig +# +# This script is in the public domain +# +# Author: Johannes Schauer Marin Rodrigues +# +# This is a wrapper around gpgv as invoked by apt. It turns EXPKEYSIG results +# from gpgv into GOODSIG results. This is necessary for apt to access very old +# timestamps from snapshot.debian.org for which the GPG key is already expired: +# +# Get:1 http://snapshot.debian.org/archive/debian/20150106T000000Z unstable InRelease [242 kB] +# Err:1 http://snapshot.debian.org/archive/debian/20150106T000000Z unstable InRelease +# The following signatures were invalid: EXPKEYSIG 8B48AD6246925553 Debian Archive Automatic Signing Key (7.0/wheezy) +# Reading package lists... +# W: GPG error: http://snapshot.debian.org/archive/debian/20150106T000000Z unstable InRelease: The following signatures were invalid: EXPKEYSIG 8B48AD6246925553 Debian Archive Automatic Signing Key (7.0/wheezy) +# E: The repository 'http://snapshot.debian.org/archive/debian/20150106T000000Z unstable InRelease' is not signed. +# +# To use this script, call apt with +# +# -o Apt::Key::gpgvcommand=/usr/libexec/mmdebstrap/gpgvnoexpkeysig +# +# Scripts doing similar things can be found here: +# +# * debuerreotype as /usr/share/debuerreotype/scripts/.gpgv-ignore-expiration.sh +# * derivative census: salsa.d.o/deriv-team/census/-/blob/master/bin/fakegpgv + +set -eu + +find_gpgv_status_fd() { + while [ "$#" -gt 0 ]; do + if [ "$1" = '--status-fd' ]; then + echo "$2" + return 0 + fi + shift + done + # default fd is stdout + echo 1 +} +GPGSTATUSFD="$(find_gpgv_status_fd "$@")" + +case $GPGSTATUSFD in + ''|*[!0-9]*) + echo "invalid --status-fd argument" >&2 + exit 1 + ;; +esac + +# we need eval because we cannot redirect a variable fd +eval 'exec gpgv "$@" '"$GPGSTATUSFD"'>&1 | sed "s/^\[GNUPG:\] EXPKEYSIG /[GNUPG:] GOODSIG /" >&'"$GPGSTATUSFD" diff --git a/taskcluster/docker/debian-raw/snapshot-hack.py b/taskcluster/docker/debian-raw/snapshot-hack.py new file mode 100755 index 0000000000..6e880f0a74 --- /dev/null +++ b/taskcluster/docker/debian-raw/snapshot-hack.py @@ -0,0 +1,105 @@ +#!/usr/bin/python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import subprocess +import sys +import threading +import urllib.request +from urllib.parse import urlparse, urlunparse + +# This script interposes between APT and its HTTP method. APT sends queries on +# stdin, and expect responses on stdout. We intercept those and change the +# snapshot.debian.org URLs it requests on the fly, if the equivalent URLs +# exist on deb.debian.org. + +URI_HEADER = "URI: " + + +def url_exists(url): + try: + req = urllib.request.Request(url, method="HEAD") + response = urllib.request.urlopen(req) + return response.getcode() == 200 + except Exception: + return False + + +def write_and_flush(fh, data): + fh.write(data) + fh.flush() + + +def output_handler(proc, url_mapping, lock): + for line in proc.stdout: + if line.startswith(URI_HEADER): + url = line[len(URI_HEADER) :].rstrip() + # APT expects back the original url it requested. + with lock: + original_url = url_mapping.get(url, None) + if original_url: + write_and_flush(sys.stdout, line.replace(url, original_url)) + continue + write_and_flush(sys.stdout, line) + + +def main(): + proc = subprocess.Popen( + ["/usr/lib/apt/methods/http"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + ) + url_mapping = {} + lock = threading.Lock() + output_thread = threading.Thread( + target=output_handler, args=(proc, url_mapping, lock), daemon=True + ) + output_thread.start() + + while True: + try: + line = sys.stdin.readline() + except KeyboardInterrupt: + # When apt cuts the connection, we receive a KeyboardInterrupt. + break + if not line: + break + + if line.startswith(URI_HEADER): + url = line[len(URI_HEADER) :].rstrip() + url_parts = urlparse(url) + # For .deb packages, if we can find the file on deb.debian.org, take it + # from there instead of snapshot.debian.org, because deb.debian.org will + # be much faster. Hopefully, most files will be available on deb.debian.org. + if url_parts.hostname == "snapshot.debian.org" and url_parts.path.endswith( + ".deb" + ): + # The url is assumed to be of the form + # http://snapshot.debian.org/archive/section/yymmddThhmmssZ/... + path_parts = url_parts.path.split("/") + # urlparse().path always starts with a / so path_parts is + # expected to look like ["", "archive", "section", "yymmddThhmmssZ", ...] + # we want to remove "archive" and "yymmddThhmmssZ" to create an url + # on deb.debian.org. + path_parts.pop(3) + path_parts.pop(1) + modified_url = urlunparse( + url_parts._replace( + netloc="deb.debian.org", path="/".join(path_parts) + ) + ) + if url_exists(modified_url): + with lock: + url_mapping[modified_url] = url + write_and_flush(proc.stdin, line.replace(url, modified_url)) + continue + write_and_flush(proc.stdin, line) + + proc.stdin.close() + output_thread.join() + + +if __name__ == "__main__": + main() diff --git a/taskcluster/docker/debian-raw/taskcluster-hack.sh b/taskcluster/docker/debian-raw/taskcluster-hack.sh new file mode 100755 index 0000000000..eecac021ec --- /dev/null +++ b/taskcluster/docker/debian-raw/taskcluster-hack.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# APT version 2.1.15 and newer changed how they handle quoting in redirections +# in a way that breaks the setup for APT repos in taskcluster artifacts +# (unfortunately, there's also no setup on the taskcluster end that would work +# with both old and newer versions of APT, short of removing redirections +# entirely). +/usr/lib/apt/methods/https | sed -u '/^New-URI:/s/+/%2b/g' -- cgit v1.2.3