1 files changed, 568 insertions, 0 deletions

diff --git a/make_mirror.sh b/make_mirror.sh
new file mode 100755
index 0000000..8849ee3
--- /dev/null
+++ b/make_mirror.sh
@@ -0,0 +1,568 @@
+#!/bin/sh
+
+set -eu
+
+# This script fills either cache.A or cache.B with new content and then
+# atomically switches the cache symlink from one to the other at the end.
+# This way, at no point will the cache be in an non-working state, even
+# when this script got canceled at any point.
+# Working with two directories also automatically prunes old packages in
+# the local repository.
+
+deletecache() {
+	dir="$1"
+	echo "running deletecache $dir">&2
+	if [ ! -e "$dir" ]; then
+		return
+	fi
+	if [ ! -e "$dir/mmdebstrapcache" ]; then
+		echo "$dir cannot be the mmdebstrap cache" >&2
+		return 1
+	fi
+	# be very careful with removing the old directory
+	# experimental is pulled in with USE_HOST_APT_CONFIG=yes on debci
+	# when testing a package from experimental
+	for dist in oldstable stable testing unstable experimental; do
+		# deleting artifacts from test "debootstrap"
+		for variant in minbase buildd -; do
+			if [ -e "$dir/debian-$dist-$variant.tar" ]; then
+				rm "$dir/debian-$dist-$variant.tar"
+			else
+				echo "does not exist: $dir/debian-$dist-$variant.tar" >&2
+			fi
+		done
+		# deleting artifacts from test "mmdebstrap"
+		for variant in essential apt minbase buildd - standard; do
+			for format in tar ext2 squashfs; do
+				if [ -e "$dir/mmdebstrap-$dist-$variant.$format" ]; then
+					# attempt to delete for all dists because DEFAULT_DIST might've been different the last time
+					rm "$dir/mmdebstrap-$dist-$variant.$format"
+				elif [ "$dist" = "$DEFAULT_DIST" ]; then
+					# only warn about non-existance when it's expected to exist
+					echo "does not exist: $dir/mmdebstrap-$dist-$variant.$format" >&2
+				fi
+			done
+		done
+		if [ -e "$dir/debian/dists/$dist" ]; then
+			rm --one-file-system --recursive "$dir/debian/dists/$dist"
+		else
+			echo "does not exist: $dir/debian/dists/$dist" >&2
+		fi
+		case "$dist" in oldstable|stable)
+			if [ -e "$dir/debian/dists/$dist-updates" ]; then
+				rm --one-file-system --recursive "$dir/debian/dists/$dist-updates"
+			else
+				echo "does not exist: $dir/debian/dists/$dist-updates" >&2
+			fi
+			;;
+		esac
+		case "$dist" in oldstable|stable)
+			if [ -e "$dir/debian-security/dists/$dist-security" ]; then
+				rm --one-file-system --recursive "$dir/debian-security/dists/$dist-security"
+			else
+				echo "does not exist: $dir/debian-security/dists/$dist-security" >&2
+			fi
+			;;
+		esac
+	done
+	for f in "$dir/debian-"*.ext4; do
+		if [ -e "$f" ]; then
+			rm --one-file-system "$f"
+		fi
+	done
+	# on i386 and amd64, the intel-microcode and amd64-microcode packages
+	# from non-free-firwame get pulled in because they are
+	# priority:standard with USE_HOST_APT_CONFIG=yes
+	for c in main non-free-firmware; do
+		if [ -e "$dir/debian/pool/$c" ]; then
+			rm --one-file-system --recursive "$dir/debian/pool/$c"
+		else
+			echo "does not exist: $dir/debian/pool/$c" >&2
+		fi
+	done
+	if [ -e "$dir/debian-security/pool/updates/main" ]; then
+		rm --one-file-system --recursive "$dir/debian-security/pool/updates/main"
+	else
+		echo "does not exist: $dir/debian-security/pool/updates/main" >&2
+	fi
+	for i in $(seq 1 6); do
+		if [ ! -e "$dir/debian$i" ]; then
+			continue
+		fi
+		rm "$dir/debian$i"
+	done
+	rm "$dir/mmdebstrapcache"
+	# remove all symlinks
+	find "$dir" -type l -delete
+
+	# now the rest should only be empty directories
+	if [ -e "$dir" ]; then
+		find "$dir" -depth -print0 | xargs -0 --no-run-if-empty rmdir
+	else
+		echo "does not exist: $dir" >&2
+	fi
+}
+
+cleanup_newcachedir() {
+	echo "running cleanup_newcachedir"
+	deletecache "$newcachedir"
+}
+
+cleanupapt() {
+	echo "running cleanupapt" >&2
+	if [ ! -e "$rootdir" ]; then
+		return
+	fi
+	for f in \
+		"$rootdir/var/cache/apt/archives/"*.deb \
+		"$rootdir/var/cache/apt/archives/partial/"*.deb \
+		"$rootdir/var/cache/apt/"*.bin \
+		"$rootdir/var/lib/apt/lists/"* \
+		"$rootdir/var/lib/dpkg/status" \
+		"$rootdir/var/lib/dpkg/lock-frontend" \
+		"$rootdir/var/lib/dpkg/lock" \
+		"$rootdir/var/lib/apt/lists/lock" \
+		"$rootdir/etc/apt/apt.conf" \
+		"$rootdir/etc/apt/sources.list.d/"* \
+		"$rootdir/etc/apt/preferences.d/"* \
+		"$rootdir/etc/apt/sources.list" \
+		"$rootdir/var/cache/apt/archives/lock"; do
+		if [ ! -e "$f" ]; then
+			echo "does not exist: $f" >&2
+			continue
+		fi
+		if [ -d "$f" ]; then
+			rmdir "$f"
+		else
+			rm "$f"
+		fi
+	done
+	find "$rootdir" -depth -print0 | xargs -0 --no-run-if-empty rmdir
+}
+
+# note: this function uses brackets instead of curly braces, so that it's run
+# in its own process and we can handle traps independent from the outside
+update_cache() (
+	dist="$1"
+	nativearch="$2"
+
+	# use a subdirectory of $newcachedir so that we can use
+	# hardlinks
+	rootdir="$newcachedir/apt"
+	mkdir -p "$rootdir"
+
+	# we only set this trap here and overwrite the previous trap, because
+	# the update_cache function is run as part of a pipe and thus in its
+	# own process which will EXIT after it finished
+	trap 'kill "$PROXYPID" || :;cleanupapt' EXIT INT TERM
+
+	for p in /etc/apt/apt.conf.d /etc/apt/sources.list.d /etc/apt/preferences.d /var/cache/apt/archives /var/lib/apt/lists/partial /var/lib/dpkg; do
+		mkdir -p "$rootdir/$p"
+	done
+
+	# read sources.list content from stdin
+	cat > "$rootdir/etc/apt/sources.list"
+
+	cat << END > "$rootdir/etc/apt/apt.conf"
+Apt::Architecture "$nativearch";
+Apt::Architectures "$nativearch";
+Dir::Etc "$rootdir/etc/apt";
+Dir::State "$rootdir/var/lib/apt";
+Dir::Cache "$rootdir/var/cache/apt";
+Apt::Install-Recommends false;
+Apt::Get::Download-Only true;
+Acquire::Languages "none";
+Dir::Etc::Trusted "/etc/apt/trusted.gpg";
+Dir::Etc::TrustedParts "/etc/apt/trusted.gpg.d";
+Acquire::http::Proxy "http://127.0.0.1:8080/";
+END
+
+	: > "$rootdir/var/lib/dpkg/status"
+
+	if [ "$dist" = "$DEFAULT_DIST" ] && [ "$nativearch" = "$HOSTARCH" ] && [ "$USE_HOST_APT_CONFIG" = "yes" ]; then
+		# we append sources and settings instead of overwriting after
+		# an empty line
+		for f in /etc/apt/sources.list /etc/apt/sources.list.d/*; do
+			[ -e "$f" ] || continue
+			[ -e "$rootdir/$f" ] && echo >> "$rootdir/$f"
+			# Filter out file:// repositories as they are added
+			# to each mmdebstrap call verbatim by
+			# debian/tests/copy_host_apt_config
+			# Also filter out all mirrors that are not of suite
+			# $DEFAULT_DIST, except experimental if the suite
+			# is unstable. This prevents packages from
+			# unstable entering a testing mirror.
+			if [ "$dist" = unstable ]; then
+				grep -v ' file://' "$f" \
+					| grep -E " (unstable|experimental) " \
+					>> "$rootdir/$f" || :
+			else
+				grep -v ' file://' "$f" \
+					| grep " $DEFAULT_DIST " \
+					>> "$rootdir/$f" || :
+			fi
+		done
+		for f in /etc/apt/preferences.d/*; do
+			[ -e "$f" ] || continue
+			[ -e "$rootdir/$f" ] && echo >> "$rootdir/$f"
+			cat "$f" >> "$rootdir/$f"
+		done
+	fi
+
+	echo "creating mirror for $dist" >&2
+	for f in /etc/apt/sources.list /etc/apt/sources.list.d/* /etc/apt/preferences.d/*; do
+		[ -e "$rootdir/$f" ] || continue
+		echo "contents of $f:" >&2
+		cat "$rootdir/$f" >&2
+	done
+
+	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get update --error-on=any
+
+	pkgs=$(APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get indextargets \
+		--format '$(FILENAME)' 'Created-By: Packages' "Architecture: $nativearch" \
+		| xargs --delimiter='\n' /usr/lib/apt/apt-helper cat-file \
+		| grep-dctrl --no-field-names --show-field=Package --exact-match \
+			\( --field=Essential yes --or --field=Priority required \
+			--or --field=Priority important --or --field=Priority standard \
+			\))
+
+	pkgs="$pkgs build-essential busybox gpg eatmydata fakechroot fakeroot"
+
+	# we need usr-is-merged to simulate debootstrap behaviour for all dists
+	# starting from Debian 12 (Bullseye)
+	case "$dist" in
+		oldstable) : ;;
+		*) pkgs="$pkgs usr-is-merged usrmerge" ;;
+	esac
+
+	# shellcheck disable=SC2086
+	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install $pkgs
+
+	rm "$rootdir/var/cache/apt/archives/lock"
+	rmdir "$rootdir/var/cache/apt/archives/partial"
+	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --option Dir::Etc::SourceList=/dev/null update
+	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get clean
+
+	cleanupapt
+
+	# this function is run in its own process, so we unset all traps before
+	# returning
+	trap "-" EXIT INT TERM
+)
+
+check_proxy_running() {
+	if timeout 1 bash -c 'exec 3<>/dev/tcp/127.0.0.1/8080 && printf "GET http://deb.debian.org/debian/dists/'"$DEFAULT_DIST"'/InRelease HTTP/1.1\nHost: deb.debian.org\n\n" >&3 && grep "Suite: '"$DEFAULT_DIST"'" <&3 >/dev/null' 2>/dev/null; then
+		return 0
+	elif timeout 1 env http_proxy="http://127.0.0.1:8080/" wget --quiet -O - "http://deb.debian.org/debian/dists/$DEFAULT_DIST/InRelease" | grep "Suite: $DEFAULT_DIST" >/dev/null; then
+		return 0
+	elif timeout 1 curl --proxy "http://127.0.0.1:8080/" --silent "http://deb.debian.org/debian/dists/$DEFAULT_DIST/InRelease" | grep "Suite: $DEFAULT_DIST" >/dev/null; then
+		return 0
+	fi
+	return 1
+}
+
+if [ -e "./shared/cache.A" ] && [ -e "./shared/cache.B" ]; then
+	echo "both ./shared/cache.A and ./shared/cache.B exist" >&2
+	echo "was a former run of the script aborted?" >&2
+	if [ -e ./shared/cache ]; then
+		echo "cache symlink points to $(readlink ./shared/cache)" >&2
+		case "$(readlink ./shared/cache)" in
+			cache.A)
+				echo "removing ./shared/cache.B" >&2
+				rm -r ./shared/cache.B
+				;;
+			cache.B)
+				echo "removing ./shared/cache.A" >&2
+				rm -r ./shared/cache.A
+				;;
+			*)
+				echo "unexpected" >&2
+				exit 1
+				;;
+		esac
+	else
+		echo "./shared/cache doesn't exist" >&2
+		exit 1
+	fi
+fi
+
+if [ -e "./shared/cache.A" ]; then
+	oldcache=cache.A
+	newcache=cache.B
+else
+	oldcache=cache.B
+	newcache=cache.A
+fi
+
+oldcachedir="./shared/$oldcache"
+newcachedir="./shared/$newcache"
+
+oldmirrordir="$oldcachedir/debian"
+newmirrordir="$newcachedir/debian"
+
+mirror="http://deb.debian.org/debian"
+security_mirror="http://security.debian.org/debian-security"
+components=main
+
+: "${DEFAULT_DIST:=unstable}"
+: "${ONLY_DEFAULT_DIST:=no}"
+: "${ONLY_HOSTARCH:=no}"
+: "${HAVE_QEMU:=yes}"
+: "${RUN_MA_SAME_TESTS:=yes}"
+# by default, use the mmdebstrap executable in the current directory
+: "${CMD:=./mmdebstrap}"
+: "${USE_HOST_APT_CONFIG:=no}"
+: "${FORCE_UPDATE:=no}"
+
+if [ "$FORCE_UPDATE" != "yes" ] && [ -e "$oldmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then
+	http_code=$(curl --output /dev/null --silent --location --head --time-cond "$oldmirrordir/dists/$DEFAULT_DIST/InRelease" --write-out '%{http_code}' "$mirror/dists/$DEFAULT_DIST/InRelease")
+	case "$http_code" in
+		200) ;; # need update
+		304) echo up-to-date; exit 0;;
+		*) echo "unexpected status: $http_code"; exit 1;;
+	esac
+fi
+
+./caching_proxy.py "$oldcachedir" "$newcachedir" &
+PROXYPID=$!
+trap 'kill "$PROXYPID" || :' EXIT INT TERM
+
+for i in $(seq 10); do
+	check_proxy_running && break
+	sleep 1
+done
+if [ ! -s "$newmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then
+	echo "failed to start proxy" >&2
+	kill $PROXYPID
+	exit 1
+fi
+
+trap 'kill "$PROXYPID" || :;cleanup_newcachedir' EXIT INT TERM
+
+mkdir -p "$newcachedir"
+touch "$newcachedir/mmdebstrapcache"
+
+HOSTARCH=$(dpkg --print-architecture)
+arches="$HOSTARCH"
+if [ "$HOSTARCH" = amd64 ]; then
+	arches="$arches arm64 i386"
+elif [ "$HOSTARCH" = arm64 ]; then
+	arches="$arches amd64 armhf"
+fi
+
+# we need the split_inline_sig() function
+# shellcheck disable=SC1091
+. /usr/share/debootstrap/functions
+
+for dist in oldstable stable testing unstable; do
+	for nativearch in $arches; do
+		# non-host architectures are only downloaded for $DEFAULT_DIST
+		if [ "$nativearch" != "$HOSTARCH" ] && [ "$DEFAULT_DIST" != "$dist" ]; then
+			continue
+		fi
+		# if ONLY_DEFAULT_DIST is set, only download DEFAULT_DIST
+		if [ "$ONLY_DEFAULT_DIST" = "yes" ] && [ "$DEFAULT_DIST" != "$dist" ]; then
+			continue
+		fi
+		if [ "$ONLY_HOSTARCH" = "yes" ] && [ "$nativearch" != "$HOSTARCH" ]; then
+			continue
+		fi
+		# we need a first pass without updates and security patches
+		# because otherwise, old package versions needed by
+		# debootstrap will not get included
+		echo "deb [arch=$nativearch] $mirror $dist $components" | update_cache "$dist" "$nativearch"
+		# we need to include the base mirror again or otherwise
+		# packages like build-essential will be missing
+		case "$dist" in oldstable|stable)
+			cat << END | update_cache "$dist" "$nativearch"
+deb [arch=$nativearch] $mirror $dist $components
+deb [arch=$nativearch] $mirror $dist-updates main
+deb [arch=$nativearch] $security_mirror $dist-security main
+END
+				;;
+		esac
+	done
+	codename=$(awk '/^Codename: / { print $2; }' < "$newmirrordir/dists/$dist/InRelease")
+	ln -s "$dist" "$newmirrordir/dists/$codename"
+
+	# split the InRelease file into Release and Release.gpg not because apt
+	# or debootstrap need it that way but because grep-dctrl does
+	split_inline_sig \
+		"$newmirrordir/dists/$dist/InRelease" \
+		"$newmirrordir/dists/$dist/Release" \
+		"$newmirrordir/dists/$dist/Release.gpg"
+	touch --reference="$newmirrordir/dists/$dist/InRelease" "$newmirrordir/dists/$dist/Release" "$newmirrordir/dists/$dist/Release.gpg"
+done
+
+kill $PROXYPID
+
+# Create some symlinks so that we can trick apt into accepting multiple apt
+# lines that point to the same repository but look different. This is to
+# avoid the warning:
+# W: Target Packages (main/binary-all/Packages) is configured multiple times...
+for i in $(seq 1 6); do
+	ln -s debian "$newcachedir/debian$i"
+done
+
+tmpdir=""
+
+cleanuptmpdir() {
+	if [ -z "$tmpdir" ]; then
+		return
+	fi
+	if [ ! -e "$tmpdir" ]; then
+		return
+	fi
+	for f in "$tmpdir/worker.sh" "$tmpdir/mmdebstrap.service"; do
+		if [ ! -e "$f" ]; then
+			echo "does not exist: $f" >&2
+			continue
+		fi
+		rm "$f"
+	done
+	rmdir "$tmpdir"
+}
+
+SOURCE_DATE_EPOCH="$(date --date="$(grep-dctrl -s Date -n '' "$newmirrordir/dists/$DEFAULT_DIST/Release")" +%s)"
+export SOURCE_DATE_EPOCH
+
+if [ "$HAVE_QEMU" = "yes" ]; then
+	# we use the caching proxy again when building the qemu image
+	#  - we can re-use the packages that were already downloaded earlier
+	#  - we make sure that the qemu image uses the same Release file even
+	#    if a mirror push happened between now and earlier
+	#  - we avoid polluting the mirror with the additional packages by
+	#    using --readonly
+	./caching_proxy.py --readonly "$oldcachedir" "$newcachedir" &
+	PROXYPID=$!
+
+	for i in $(seq 10); do
+		check_proxy_running && break
+		sleep 1
+	done
+	if [ ! -s "$newmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then
+		echo "failed to start proxy" >&2
+		kill $PROXYPID
+		exit 1
+	fi
+
+	tmpdir="$(mktemp -d)"
+	trap 'kill "$PROXYPID" || :;cleanuptmpdir; cleanup_newcachedir' EXIT INT TERM
+
+	pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,squashfs-tools-ng,genext2fs,linux-image-generic,passwd
+	if [ ! -e ./mmdebstrap ]; then
+		pkgs="$pkgs,mmdebstrap"
+	fi
+	arches=$HOSTARCH
+	if [ "$RUN_MA_SAME_TESTS" = "yes" ]; then
+		case "$HOSTARCH" in
+			amd64)
+				arches=amd64,arm64
+				pkgs="$pkgs,libfakechroot:arm64,libfakeroot:arm64"
+				;;
+			arm64)
+				arches=arm64,amd64
+				pkgs="$pkgs,libfakechroot:amd64,libfakeroot:amd64"
+				;;
+		esac
+	fi
+
+	cat << END > "$tmpdir/mmdebstrap.service"
+[Unit]
+Description=mmdebstrap worker script
+
+[Service]
+Type=oneshot
+ExecStart=/worker.sh
+
+[Install]
+WantedBy=multi-user.target
+END
+	# here is something crazy:
+	# as we run mmdebstrap, the process ends up being run by different users with
+	# different privileges (real or fake). But for being able to collect
+	# Devel::Cover data, they must all share a single directory. The only way that
+	# I found to make this work is to mount the database directory with a
+	# filesystem that doesn't support ownership information at all and a umask that
+	# gives read/write access to everybody.
+	# https://github.com/pjcj/Devel--Cover/issues/223
+	cat << 'END' > "$tmpdir/worker.sh"
+#!/bin/sh
+echo 'root:root' | chpasswd
+mount -t 9p -o trans=virtio,access=any,msize=128k mmdebstrap /mnt
+# need to restart mini-httpd because we mounted different content into www-root
+systemctl restart mini-httpd
+
+ip link set enp0s1 down || :
+
+handler () {
+	while IFS= read -r line || [ -n "$line" ]; do
+		printf "%s %s: %s\n" "$(date -u -d "0 $(date +%s.%3N) seconds - $2 seconds" +"%T.%3N")" "$1" "$line"
+	done
+}
+
+(
+	cd /mnt;
+	if [ -e cover_db.img ]; then
+		mkdir -p cover_db
+		mount -o loop,umask=000 cover_db.img cover_db
+	fi
+
+	now=$(date +%s.%3N)
+	ret=0
+	{ { { { {
+	          sh -x ./test.sh 2>&1 1>&4 3>&- 4>&-; echo $? >&2;
+	        } | handler E "$now" >&3;
+	      } 4>&1 | handler O "$now" >&3;
+	    } 2>&1;
+	  } | { read xs; exit $xs; };
+	} 3>&1 || ret=$?
+	echo $ret > /mnt/exitstatus.txt
+	if [ -e cover_db.img ]; then
+		df -h cover_db
+		umount cover_db
+	fi
+) > /mnt/output.txt 2>&1
+umount /mnt
+systemctl poweroff
+END
+	chmod +x "$tmpdir/worker.sh"
+	if [ -z ${DISK_SIZE+x} ]; then
+		DISK_SIZE=10G
+	fi
+	# set PATH to pick up the correct mmdebstrap variant
+	env PATH="$(dirname "$(realpath --canonicalize-existing "$CMD")"):$PATH" \
+		debvm-create --skip=usrmerge,systemdnetwork \
+		--size="$DISK_SIZE" --release="$DEFAULT_DIST" \
+		--output="$newcachedir/debian-$DEFAULT_DIST.ext4" -- \
+		--architectures="$arches" --include="$pkgs" \
+		--setup-hook='echo "Acquire::http::Proxy \"http://127.0.0.1:8080/\";" > "$1/etc/apt/apt.conf.d/00proxy"' \
+		--hook-dir=/usr/share/mmdebstrap/hooks/maybe-merged-usr \
+		--customize-hook='rm "$1/etc/apt/apt.conf.d/00proxy"' \
+		--customize-hook='mkdir -p "$1/etc/systemd/system/multi-user.target.wants"' \
+		--customize-hook='ln -s ../mmdebstrap.service "$1/etc/systemd/system/multi-user.target.wants/mmdebstrap.service"' \
+		--customize-hook='touch "$1/mmdebstrap-testenv"' \
+		--customize-hook='copy-in "'"$tmpdir"'/mmdebstrap.service" /etc/systemd/system/' \
+		--customize-hook='copy-in "'"$tmpdir"'/worker.sh" /' \
+		--customize-hook='echo 127.0.0.1 localhost > "$1/etc/hosts"' \
+		--customize-hook='printf "START=1\nDAEMON_OPTS=\"-h 127.0.0.1 -p 80 -u nobody -dd /mnt/cache -i /var/run/mini-httpd.pid -T UTF-8\"\n" > "$1/etc/default/mini-httpd"' \
+		"$mirror"
+
+	kill $PROXYPID
+	cleanuptmpdir
+	trap "cleanup_newcachedir" EXIT INT TERM
+fi
+
+# delete possibly leftover symlink
+if [ -e ./shared/cache.tmp ]; then
+	rm ./shared/cache.tmp
+fi
+# now atomically switch the symlink to point to the other directory
+ln -s $newcache ./shared/cache.tmp
+mv --no-target-directory ./shared/cache.tmp ./shared/cache
+
+deletecache "$oldcachedir"
+
+trap - EXIT INT TERM
+
+echo "$0 finished successfully" >&2