Merging upstream version 1.5.2.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

15 files changed, 140 insertions, 84 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 21ed927..60e364f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,8 @@
+1.5.2 (2024-06-26)
+------------------
+
+ - mmdebstrap-autopkgtest-build-qemu produces bit-by-bit reproducible output
+
 1.5.1 (2024-06-03)
 ------------------
 
diff --git a/README.md b/README.md
index bfb460a..e4d8d83 100644
--- a/README.md
+++ b/README.md
@@ -158,16 +158,19 @@ Contributors
 
  - Johannes Schauer Marin Rodrigues (main author)
  - Helmut Grohne
+ - Jochen Sprickerhof
  - Gioele Barabucci
  - Benjamin Drung
- - Jochen Sprickerhof
  - Josh Triplett
  - Konstantin Demin
+ - Chris Hofstaedtler
+ - Colin Watson
  - David Kalnischkies
  - Emilio Pozuelo Monfort
  - Francesco Poli
  - Jakub Wilk
  - Joe Groocock
+ - Max-Julian Pogner
  - Nicolas Vigier
  - Raul Tambre
  - Steve Dodd
diff --git a/coverage.sh b/coverage.sh
index c555c8e..e02cf35 100755
--- a/coverage.sh
+++ b/coverage.sh
@@ -106,7 +106,7 @@ fi
 
 # check if the wiki has to be updated with pod2markdown output
 if [ "${DEBEMAIL:-}" = "josch@debian.org" ]; then
-	bash -exc "diff -u <(curl --silent https://gitlab.mister-muffin.de/josch/mmdebstrap/wiki/raw/Home | dos2unix) <(pod2markdown < mmdebstrap)" || :
+	bash -exc "diff -u <(curl --silent https://gitlab.mister-muffin.de/josch/mmdebstrap/wiki/raw/Home | dos2unix; echo) <(pod2markdown < mmdebstrap)" || :
 fi
 
 rm -f shared/test.sh shared/tar1.txt shared/tar2.txt shared/pkglist.txt shared/doc-debian.tar.list shared/mmdebstrap shared/tarfilter shared/proxysolver
diff --git a/coverage.txt b/coverage.txt
index c040b8b..f71e14a 100644
--- a/coverage.txt
+++ b/coverage.txt
@@ -64,7 +64,7 @@ Variants: essential apt minbase buildd - standard
 Skip-If:
  variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
  mode == "fakechroot" and variant in ["-", "standard"] # no extended attributes
- variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
+ variant == "standard" and dist in ["oldstable", "stable"] and hostarch in ["armel", "armhf", "mipsel"] # #1031276
 
 Test: check-for-bit-by-bit-identical-format-output
 Modes: unshare fakechroot
@@ -73,7 +73,7 @@ Variants: essential apt minbase buildd - standard
 Skip-If:
  variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
  mode == "fakechroot" and variant in ["-", "standard"] # no extended attributes
- variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
+ variant == "standard" and dist in ["oldstable", "stable"] and hostarch in ["armel", "armhf", "mipsel"] # #1031276
 
 Test: tarfilter-idshift
 Needs-QEMU: true
@@ -273,17 +273,13 @@ Test: debootstrap-no-op-options
 Needs-Root: true
 
 Test: verbose
-Variants: - standard
+Variants: standard
 Skip-If:
- variant == "-" and hostarch not in ["armel", "armhf", "mipsel"] # #1031276
- variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
  variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
 
 Test: debug
-Variants: - standard
+Variants: standard
 Skip-If:
- variant == "-" and hostarch not in ["armel", "armhf", "mipsel"] # #1031276
- variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
  variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
 
 Test: quiet
diff --git a/mmdebstrap b/mmdebstrap
index dd168df..aeebebb 100755
--- a/mmdebstrap
+++ b/mmdebstrap
@@ -23,7 +23,7 @@
 use strict;
 use warnings;
 
-our $VERSION = '1.5.1';
+our $VERSION = '1.5.2';
 
 use English;
 use Getopt::Long;
@@ -4293,8 +4293,8 @@ sub get_sourceslist_by_suite {
         # the security mirror changes, starting with bullseye
         # https://lists.debian.org/87r26wqr2a.fsf@43-1.org
         my $bullseye_or_later = 0;
-        if (any { $_ eq $suite } ('stable', 'bullseye', 'bookworm', 'trixie'))
-        {
+        if (any { $_ eq $suite }
+            ('oldstable', 'stable', 'bullseye', 'bookworm', 'trixie')) {
             $bullseye_or_later = 1;
         }
         my $distro_info = '/usr/share/distro-info/debian.csv';
@@ -5737,6 +5737,9 @@ sub main() {
         if (!can_execute 'mke2fs', '-V') {
             error "need mke2fs for ext4 format";
         }
+        require DynaLoader;
+        my $libarchive = DynaLoader::dl_load_file("libarchive.so.13", 0)
+          or error "need libarchive for ext4 format";
         $blocksize = 4096;
     } elsif ($options->{format} eq 'squashfs') {
         if (!can_execute 'tar2sqfs') {
@@ -6970,13 +6973,29 @@ The above uses C<--map-auto> to map the block of user/group ids for the
 effective user/group to a block starting at user/group ID 0. We also want to
 map the current effective user/group ID into the subuid/subgid range using
 C<--map-user> and C<--map-group>, respectively. But if that uid/gid overlaps
-with the respective range, a "hole" will be removed from the mapping and the
-remaining uid/gid values will get shifted. Thus, we map the current effective
-user/group ID to the highest possible uid/gid, putting them at the end. Since
-that means that the user/group will be "nobody" and not "root" inside the
-namespace, C<--keep-caps> propagate permitted capabilities into the ambient set
-and thus give the user C<CAP_DAC_OVERRIDE> and other capabilities that it
-would've had.
+with the respective range, a "hole" will be removed from the mapping by the
+L<unshare(1)> utility and the remaining uid/gid values will get shifted. Thus,
+we map the current effective user/group ID to the highest possible uid/gid,
+putting them at the end. Since that means that the user/group will be "nobody"
+and not "root" inside the namespace, C<--keep-caps> propagate permitted
+capabilities into the ambient set and thus give the user C<CAP_DAC_OVERRIDE>
+and other capabilities that it would've had. The following does B<NOT> work:
+
+    $ unshare --map-root-user --map-auto ...            # or equavalient:
+    $ unshare --map-user=0 --map-group=0 --map-users=auto --map-groups=auto ...
+
+Using the format of L<lxc-usernsexec(1)>, these options will result in the
+following mapping:
+
+    0:$UID:1 + 1:$SUBUIDBASE:65535
+
+So a hole is punched in the automatically mapped range at the very beginning to
+make space for the user id of the current user as root inside the namespace and
+the remaining uids are shifted. With C<--map-user=65536> the mapping is as
+follows and places the current user at the end of the range, preventing any
+shifting:
+
+    0:$SUBUIDBASE:65536 + 65536:$UID:1
 
 Lastly, if you don't mind using superuser privileges and have systemd-nspawn
 available and you know your subuid/subgid offset (100000 in this example):
diff --git a/mmdebstrap-autopkgtest-build-qemu b/mmdebstrap-autopkgtest-build-qemu
index 2d1b803..294ab58 100755
--- a/mmdebstrap-autopkgtest-build-qemu
+++ b/mmdebstrap-autopkgtest-build-qemu
@@ -29,10 +29,11 @@ B<mmdebstrap-autopkgtest-build-qemu> [I<OPTIONS>] B<--boot>=B<efi> I<RELEASE> I<
 B<mmdebstrap-autopkgtest-build-qemu> is a mostly compatible drop-in replacement
 for L<autopkgtest-build-qemu(1)> with two main differences: Firstly, it uses
 L<mmdebstrap(1)> instead of L<vmdb2(1)> and thus is able to create QEMU disk
-images without requiring superuser privileges.  Secondly, it uses
-L<systemd-boot(7)> and thus only supports booting via EFI. For architectures
-for which L<autopkgtest-virt-qemu(1)> does not default to EFI booting you must
-pass B<--boot=efi> when invoking the autopkgtest virt backend.
+images without requiring superuser privileges and with bit-by-bit reproducible
+output. Secondly, it uses L<systemd-boot(7)> and thus only supports booting via
+EFI. For architectures for which L<autopkgtest-virt-qemu(1)> does not default
+to EFI booting you must pass B<--boot=efi> when invoking the autopkgtest virt
+backend.
 
 =head1 POSITIONAL PARAMETERS
 
@@ -110,6 +111,12 @@ all path components or by creating the image in a world-readable directory like
 Make sure to add B<--boot=efi> to both the B<mmdebstrap-autopkgtest-build-qemu>
 as well as the B<autopkgtest-virt-qemu> invocation.
 
+Create bit-by-bit reproducible images from a given snapshot.d.o timestamp.
+
+    SOURCE_DATE_EPOCH=1612543740 mmdebstrap-autopkgtest-build-qemu --boot=efi \
+        --mirror=http://snapshot.debian.org/archive/debian/20210205T164900Z/ \
+        unstable /path/to/debian-unstable.img
+
 =head1 SEE ALSO
 
 L<autopkgtest-build-qemu(1)>, L<autopkgtest-virt-qemu(1)>, L<mmdebstrap(1)>, L<autopkgtest(1)>
@@ -235,26 +242,31 @@ case "$ARCHITECTURE" in
 		EFIIMG=bootx64.efi
 		QEMUARCH=x86_64
 		VMFPKG=ovmf
+		LINUXIMAGE=linux-image-amd64
 	;;
 	arm64)
 		EFIIMG=bootaa64.efi
 		QEMUARCH=aarch64
 		VMFPKG=qemu-efi-aarch64
+		LINUXIMAGE=linux-image-arm64
 	;;
 	armhf)
 		EFIIMG=bootarm.efi
 		QEMUARCH=arm
 		VMFPKG=qemu-efi-arm
+		LINUXIMAGE=linux-image-armmp
 	;;
 	i386)
 		EFIIMG=bootia32.efi
 		QEMUARCH=i386
 		VMFPKG=ovmf-ia32
+		LINUXIMAGE=linux-image-686-pae
 	;;
 	riscv64)
 		EFIIMG=bootriscv64.efi
 		QEMUARCH=riscv64
 		VMFPKG=
+		LINUXIMAGE=linux-image-riscv64
 	;;
 	*)
 		die "unsupported architecture: $ARCHITECTURE"
@@ -268,7 +280,7 @@ test_installed() {
 	fi
 }
 
-for pkg in autopkgtest dosfstools e2fsprogs fdisk mount mtools passwd uidmap; do
+for pkg in autopkgtest dosfstools e2fsprogs fdisk mount mtools passwd uidmap libarchive13; do
 	test_installed "$pkg"
 done
 
@@ -315,36 +327,27 @@ FAT_SIZE_SECTORS=$((1024*254))
 #  - snapshots and overlays work just as well with raw images
 #  - users who prefer qcow2 get to choose to run it themselves with their own
 #    custom options like compression
-#
-# --map-users=auto --map-user=0     => 0:$UID:1 + 1:$SUBUIDBASE:65535
-# --map-users=auto --map-user=65536 => 0:$SUBUIDBASE:65536 + 65536:$UID:1
-#
-# Make the image writeable to the first subgid. mmdebstrap will map this gid to
-# the root group. unshare instead will map the current gid to 0 and the first
-# subgid to 1. Therefore mmdebstrap will be able to write to the image.
-rm -f "$IMAGE"
-: >"$IMAGE"
-unshare --map-user=0 --map-group=0 --map-groups=auto chown 0:1 "$IMAGE"
-chmod 0660 "$IMAGE"
-
-# Make sure that the unshared user is able to access the file.
-# Alternatively to using /sbin/mkfs.ext4 could use --format=ext2 which would
-# add an extra copy operation and come with the limitations of ext2.
-# Another solution: https://github.com/tytso/e2fsprogs/pull/118
-if ! mmdebstrap --unshare-helper touch "$IMAGE"; then
-	die "$IMAGE cannot be accessed by the unshared user -- either make all path components up to the image itself world-executable or place the image into a world-readable path like /tmp"
-fi
 
 set -- \
 	--mode=unshare \
+	--format=tar \
 	--variant=important \
 	--architecture="$ARCHITECTURE"
 
-test "$RELEASE" = jessie &&
+case $MIRROR in http://snapshot.debian.org/archive/*|https://snapshot.debian.org/archive/*)
+	set -- "$@" --aptopt='Acquire::Check-Valid-Until "false"';;
+esac
+
+EXT_FEATURES=
+if test "$RELEASE" = jessie; then
+	set -- "$@" --keyring=/usr/share/keyrings/debian-archive-removed-keys.gpg
+	set -- "$@" --aptopt='Apt::Key::gpgvcommand "/usr/libexec/mmdebstrap/gpgvnoexpkeysig"'
 	set -- "$@" --hook-dir=/usr/share/mmdebstrap/hooks/jessie-or-older
+	EXT_FEATURES="^metadata_csum,^metadata_csum_seed,^orphan_file"
+fi
 
 set -- "$@" \
-	"--include=init,linux-image-$ARCHITECTURE,python3" \
+	"--include=init,$LINUXIMAGE,python3" \
 	'--customize-hook=echo host >"$1/etc/hostname"' \
 	'--customize-hook=echo 127.0.0.1 localhost host >"$1/etc/hosts"' \
 	'--customize-hook=passwd --root "$1" --delete root' \
@@ -359,30 +362,35 @@ if test -n "$SCRIPT"; then
 		'--customize-hook=rm -f "$1/userscript"'
 fi
 
-EXT4_OFFSET_BYTES=$(( (FAT_OFFSET_SECTORS + FAT_SIZE_SECTORS) * 512))
-EXT4_OPTIONS="offset=$EXT4_OFFSET_BYTES,assume_storage_prezeroed=1"
-
-# the --no-mtab option to mount is a workaround for https://github.com/util-linux/util-linux/issues/2981
-# revert 8c0ddc32660ca4e98c988966251f9c05d6bcccef once it is no longer needed
 set -- "$@" \
 	"--customize-hook=download vmlinuz '$WORKDIR/kernel'" \
 	"--customize-hook=download initrd.img '$WORKDIR/initrd'" \
-	'--customize-hook=mount --no-mtab --bind "$1" "$1/mnt"' \
-	'--customize-hook=mount --no-mtab --bind "$1/mnt/mnt" "$1/mnt/dev"' \
-	'--customize-hook=/sbin/mkfs.ext4 -d "$1/mnt" -L autopkgtestvm -E '"'$EXT4_OPTIONS' '$IMAGE' '$SIZE'" \
-	'--customize-hook=umount --lazy --no-mtab "$1/mnt/dev"' \
-	'--customize-hook=umount --lazy --no-mtab "$1/mnt"' \
 	"$RELEASE" \
-	/dev/null
+	-
 
 test -n "$MIRROR" && set -- "$@" "$MIRROR"
 test -n "$KEYRING" && set -- "$@" "--keyring=$KEYRING"
 
-echo "mmdebstrap $*"
-mmdebstrap "$@" || die "mmdebstrap failed"
+echo "+ mmdebstrap $*" >&2
+# https://github.com/koalaman/shellcheck/issues/2555
+# shellcheck disable=SC3040
+set -o pipefail
+mmdebstrap "$@" | {
+	set -- -t ext4 -L autopkgtestvm -d -
+	if test -n "$EXT_FEATURES"; then
+		set -- "$@" -O "$EXT_FEATURES"
+	fi
+	EXTOPTS="offset=$(( (FAT_OFFSET_SECTORS + FAT_SIZE_SECTORS) * 512))"
+	if test -n "${SOURCE_DATE_EPOCH:-}"; then
+		uuid="$(uuidgen --sha1 --namespace="$(uuidgen --sha1 --namespace='@dns' --name mister-muffin.de)" --name "$SOURCE_DATE_EPOCH")"
+		set -- "$@" -U "$uuid"
+		EXTOPTS="$EXTOPTS,hash_seed=$uuid"
+	fi
+	set -- "$@" -E "$EXTOPTS" "$IMAGE" "$SIZE"
+	echo "+ mke2fs $*" >&2
+	/sbin/mke2fs "$@"
+}
 
-unshare -U -r --map-groups=auto chown 0:0 "$IMAGE"
-chmod "$(printf %o "$(( 0666 & ~0$(umask) ))")" "$IMAGE"
 
 echo "root=LABEL=autopkgtestvm rw console=ttyS0" > "$WORKDIR/cmdline"
 
@@ -456,4 +464,5 @@ if test -n "$VMFPKG" && test "$(dpkg-query -f '${db:Status-Status}' -W "$VMFPKG"
 	echo "I: you might need to install $VMFPKG to use this image with autopkgtest-virt-qemu" >&2
 fi
 
-echo "I: don't forget to pass --boot=efi when running autopkgtest-virt-qemu with this image" >&2
+echo "I: SUCCESS! Your new image can be found here: $IMAGE" >&2
+echo "I: Don't forget to pass --boot=efi when running autopkgtest-virt-qemu with this image" >&2
diff --git a/tests/check-against-debootstrap-dist b/tests/check-against-debootstrap-dist
index b5706c6..40e5798 100644
--- a/tests/check-against-debootstrap-dist
+++ b/tests/check-against-debootstrap-dist
@@ -70,6 +70,8 @@ rm /tmp/debian-{{ DIST }}-debootstrap/var/cache/debconf/templates.dat-old \
 	/tmp/debian-{{ DIST }}-mm/var/cache/debconf/templates.dat-old
 rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/status-old \
 	/tmp/debian-{{ DIST }}-mm/var/lib/dpkg/status-old
+rm -f /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/diversions-old \
+	/tmp/debian-{{ DIST }}-mm/var/lib/dpkg/diversions-old
 # remove dpkg files
 rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/available
 rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/cmethopt
@@ -155,24 +157,44 @@ if [ "{{ VARIANT }}" = "-" ] && [ "{{ DIST}}" = oldstable ]; then
 fi
 
 for log in faillog lastlog; do
-	if ! cmp /tmp/debian-{{ DIST }}-debootstrap/var/log/$log /tmp/debian-{{ DIST }}-mm/var/log/$log >&2;then
+	f1="/tmp/debian-{{ DIST }}-debootstrap/var/log/$log"
+	f2="/tmp/debian-{{ DIST }}-mm/var/log/$log"
+	# skip cmp if file is absent in both chroots
+	if [ ! -e "$f1" ] && [ ! -e "$f2" ]; then
+		continue
+	fi
+	if ! cmp "$f1" "$f2" >&2;then
 		# if the files differ, make sure they are all zeroes
-		cmp -n "$(stat -c %s "/tmp/debian-{{ DIST }}-debootstrap/var/log/$log")" "/tmp/debian-{{ DIST }}-debootstrap/var/log/$log" /dev/zero >&2
-		cmp -n "$(stat -c %s "/tmp/debian-{{ DIST }}-mm/var/log/$log")" "/tmp/debian-{{ DIST }}-mm/var/log/$log" /dev/zero >&2
+		cmp -n "$(stat -c %s "$f1")" "$f1" /dev/zero >&2
+		cmp -n "$(stat -c %s "$f2")" "$f2" /dev/zero >&2
 		# then delete them
-		rm /tmp/debian-{{ DIST }}-debootstrap/var/log/$log /tmp/debian-{{ DIST }}-mm/var/log/$log
+		rm "$f1" "$f2"
 	fi
 done
 
-# the order in which systemd and cron get installed differ and thus the order
-# of lines in /etc/group and /etc/gshadow differs
 if [ "{{ VARIANT }}" = "-" ]; then
+	# the order in which systemd and cron get installed differ and thus the order
+	# of lines in /etc/group and /etc/gshadow differs
 	for f in group group- gshadow gshadow-; do
 		for d in mm debootstrap; do
 			sort /tmp/debian-{{ DIST }}-$d/etc/$f > /tmp/debian-{{ DIST }}-$d/etc/$f.bak
 			mv /tmp/debian-{{ DIST }}-$d/etc/$f.bak /tmp/debian-{{ DIST }}-$d/etc/$f
 		done
 	done
+	# the order in which systemd and passwd get installed differ and thus
+	# the order of lines in /etc/shadow and /etc/shadow- differs
+	for f in shadow shadow-; do
+		for d in mm debootstrap; do
+			sort /tmp/debian-{{ DIST }}-$d/etc/$f > /tmp/debian-{{ DIST }}-$d/etc/$f.bak
+			mv /tmp/debian-{{ DIST }}-$d/etc/$f.bak /tmp/debian-{{ DIST }}-$d/etc/$f
+		done
+	done
+	# and since the order was different, ignore the *- files
+	for f in shadow- passwd-; do
+		for d in mm debootstrap; do
+			rm /tmp/debian-{{ DIST }}-$d/etc/$f
+		done
+	done
 fi
 
 # since debootstrap 1.0.133 there is no tzdata in the buildd variant and thus
diff --git a/tests/debootstrap b/tests/debootstrap
index 63c217d..9e1709e 100644
--- a/tests/debootstrap
+++ b/tests/debootstrap
@@ -5,6 +5,12 @@ export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
 
 tmpdir="$(mktemp -d)"
 chmod 755 "$tmpdir"
-debootstrap "$([ "{{ DIST }}" = oldstable ] && echo --no-merged-usr || echo --merged-usr)" --variant={{ VARIANT }} {{ DIST }} "$tmpdir" {{ MIRROR }}
+ret=0
+debootstrap "$([ "{{ DIST }}" = oldstable ] && echo --no-merged-usr || echo --merged-usr)" --variant={{ VARIANT }} {{ DIST }} "$tmpdir" {{ MIRROR }} || ret=$?
+if [ "$ret" -ne 0 ]; then
+	echo "E: debootstrap failed, dumping $tmpdir/debootstrap/debootstrap.log"
+	cat "$tmpdir/debootstrap/debootstrap.log"
+	exit 1
+fi
 tar --sort=name --mtime=@$SOURCE_DATE_EPOCH --clamp-mtime --numeric-owner --one-file-system --xattrs -C "$tmpdir" -c . > "./cache/debian-{{ DIST }}-{{ VARIANT }}.tar"
 rm -r "$tmpdir"
diff --git a/tests/debug b/tests/debug
index 5612115..bce3fd2 100644
--- a/tests/debug
+++ b/tests/debug
@@ -7,9 +7,7 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
 
 # we use variant standard in verbose mode to see the maximum number of packages
 # that was chosen in case of USE_HOST_APT_CONFIG=yes
-# we use variant important on arches where variant standard is not bit-by-bit
-# reproducible due to #1031276
-case {{ VARIANT }} in standard|-) : ;; *) exit 1;; esac
+case {{ VARIANT }} in standard) : ;; *) exit 1;; esac
 
 {{ CMD }} --variant={{ VARIANT }} --debug {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 
diff --git a/tests/eatmydata-via-hook-dir b/tests/eatmydata-via-hook-dir
index 0df72df..b554508 100644
--- a/tests/eatmydata-via-hook-dir
+++ b/tests/eatmydata-via-hook-dir
@@ -37,7 +37,6 @@ elfheader="$elfheader\\001\\000"
  tar -C /tmp/debian-chroot --one-file-system -c . \
 	 | tar -t \
 	 | sort \
-	 | grep -v '^\./var/lib/dpkg/diversions\(-old\)\?$' \
 	 | diff -u tar1.txt -
 rm /tmp/checkeatmydata.sh
 rm -r /tmp/debian-chroot
diff --git a/tests/empty-sources.list b/tests/empty-sources.list
index bf384f3..f9a6a6a 100644
--- a/tests/empty-sources.list
+++ b/tests/empty-sources.list
@@ -3,6 +3,6 @@ set -eu
 export LC_ALL=C.UTF-8
 trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
 printf '' | {{ CMD }} --mode={{ MODE }} --variant=apt \
-	--setup-hook='echo "deb {{ MIRROR }} {{ DIST }} main" > "$1"/etc/apt/sources.list' \
+	--setup-hook='test -e "$1"/etc/apt/sources.list || echo "deb {{ MIRROR }} {{ DIST }} main" > "$1"/etc/apt/sources.list' \
 	{{ DIST }} /tmp/debian-chroot.tar -
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
diff --git a/tests/jessie-or-older b/tests/jessie-or-older
index a3a2ace..db941b6 100644
--- a/tests/jessie-or-older
+++ b/tests/jessie-or-older
@@ -30,6 +30,7 @@ filter() {
 		--path-exclude=/var/cache/debconf/templates.dat-old \
 		--path-exclude=/var/lib/dpkg/available \
 		--path-exclude=/var/lib/dpkg/diversions \
+		--path-exclude=/var/lib/dpkg/diversions-old \
 		--path-exclude=/var/lib/dpkg/cmethopt \
 		--path-exclude=/var/lib/dpkg/status-old \
 		--path-exclude=/var/lib/shells.state
diff --git a/tests/missing-dev-sys-proc-inside-the-chroot b/tests/missing-dev-sys-proc-inside-the-chroot
index 9c9b1c0..935ff31 100644
--- a/tests/missing-dev-sys-proc-inside-the-chroot
+++ b/tests/missing-dev-sys-proc-inside-the-chroot
@@ -17,8 +17,8 @@ if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto
 	prefix="runuser -u ${SUDO_USER:-user} --"
 fi
 
-# creating /sbin manually because of #1071078
+# do not install base-files, so that /proc, /sys and /dev are missing
 $prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} \
-	--setup-hook='mkdir "$1/sbin"' \
+	--setup-hook='for dir in bin lib lib32 lib64 libo32 libx32 sbin; do ln -s "usr/$dir" "$1/$dir"; done' \
 	--include=dpkg,dash,diffutils,coreutils,libc-bin,sed \
 	{{ DIST }} /dev/null {{ MIRROR }}
diff --git a/tests/pivot_root b/tests/pivot_root
index 860c41b..7aa025e 100644
--- a/tests/pivot_root
+++ b/tests/pivot_root
@@ -17,19 +17,19 @@ if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto
 fi
 
 $prefix {{ CMD }} --mode={{ MODE }} --variant=apt \
-	--include=mount \
+	--include=mount,perl \
 	{{ DIST }} /tmp/chroot1.tar {{ MIRROR }}
 
 if [ {{ MODE }} = "unshare" ]; then
 	# calling pivot_root in root mode does not work for mysterious reasons:
 	# pivot_root: failed to change root from `.' to `mnt': Invalid argument
-	$prefix {{ CMD }} --mode={{ MODE }} --variant=apt --include=mount \
+	$prefix {{ CMD }} --mode={{ MODE }} --variant=apt --include=mount,perl \
 		--customize-hook='mkdir -p "$1/mnt" "$1/oldroot"' \
 		--customize-hook='[ ! -e /usr/bin/mmdebstrap ] || cp -aT /usr/bin/mmdebstrap "$1/usr/bin/mmdebstrap"' \
 		--customize-hook='[ ! -e ./mmdebstrap ] || cp -aT ./mmdebstrap "$1/mnt/mmdebstrap"' \
 		--customize-hook='mount -o rbind "$1" /mnt && cd /mnt && /sbin/pivot_root . oldroot' \
 		--customize-hook='unshare -U echo nested unprivileged unshare' \
-		--customize-hook='env --chdir=/mnt {{ CMD }} --mode=unshare --variant=apt --include=mount {{ DIST }} /tmp/chroot3.tar {{ MIRROR }}' \
+		--customize-hook='env --chdir=/mnt {{ CMD }} --mode=unshare --variant=apt --include=mount,perl {{ DIST }} /tmp/chroot3.tar {{ MIRROR }}' \
 		--customize-hook='copy-out /tmp/chroot3.tar /tmp' \
 		--customize-hook='rm -f "/usr/bin/mmdebstrap" "/mnt/mmdebstrap"' \
 		--customize-hook='umount -l oldroot sys' \
@@ -41,11 +41,11 @@ if [ {{ MODE }} = "unshare" ]; then
 		rm /tmp/chroot2.tar /tmp/chroot3.tar
 fi
 
-$prefix {{ CMD }} --mode={{ MODE }} --variant=apt --include=mount \
+$prefix {{ CMD }} --mode={{ MODE }} --variant=apt --include=mount,perl \
 	--customize-hook='mkdir -p "$1/mnt"' \
 	--customize-hook='[ ! -e /usr/bin/mmdebstrap ] || cp -aT /usr/bin/mmdebstrap "$1/usr/bin/mmdebstrap"' \
 	--customize-hook='[ ! -e ./mmdebstrap ] || cp -aT ./mmdebstrap "$1/mnt/mmdebstrap"' \
-	--chrooted-customize-hook='env --chdir=/mnt {{ CMD }} --mode=unshare --variant=apt --include=mount {{ DIST }} /tmp/chroot3.tar {{ MIRROR }}' \
+	--chrooted-customize-hook='env --chdir=/mnt {{ CMD }} --mode=unshare --variant=apt --include=mount,perl {{ DIST }} /tmp/chroot3.tar {{ MIRROR }}' \
 	--customize-hook='copy-out /tmp/chroot3.tar /tmp' \
 	--customize-hook='rm -f "$1/usr/bin/mmdebstrap" "$1/mnt/mmdebstrap"' \
 	{{ DIST }} /tmp/chroot2.tar {{ MIRROR }}
diff --git a/tests/verbose b/tests/verbose
index b0b0fb9..5a36310 100644
--- a/tests/verbose
+++ b/tests/verbose
@@ -7,9 +7,7 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
 
 # we use variant standard in verbose mode to see the maximum number of packages
 # that was chosen in case of USE_HOST_APT_CONFIG=yes
-# we use variant important on arches where variant standard is not bit-by-bit
-# reproducible due to #1031276
-case {{ VARIANT }} in standard|-) : ;; *) exit 1;; esac
+case {{ VARIANT }} in standard) : ;; *) exit 1;; esac
 
 {{ CMD }} --variant={{ VARIANT }} --verbose {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}