diff options
Diffstat (limited to 'scripts/run_bisect_qemu.sh')
| -rwxr-xr-x | scripts/run_bisect_qemu.sh | 365 |
1 files changed, 365 insertions, 0 deletions
diff --git a/scripts/run_bisect_qemu.sh b/scripts/run_bisect_qemu.sh new file mode 100755 index 0000000..d0f8dcc --- /dev/null +++ b/scripts/run_bisect_qemu.sh @@ -0,0 +1,365 @@ +#!/bin/sh +# +# Copyright 2020 Johannes Schauer Marin Rodrigues <josch@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. + +# this script is part of debbisect and usually called by debbisect itself +# +# it accepts eight or ten arguments: +# 1. dependencies +# 2. script name or shell snippet +# 3. mirror URL +# 4. architecture +# 5. suite +# 6. components +# 7. memsize +# 8. disksize +# 9. (optional) second mirror URL +# 10. (optional) package to upgrade +# +# It will create an ephemeral qemu virtual machine using mmdebstrap and +# guestfish using (3.) as mirror, (4.) as architecture, (5.) as suite and +# (6.) as components, install the dependencies given in (1.) and execute the +# script given in (2.). +# Its output is the exit code of the script as well as a file ./pkglist +# containing the output of "dpkg-query -W" inside the chroot. +# +# If not only six but eight arguments are given, then the second mirror URL +# (9.) will be added to the apt sources and the single package (10.) will be +# upgraded to its version from (9.). +# +# shellcheck disable=SC2016 + +set -exu + +if [ $# -ne 8 ] && [ $# -ne 10 ]; then + echo "usage: $0 depends script mirror1 architecture suite components memsize disksize [mirror2 toupgrade]" + exit 1 +fi + +depends=$1 +script=$2 +mirror1=$3 +architecture=$4 +suite=$5 +components=$6 +memsize=$7 +disksize=$8 + +if [ $# -eq 10 ]; then + mirror2=$9 + toupgrade=${10} +fi + +case $architecture in + alpha) qemuarch=alpha;; + amd64) qemuarch=x86_64;; + arm) qemuarch=arm;; + arm64) qemuarch=aarch64;; + armel) qemuarch=arm;; + armhf) qemuarch=arm;; + hppa) qemuarch=hppa;; + i386) qemuarch=i386;; + m68k) qemuarch=m68k;; + mips) qemuarch=mips;; + mips64) qemuarch=mips64;; + mips64el) qemuarch=mips64el;; + mipsel) qemuarch=mipsel;; + powerpc) qemuarch=ppc;; + ppc64) qemuarch=ppc64;; + ppc64el) qemuarch=ppc64le;; + riscv64) qemuarch=riscv64;; + s390x) qemuarch=s390x;; + sh4) qemuarch=sh4;; + sparc) qemuarch=sparc;; + sparc64) qemuarch=sparc64;; + *) echo "no qemu support for $architecture"; exit 1;; +esac +case $architecture in + i386) linuxarch=686-pae;; + amd64) linuxarch=amd64;; + arm64) linuxarch=arm64;; + armhf) linuxarch=armmp;; + ia64) linuxarch=itanium;; + m68k) linuxarch=m68k;; + armel) linuxarch=marvell;; + hppa) linuxarch=parisc;; + powerpc) linuxarch=powerpc;; + ppc64) linuxarch=powerpc64;; + ppc64el) linuxarch=powerpc64le;; + riscv64) linuxarch=riscv64;; + s390x) linuxarch=s390x;; + sparc64) linuxarch=sparc64;; + *) echo "no kernel image for $architecture"; exit 1;; +esac + +TMPDIR=$(mktemp --tmpdir --directory debbisect_qemu.XXXXXXXXXX) +cleantmp() { + for f in customize.sh id_rsa id_rsa.pub qemu.log config; do + rm -f "$TMPDIR/$f" + done + rmdir "$TMPDIR" +} + +trap cleantmp EXIT +# the temporary directory must be world readable (for example in unshare mode) +chmod a+xr "$TMPDIR" + +ssh-keygen -q -t rsa -f "$TMPDIR/id_rsa" -N "" + +cat << SCRIPT > "$TMPDIR/customize.sh" +#!/bin/sh +set -exu + +rootfs="\$1" + +# setup various files in /etc +echo host > "\$rootfs/etc/hostname" +echo "127.0.0.1 localhost host" > "\$rootfs/etc/hosts" +echo "/dev/vda1 / auto errors=remount-ro 0 1" > "\$rootfs/etc/fstab" +cat /etc/resolv.conf > "\$rootfs/etc/resolv.conf" + +# setup users +chroot "\$rootfs" passwd --delete root +chroot "\$rootfs" useradd --home-dir /home/user --create-home user +chroot "\$rootfs" passwd --delete user + +# extlinux config to boot from /dev/vda1 with predictable network interface +# naming and a serial console for logging +cat << END > "\$rootfs/extlinux.conf" +default linux +timeout 0 + +label linux +kernel /vmlinuz +append initrd=/initrd.img root=/dev/vda1 net.ifnames=0 console=ttyS0 +END + +# network interface config +# we can use eth0 because we boot with net.ifnames=0 for predictable interface +# names +cat << END > "\$rootfs/etc/network/interfaces" +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet dhcp +END + +# copy in the public key +mkdir "\$rootfs/root/.ssh" +cp "$TMPDIR/id_rsa.pub" "\$rootfs/root/.ssh/authorized_keys" +chroot "\$rootfs" chown 0:0 /root/.ssh/authorized_keys +SCRIPT +chmod +x "$TMPDIR/customize.sh" + +# The following hacks are needed to go back as far as 2006-08-10: +# +# - Acquire::Check-Valid-Until "false" allows Release files with an expired +# Valid-Until dates +# - Apt::Key::gpgvcommand allows expired GPG keys +# - Apt::Hashes::SHA1::Weak "yes" allows GPG keys with weak SHA1 signature +# - /usr/share/keyrings lets apt use debian-archive-removed-keys.gpg +# - /usr/share/mmdebstrap/hooks/jessie-or-older performs some setup that is +# only required for Debian Jessie or older +# +mmdebstrap --architecture="$architecture" --verbose --variant=apt --components="$components" \ + --aptopt='Acquire::Check-Valid-Until "false"' \ + --aptopt='Apt::Key::gpgvcommand "/usr/libexec/mmdebstrap/gpgvnoexpkeysig"' \ + --aptopt='Apt::Hashes::SHA1::Weak "yes"' \ + --keyring=/usr/share/keyrings \ + --hook-dir=/usr/share/mmdebstrap/hooks/maybe-jessie-or-older \ + --hook-dir=/usr/share/mmdebstrap/hooks/maybe-merged-usr \ + --skip=check/signed-by \ + --include='openssh-server,systemd-sysv,ifupdown,netbase,isc-dhcp-client,udev,policykit-1,linux-image-'"$linuxarch" \ + --customize-hook="$TMPDIR/customize.sh" \ + "$suite" debian-rootfs.tar "$mirror1" + +# use guestfish to prepare the host system +# +# - create a single 4G partition and unpack the rootfs tarball into it +# - unpack the tarball of the container into / +# - put a syslinux MBR into the first 440 bytes of the drive +# - install extlinux and make partition bootable +# +# useful stuff to debug any errors: +# LIBGUESTFS_BACKEND_SETTINGS=force_tcg +# libguestfs-test-tool || true +# export LIBGUESTFS_DEBUG=1 LIBGUESTFS_TRACE=1 +guestfish -N "debian-rootfs.img=disk:$disksize" -- \ + part-disk /dev/sda mbr : \ + mkfs ext4 /dev/sda1 : \ + mount /dev/sda1 / : \ + tar-in "debian-rootfs.tar" / : \ + upload /usr/lib/SYSLINUX/mbr.bin /mbr.bin : \ + copy-file-to-device /mbr.bin /dev/sda size:440 : \ + rm /mbr.bin : \ + extlinux / : \ + sync : \ + umount / : \ + part-set-bootable /dev/sda 1 true : \ + shutdown + + +# start the host system +# prefer using kvm but fall back to tcg if not available +# avoid entropy starvation by feeding the crypt system with random bits from /dev/urandom +# the default memory size of 128 MiB is not enough for Debian, so we go with 1G +# use a virtio network card instead of emulating a real network device +# we don't need any graphics +# this also multiplexes the console and the monitor to stdio +# creates a multiplexed stdio backend connected to the serial port and the qemu +# monitor +# redirect tcp connections on port 10022 localhost to the host system port 22 +# redirect all output to a file +# run in the background +timeout --kill-after=60s 60m \ + qemu-system-"$qemuarch" \ + -M accel=kvm:tcg \ + -no-user-config \ + -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \ + -m "$memsize" \ + -net nic,model=virtio \ + -nographic \ + -serial mon:stdio \ + -net user,hostfwd=tcp:127.0.0.1:10022-:22 \ + -drive file="debian-rootfs.img",format=raw,if=virtio \ + > "$TMPDIR/qemu.log" </dev/null 2>&1 & + +# store the pid +QEMUPID=$! + +# use a function here, so that we can properly quote the path to qemu.log +showqemulog() { + cat --show-nonprinting "$TMPDIR/qemu.log" +} + +# show the log and kill qemu in case the script exits first +trap 'showqemulog; cleantmp; kill $QEMUPID' EXIT + +# the default ssh command does not store known hosts and even ignores host keys +# it identifies itself with the rsa key generated above +# pseudo terminal allocation is disabled or otherwise, programs executed via +# ssh might wait for input on stdin of the ssh process + +cat << END > "$TMPDIR/config" +Host qemu + Hostname 127.0.0.1 + User root + Port 10022 + UserKnownHostsFile /dev/null + StrictHostKeyChecking no + IdentityFile $TMPDIR/id_rsa + RequestTTY no +END + +TIMESTAMP=$(sleepenh 0 || [ $? -eq 1 ]) +TIMEOUT=5 +NUM_TRIES=40 +i=0 +while true; do + rv=0 + ssh -F "$TMPDIR/config" -o ConnectTimeout=$TIMEOUT qemu echo success || rv=1 + [ $rv -eq 0 ] && break + # if the command before took less than $TIMEOUT seconds, wait the remaining time + TIMESTAMP=$(sleepenh "$TIMESTAMP" "$TIMEOUT" || [ $? -eq 1 ]); + i=$((i+1)) + if [ $i -ge $NUM_TRIES ]; then + break + fi +done + +if [ $i -eq $NUM_TRIES ]; then + echo "timeout reached: unable to connect to qemu via ssh" + exit 1 +fi + +# if any url in sources.list points to 127.0.0.1 then we have to replace them +# by the host IP as seen by the qemu guest +cat << SCRIPT | ssh -F "$TMPDIR/config" qemu sh +set -eu +if [ -e /etc/apt/sources.list ]; then + sed -i 's/http:\/\/127.0.0.1:/http:\/\/10.0.2.2:/' /etc/apt/sources.list +fi +find /etc/apt/sources.list.d -type f -name '*.list' -print0 \ + | xargs --null --no-run-if-empty sed -i 's/http:\/\/127.0.0.1:/http:\/\/10.0.2.2:/' +SCRIPT + +# we install dependencies now and not with mmdebstrap --include in case some +# dependencies require a full system present +if [ -n "$depends" ]; then + ssh -F "$TMPDIR/config" qemu apt-get update + # shellcheck disable=SC2046,SC2086 + ssh -F "$TMPDIR/config" qemu env DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true apt-get --yes install --no-install-recommends $(echo $depends | tr ',' ' ') +fi + +# in its ten-argument form, a single package has to be upgraded to its +# version from the first bad timestamp +if [ $# -eq 10 ]; then + # replace content of sources.list with first bad timestamp + mirror2=$(echo "$mirror2" | sed 's/http:\/\/127.0.0.1:/http:\/\/10.0.2.2:/') + echo "deb $mirror2 $suite $(echo "$components" | tr ',' ' ')" | ssh -F "$TMPDIR/config" qemu "cat > /etc/apt/sources.list" + ssh -F "$TMPDIR/config" qemu apt-get update + # upgrade a single package (and whatever else apt deems necessary) + before=$(ssh -F "$TMPDIR/config" qemu dpkg-query -W) + ssh -F "$TMPDIR/config" qemu env DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true apt-get --yes install --no-install-recommends "$toupgrade" + after=$(ssh -F "$TMPDIR/config" qemu dpkg-query -W) + # make sure that something was upgraded + if [ "$before" = "$after" ]; then + echo "nothing got upgraded -- this should never happen" >&2 + exit 1 + fi + ssh -F "$TMPDIR/config" qemu dpkg-query -W > "./debbisect.$DEBIAN_BISECT_TIMESTAMP.$toupgrade.pkglist" +else + ssh -F "$TMPDIR/config" qemu dpkg-query -W > "./debbisect.$DEBIAN_BISECT_TIMESTAMP.pkglist" +fi + +ssh -F "$TMPDIR/config" qemu dpkg-query --list -no-pager + +# explicitly export all necessary variables +# because we use set -u this also makes sure that this script has these +# variables set in the first place +export DEBIAN_BISECT_EPOCH="$DEBIAN_BISECT_EPOCH" +export DEBIAN_BISECT_TIMESTAMP="$DEBIAN_BISECT_TIMESTAMP" +if [ -z ${DEBIAN_BISECT_MIRROR+x} ]; then + # DEBIAN_BISECT_MIRROR was unset (caching is disabled) + true +else + # replace the localhost IP by the IP of the host as seen by qemu + DEBIAN_BISECT_MIRROR=$(echo "$DEBIAN_BISECT_MIRROR" | sed 's/http:\/\/127.0.0.1:/http:\/\/10.0.2.2:/') + export DEBIAN_BISECT_MIRROR="$DEBIAN_BISECT_MIRROR" +fi + + +# either execute $script as a script from $PATH or as a shell snippet +ret=0 +if [ -x "$script" ] || echo "$script" | grep --invert-match --silent --perl-regexp '[^\w@\%+=:,.\/-]'; then + "$script" "$TMPDIR/config" || ret=$? +else + sh -c "$script" exec "$TMPDIR/config" || ret=$? +fi + +# since we installed systemd-sysv, systemctl is available +ssh -F "$TMPDIR/config" qemu systemctl poweroff + +wait $QEMUPID + +trap - EXIT + +showqemulog +cleantmp + +if [ "$ret" -eq 0 ]; then + exit 0 +else + exit 1 +fi |