summaryrefslogtreecommitdiffstats
path: root/mmdebstrap-autopkgtest-build-qemu
blob: 19175e5c04c13cca05f9bd1ce614fb7e7fca7eac (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
#!/bin/sh
# Copyright 2023 Johannes Schauer Marin Rodrigues <josch@debian.org>
# Copyright 2023 Helmut Grohne <helmut@subdivi.de>
# SPDX-License-Identifier: MIT

# We generally use single quotes to avoid variable expansion:
# shellcheck disable=SC2016

# Replacement for autopkgtest-build-qemu and vmdb2 for all architectures
# supporting EFI booting (amd64, arm64, armhf, i386, riscv64).
# For use as replacement for autopkgtest-build-qemu and vmdb2 on ppc64el which
# neither supports extlinux nor efi booting there is an unmaintained script
# which uses grub instead to boot:
#
#     https://gitlab.mister-muffin.de/josch/mmdebstrap/src/commit/
#     e523741610a4ed8579642bfc755956f64c847ef3/mmdebstrap-autopkgtest-build-qemu

: <<'POD2MAN'
=head1 NAME

mmdebstrap-autopkgtest-build-qemu - autopkgtest-build-qemu without vmdb2 but mmdebstrap and EFI boot

=head1 SYNOPSIS

B<mmdebstrap-autopkgtest-build-qemu> [I<OPTIONS>] B<--boot>=B<efi> I<RELEASE> I<IMAGE>

=head1 DESCRIPTION

B<mmdebstrap-autopkgtest-build-qemu> is a mostly compatible drop-in replacement
for L<autopkgtest-build-qemu(1)> with two main differences: Firstly, it uses
L<mmdebstrap(1)> instead of L<vmdb2(1)> and thus is able to create QEMU disk
images without requiring superuser privileges.  Secondly, it uses
L<systemd-boot(7)> and thus only supports booting via EFI. For architectures
for which L<autopkgtest-virt-qemu(1)> does not default to EFI booting you must
pass B<--boot=efi> when invoking the autopkgtest virt backend.

=head1 POSITIONAL PARAMETERS

=over 8

=item I<RELEASE>

The release to download from the I<MIRROR>.  This parameter is required.

=item I<IMAGE>

The file to write, in raw format. This parameter is required.

=back

=head1 OPTIONS

=over 8

=item B<--mirror>=I<MIRROR>

Specify  which  distribution  to install.  It defaults to
http://deb.debian.org/debian (i.e. Debian), but you can pass a mirror of any
Debian derivative.

=item B<--architecture>=I<ARCHITECTURE>

Set the architecture for the virtual machine image, specified as a L<dpkg(1)>
architecture.  If  omitted, the host architecture is assumed.

B<--arch>=I<ARCH> is an alias for this option.

=item B<--script>=I<SCRIPT>

Specifies a user script that will be called with the root filesystem of the
image as its first parameter. This script can them make any necesssary
modifications to the root filesystem.

The script must be a POSIX shell script, and should not depend on bash-specific
features. This  script will be executed inside a L<chroot(1)> call in the
virtual machine root filesystem.

=item B<--size>=I<SIZE>

Specifies the image size for the virtual machine, defaulting to 25G.

=item B<--apt-proxy>=I<PROXY>

Specify  an  apt proxy to use in the virtual machine.  By default, if you have
an apt proxy configured on the host, the virtual machine will automatically use
this, otherwise there is no default.

=item B<--boot>=B<efi>, B<--efi>

Select the way the generated image will expect to be booted. Unless you
explicitly select --boot=efi, operation will fail.

=item B<--keyring>=I<KEYRING>

Passes an additional B<--keyring> parameter to B<mmdebstrap>.

=back

=head1 EXAMPLES

Make sure, that F</path/to/debian-unstable.img> is a path that the unshared
user has access to. This can be done by ensuring world-execute permissions on
all path components or by creating the image in a world-readable directory like
/tmp before copying it into its final location.

    $ mmdebstrap-autopkgtest-build-qemu --boot=efi --arch=amd64 unstable /path/to/debian-unstable.img
    [...]
    $ autopkgtest mypackage -- qemu --boot=efi --dpkg-architecture=amd64 /path/to/debian-unstable.img

Make sure to add B<--boot=efi> to both the B<mmdebstrap-autopkgtest-build-qemu>
as well as the B<autopkgtest-virt-qemu> invocation.

=head1 SEE ALSO

L<autopkgtest-build-qemu(1)>, L<autopkgtest-virt-qemu(1)>, L<mmdebstrap(1)>, L<autopkgtest(1)>

=cut
POD2MAN

set -eu

die() {
	echo "$*" 1>&2
	exit 1
}
usage() {
	die "usage: $0 [--architecture=|--apt-proxy=|--keyring=|--mirror=|--script=|--size=] --boot=efi <RELEASE> <IMAGE>"
}
usage_error() {
	echo "error: $*" 1>&2
	usage
}

BOOT=auto
ARCHITECTURE=$(dpkg --print-architecture)
IMAGE=
MIRROR=
KEYRING=
RELEASE=
SIZE=25G
SCRIPT=

# consumed by setup-testbed
export AUTOPKGTEST_BUILD_QEMU=1

opt_boot() {
	BOOT="$1"
}
opt_architecture() {
	ARCHITECTURE="$1"
}
opt_arch() {
	ARCHITECTURE="$1"
}
opt_apt_proxy() {
	# consumed by setup-testbed
	export AUTOPKGTEST_APT_PROXY="$1"
	# consumed by mmdebstrap
	if test "$1" = DIRECT; then
		unset http_proxy
	else
		export http_proxy="$1"
	fi
}
opt_keyring() {
	KEYRING="$1"
}
opt_mirror() {
	# consumed by setup-testbed
	export MIRROR="$1"
}
opt_script() {
	test -f "$1" || die "passed script '$1' does not refer to a file"
	SCRIPT="$1"
}
opt_size() {
	SIZE="$1"
}

positional=1
positional_1() {
	# consumed by setup-testbed
	export RELEASE="$1"
}
positional_2() {
	IMAGE="$1"
}
positional_3() { opt_mirror "$@"; }
positional_4() { opt_architecture "$@"; }
positional_5() { opt_script "$@"; }
positional_6() { opt_size "$@"; }
positional_7() {
	die "too many positional options"
}

while test "$#" -gt 0; do
	case "$1" in
		--architecture=*|--arch=*|--boot=*|--keyring=*|--mirror=*|--script=*|--size=*)
			optname="${1%%=*}"
			"opt_${optname#--}" "${1#*=}"
		;;
		--apt-proxy=*)
			opt_apt_proxy "${1#*=}"
		;;
		--architecture|--arch|--boot|--keyring|--mirror|--script|--size)
			test "$#" -ge 2 || usage_error "missing argument for $1"
			"opt_${1#--}" "$2"
			shift
		;;
		--apt-proxy)
			test "$#" -ge 2 || usage_error "missing argument for $1"
			opt_apt_proxy "$2"
			shift
		;;
		--efi)
			opt_boot efi
		;;
		--*)
			usage_error "unrecognized argument $1"
		;;
		*)
			"positional_$positional" "$1"
			positional=$((positional + 1))
		;;
	esac
	shift
done

test -z "$RELEASE" -o -z "$IMAGE" && usage_error "missing positional arguments"
test "$BOOT" = efi ||
	die "this tool does not support boot modes other than efi"

case "$ARCHITECTURE" in
	amd64)
		EFIIMG=bootx64.efi
		QEMUARCH=x86_64
		VMFPKG=ovmf
	;;
	arm64)
		EFIIMG=bootaa64.efi
		QEMUARCH=aarch64
		VMFPKG=qemu-efi-aarch64
	;;
	armhf)
		EFIIMG=bootarm.efi
		QEMUARCH=arm
		VMFPKG=qemu-efi-arm
	;;
	i386)
		EFIIMG=bootia32.efi
		QEMUARCH=i386
		VMFPKG=ovmf-ia32
	;;
	riscv64)
		EFIIMG=bootriscv64.efi
		QEMUARCH=riscv64
		VMFPKG=
	;;
	*)
		die "unsupported architecture: $ARCHITECTURE"
	;;
esac

if test "$(dpkg-query -f '${db:Status-Status}' -W binutils-multiarch)" = installed; then
	GNU_PREFIX=
else
	GNU_ARCHITECTURE="$(dpkg-architecture "-a$ARCHITECTURE" -qDEB_HOST_GNU_TYPE)"
	GNU_PREFIX="$GNU_ARCHITECTURE-"
	GNU_SUFFIX="-$(echo "$GNU_ARCHITECTURE" | tr _ -)"
	test "$(dpkg-query -f '${db:Status-Status}' -W "binutils$GNU_SUFFIX")" = installed ||
		die "please install binutils$GNU_SUFFIX or binutils-multiarch"
fi

arches=" $(dpkg --print-architecture) $(dpkg --print-foreign-architectures | tr '\n' ' ') "
case $arches in
	*" $ARCHITECTURE "*) : ;; # nothing to do
	*) die "enable $ARCHITECTURE by running: sudo dpkg --add-architecture $ARCHITECTURE && sudo apt update" ;;
esac

for pkg in autopkgtest dosfstools e2fsprogs fdisk mount mtools passwd "systemd-boot-efi:$ARCHITECTURE" uidmap; do
	if [ "$(dpkg-query -f '${db:Status-Status}' -W "$pkg")" != installed ]; then
		die "please install $pkg"
	fi
done

BOOTSTUB="/usr/lib/systemd/boot/efi/linux${EFIIMG#boot}.stub"

WORKDIR=

cleanup() {
	test -n "$WORKDIR" && rm -Rf "$WORKDIR"
}

trap cleanup EXIT INT TERM QUIT

WORKDIR=$(mktemp -d)

FAT_OFFSET_SECTORS=$((1024*2))
FAT_SIZE_SECTORS=$((1024*254))

# The image is raw and not in qcow2 format because:
#  - faster run-time as the "qemu-image convert" step is not needed
#  - image can be used independent of qemu tooling
#  - modifying the image just with "mount" instead of requiring qemu-nbd
#  - sparse images make the file just as small as with qcow2
#  - trim support is more difficult on qcow2
#  - snapshots and overlays work just as well with raw images
#  - users who prefer qcow2 get to choose to run it themselves with their own
#    custom options like compression
#
# Make the image writeable to the first subgid. mmdebstrap will map this gid to
# the root group. unshare instead will map the current gid to 0 and the first
# subgid to 1. Therefore mmdebstrap will be able to write to the image.
rm -f "$IMAGE"
: >"$IMAGE"
unshare -U -r --map-groups=auto chown 0:1 "$IMAGE"
chmod 0660 "$IMAGE"

# Make sure that the unshared user is able to access the file.
# Alternatively to using /sbin/mkfs.ext4 could use --format=ext2 which would
# add an extra copy operation and come with the limitations of ext2.
# Another solution: https://github.com/tytso/e2fsprogs/pull/118
if ! mmdebstrap --unshare-helper touch "$IMAGE"; then
	die "$IMAGE cannot be accessed by the unshared user -- either make all path components up to the image itself world-executable or place the image into a world-readable path like /tmp"
fi

set -- \
	--mode=unshare \
	--variant=important \
	--architecture="$ARCHITECTURE"

test "$RELEASE" = jessie &&
	set -- "$@" --hook-dir=/usr/share/mmdebstrap/hooks/jessie-or-older

set -- "$@" \
	"--include=init,linux-image-$ARCHITECTURE,python3" \
	'--customize-hook=echo host >"$1/etc/hostname"' \
	'--customize-hook=echo 127.0.0.1 localhost host >"$1/etc/hosts"' \
	'--customize-hook=passwd --root "$1" --delete root' \
	'--customize-hook=useradd --root "$1" --home-dir /home/user --create-home user' \
	'--customize-hook=passwd --root "$1" --delete user' \
	'--customize-hook=/usr/share/autopkgtest/setup-commands/setup-testbed'

if test -n "$SCRIPT"; then
	set -- "$@" \
		"--customize-hook=upload '$SCRIPT' /userscript" \
		"--chrooted-customize-hook=sh /userscript" \
		'--customize-hook=rm -f "$1/userscript"'
fi

EXT4_OFFSET_BYTES=$(( (FAT_OFFSET_SECTORS + FAT_SIZE_SECTORS) * 512))
EXT4_OPTIONS="offset=$EXT4_OFFSET_BYTES,assume_storage_prezeroed=1"
set -- "$@" \
	"--customize-hook=download vmlinuz '$WORKDIR/kernel'" \
	"--customize-hook=download initrd.img '$WORKDIR/initrd'" \
	'--customize-hook=mount --bind "$1" "$1/mnt"' \
	'--customize-hook=mount --bind "$1/mnt/mnt" "$1/mnt/dev"' \
	'--customize-hook=/sbin/mkfs.ext4 -d "$1/mnt" -L autopkgtestvm -E '"'$EXT4_OPTIONS' '$IMAGE' '$SIZE'" \
	'--customize-hook=umount --lazy "$1/mnt"' \
	"$RELEASE" \
	/dev/null

test -n "$MIRROR" && set -- "$@" "$MIRROR"
test -n "$KEYRING" && set -- "$@" "--keyring=$KEYRING"

echo "mmdebstrap $*"
mmdebstrap "$@" || die "mmdebstrap failed"

unshare -U -r --map-groups=auto chown 0:0 "$IMAGE"
chmod "$(printf %o "$(( 0666 & ~0$(umask) ))")" "$IMAGE"

echo "root=LABEL=autopkgtestvm rw console=ttyS0" > "$WORKDIR/cmdline"

align_size() {
	echo "$(( ($1) + ($2) - 1 - (($1) + ($2) - 1) % ($2) ))"
}

alignment=$("${GNU_PREFIX}objdump" -p "$BOOTSTUB" | sed 's/^SectionAlignment\s\+\([0-9]\)/0x/;t;d')
test -z "$alignment" && die "failed to discover the alignment of the efi stub"
echo "determined efi vma alignment as $alignment"
test "$RELEASE" = jessie -a "$((alignment))" -lt "$((1024*1024))" && {
	echo "increasing efi vma alignment for jessie"
	alignment=$((1024*1024))
}
lastoffset=0
# shellcheck disable=SC2034  # unused variables serve documentation
lastoffset="$("${GNU_PREFIX}objdump" -h "$BOOTSTUB" |
	while read -r idx name size vma lma fileoff algn behind; do
		test -z "$behind" -a "${algn#"2**"}" != "$algn" || continue
		offset=$(( 0x$vma + 0x$size ))
		test "$offset" -gt "$lastoffset" || continue
		lastoffset="$offset"
		echo "$lastoffset"
	done | tail -n1)"
lastoffset=$(align_size "$lastoffset" "$alignment")
echo "determined minimum efi vma offset as $lastoffset"

cmdline_size="$(stat -Lc%s "$WORKDIR/cmdline")"
cmdline_size="$(align_size "$cmdline_size" "$alignment")"
linux_size="$(stat -Lc%s "$WORKDIR/kernel")"
linux_size="$(align_size "$linux_size" "$alignment")"
cmdline_offset="$lastoffset"
linux_offset=$((cmdline_offset + cmdline_size))
initrd_offset=$((linux_offset + linux_size))

SOURCE_DATE_EPOCH=0 \
	"${GNU_PREFIX}objcopy" \
	--enable-deterministic-archives \
	--add-section .cmdline="$WORKDIR/cmdline" \
	--change-section-vma .cmdline="$(printf 0x%x "$cmdline_offset")" \
	--add-section .linux="$WORKDIR/kernel" \
	--change-section-vma .linux="$(printf 0x%x "$linux_offset")" \
	--add-section .initrd="$WORKDIR/initrd" \
	--change-section-vma .initrd="$(printf 0x%x "$initrd_offset")" \
	"$BOOTSTUB" "$WORKDIR/efiimg"

rm -f "$WORKDIR/kernel" "$WORKDIR/initrd"

truncate -s "$((FAT_SIZE_SECTORS * 512))" "$WORKDIR/fat"
/sbin/mkfs.fat -F 32 --invariant "$WORKDIR/fat"
mmd -i "$WORKDIR/fat" EFI EFI/BOOT
mcopy -i "$WORKDIR/fat" "$WORKDIR/efiimg" "::EFI/BOOT/$EFIIMG"

rm -f "$WORKDIR/efiimg"

truncate --size="+$((34*512))" "$IMAGE"
/sbin/sfdisk "$IMAGE" <<EOF
label: gpt
unit: sectors

start=$FAT_OFFSET_SECTORS, size=$FAT_SIZE_SECTORS, type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B
start=$((FAT_OFFSET_SECTORS + FAT_SIZE_SECTORS)), type=0FC63DAF-8483-4772-8E79-3D69D8477DE4
EOF

dd if="$WORKDIR/fat" of="$IMAGE" conv=notrunc,sparse bs=512 "seek=$FAT_OFFSET_SECTORS" status=none

if test "$(dpkg --print-architecture)" != "$ARCHITECTURE" && test "$(dpkg-query -f '${db:Status-Status}' -W "qemu-system-$QEMUARCH")" != installed; then
	echo "I: you might need to install a package providing qemu-system-$QEMUARCH to use this image with autopkgtest-virt-qemu" >&2
fi
if test -n "$VMFPKG" && test "$(dpkg-query -f '${db:Status-Status}' -W "$VMFPKG")" != installed; then
	echo "I: you might need to install $VMFPKG to use this image with autopkgtest-virt-qemu" >&2
fi

echo "I: don't forget to pass --boot=efi when running autopkgtest-virt-qemu with this image" >&2