diff options
25 files changed, 451 insertions, 291 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index fea8b9a..21ed927 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +1.5.1 (2024-06-03) +------------------ + + - in root and unshare mode, run 'mount --make-rprivate /' before bind-mounting + - switch apt variant from using 'apt-get dist-upgrade' to apt patterns + +1.5.0 (2024-05-14) +------------------ + + - add --format=ext4 + 1.4.3 (2024-02-01) ------------------ diff --git a/coverage.py b/coverage.py index 7e911cf..01ee7b5 100755 --- a/coverage.py +++ b/coverage.py @@ -34,7 +34,7 @@ all_variants = [ "standard", ] default_format = "auto" -all_formats = ["auto", "directory", "tar", "squashfs", "ext2", "null"] +all_formats = ["auto", "directory", "tar", "squashfs", "ext2", "ext4", "null"] mirror = os.getenv("mirror", "http://127.0.0.1/debian") hostarch = subprocess.check_output(["dpkg", "--print-architecture"]).decode().strip() diff --git a/coverage.txt b/coverage.txt index fb09b19..c040b8b 100644 --- a/coverage.txt +++ b/coverage.txt @@ -59,7 +59,7 @@ Needs-QEMU: true Test: mmdebstrap Needs-Root: true Modes: root -Formats: tar squashfs ext2 +Formats: tar squashfs ext2 ext4 Variants: essential apt minbase buildd - standard Skip-If: variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558 @@ -68,7 +68,7 @@ Skip-If: Test: check-for-bit-by-bit-identical-format-output Modes: unshare fakechroot -Formats: tar squashfs ext2 +Formats: tar squashfs ext2 ext4 Variants: essential apt minbase buildd - standard Skip-If: variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558 diff --git a/hooks/copy-host-apt-sources-and-preferences/setup00.sh b/hooks/copy-host-apt-sources-and-preferences/setup00.sh index 07caa78..9aafd78 100755 --- a/hooks/copy-host-apt-sources-and-preferences/setup00.sh +++ b/hooks/copy-host-apt-sources-and-preferences/setup00.sh @@ -1,4 +1,13 @@ #!/bin/sh +# +# This script makes sure that the apt sources.list and preferences from outside +# the chroot also exist inside the chroot by *appending* them to any existing +# files. If you do not want to keep the original content, add another setup +# hook before this one which cleans up the files you don't want to keep. +# +# If instead of copying sources.list verbatim you want to mangle its contents, +# consider using python-apt for that. An example can be found in the Debian +# packaging of mmdebstrap in ./debian/tests/sourcesfilter set -eu diff --git a/hooks/file-mirror-automount/setup00.sh b/hooks/file-mirror-automount/setup00.sh index 61f60f2..6ccbdaf 100755 --- a/hooks/file-mirror-automount/setup00.sh +++ b/hooks/file-mirror-automount/setup00.sh @@ -15,7 +15,7 @@ env APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-get indextargets --no-release-info - | while read -r path; do mkdir -p "$rootdir/run/mmdebstrap" if [ ! -d "/$path" ]; then - echo "/$path is not an existing directory" >&2 + echo "W: /$path is not an existing directory" >&2 continue fi case $MMDEBSTRAP_MODE in diff --git a/hooks/maybe-merged-usr/essential00.sh b/hooks/maybe-merged-usr/essential00.sh index a23f2f7..656057e 100755 --- a/hooks/maybe-merged-usr/essential00.sh +++ b/hooks/maybe-merged-usr/essential00.sh @@ -15,6 +15,10 @@ case "$ver" in echo "usr-is-merged package from src:usrmerge installed -- not running merged-usr essential hook" >&2 exit 0 ;; + 'not-installed ') + echo "usr-is-merged was not installed in a previous hook -- not running merged-usr essential hook" >&2 + exit 0 + ;; *) echo "unexpected situation for package usr-is-merged: $ver" >&2 exit 1 diff --git a/hooks/maybe-merged-usr/extract00.sh b/hooks/maybe-merged-usr/extract00.sh index dc88450..00bb037 100755 --- a/hooks/maybe-merged-usr/extract00.sh +++ b/hooks/maybe-merged-usr/extract00.sh @@ -4,12 +4,22 @@ set -eu env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-get update --error-on=any -# if the usr-is-merged package cannot be installed with apt, do nothing -if ! env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged > /dev/null 2>&1; then - echo "no package called usr-is-merged found -- not running merged-usr extract hook" >&2 - exit 0 +if env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged > /dev/null 2>&1; then + # if apt-cache exited successfully, then usr-is-merged exists either as + # a real or virtual package + if env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged 2>/dev/null | grep -q "Package: usr-is-merged"; then + echo "usr-is-merged found -- running merged-usr extract hook" >&2 + else + # The usr-is-merged must be virtual, so assume that nothing + # has to be done. This is the case with Debian Trixie or later + # or with Ubuntu Lunar or later + echo "usr-is-merged found but not real -- not running merged-usr extract hook" >&2 + exit 0 + fi else - echo "package usr-is-merged found -- running merged-usr extract hook" >&2 + # if the usr-is-merged package cannot be installed with apt, do nothing + echo "no package providing usr-is-merged found -- not running merged-usr extract hook" >&2 + exit 0 fi # resolve the script path using several methods in order: diff --git a/hooks/maybe-merged-usr/setup00.sh b/hooks/maybe-merged-usr/setup00.sh index a6bd712..6568af2 100755 --- a/hooks/maybe-merged-usr/setup00.sh +++ b/hooks/maybe-merged-usr/setup00.sh @@ -4,12 +4,22 @@ set -eu env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-get update --error-on=any -# if the usr-is-merged package cannot be installed with apt, do nothing -if ! env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged > /dev/null 2>&1; then - echo "no package called usr-is-merged found -- not running merged-usr setup hook" >&2 - exit 0 +if env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged > /dev/null 2>&1; then + # if apt-cache exited successfully, then usr-is-merged exists either as + # a real or virtual package + if env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged 2>/dev/null | grep -q "Package: usr-is-merged"; then + echo "usr-is-merged found -- running merged-usr setup hook" >&2 + else + # The usr-is-merged must be virtual, so assume that nothing + # has to be done. This is the case with Debian Trixie or later + # or with Ubuntu Lunar or later + echo "usr-is-merged found but not real -- not running merged-usr setup hook" >&2 + exit 0 + fi else - echo "package usr-is-merged found -- running merged-usr setup hook" >&2 + # if the usr-is-merged package cannot be installed with apt, do nothing + echo "no package providing usr-is-merged found -- not running merged-usr setup hook" >&2 + exit 0 fi # resolve the script path using several methods in order: diff --git a/make_mirror.sh b/make_mirror.sh index 8849ee3..419503b 100755 --- a/make_mirror.sh +++ b/make_mirror.sh @@ -33,7 +33,7 @@ deletecache() { done # deleting artifacts from test "mmdebstrap" for variant in essential apt minbase buildd - standard; do - for format in tar ext2 squashfs; do + for format in tar ext2 ext4 squashfs; do if [ -e "$dir/mmdebstrap-$dist-$variant.$format" ]; then # attempt to delete for all dists because DEFAULT_DIST might've been different the last time rm "$dir/mmdebstrap-$dist-$variant.$format" @@ -236,7 +236,11 @@ END esac # shellcheck disable=SC2086 - APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install $pkgs + APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install $pkgs \ + || APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install \ + -oDebug::pkgProblemResolver=true -oDebug::pkgDepCache::Marker=1 \ + -oDebug::pkgDepCache::AutoInstall=1 \ + $pkgs rm "$rootdir/var/cache/apt/archives/lock" rmdir "$rootdir/var/cache/apt/archives/partial" @@ -449,7 +453,7 @@ if [ "$HAVE_QEMU" = "yes" ]; then tmpdir="$(mktemp -d)" trap 'kill "$PROXYPID" || :;cleanuptmpdir; cleanup_newcachedir' EXIT INT TERM - pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,squashfs-tools-ng,genext2fs,linux-image-generic,passwd + pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,squashfs-tools-ng,genext2fs,linux-image-generic,passwd,e2fsprogs,uuid-runtime if [ ! -e ./mmdebstrap ]; then pkgs="$pkgs,mmdebstrap" fi @@ -546,6 +550,7 @@ END --customize-hook='copy-in "'"$tmpdir"'/worker.sh" /' \ --customize-hook='echo 127.0.0.1 localhost > "$1/etc/hosts"' \ --customize-hook='printf "START=1\nDAEMON_OPTS=\"-h 127.0.0.1 -p 80 -u nobody -dd /mnt/cache -i /var/run/mini-httpd.pid -T UTF-8\"\n" > "$1/etc/default/mini-httpd"' \ + --customize-hook='touch "$1/etc/systemd/system/tmp.mount"' \ "$mirror" kill $PROXYPID @@ -23,7 +23,7 @@ use strict; use warnings; -our $VERSION = '1.4.3'; +our $VERSION = '1.5.1'; use English; use Getopt::Long; @@ -36,7 +36,8 @@ use File::Find; use Cwd qw(abs_path getcwd); require "syscall.ph"; ## no critic (Modules::RequireBarewordIncludes) require "sys/ioctl.ph"; ## no critic (Modules::RequireBarewordIncludes) -use Fcntl qw(S_IFCHR S_IFBLK FD_CLOEXEC F_GETFD F_SETFD); +use Fcntl + qw(S_IFCHR S_IFBLK FD_CLOEXEC F_GETFD F_SETFD LOCK_EX O_RDONLY O_DIRECTORY); use List::Util qw(any none); use POSIX qw(SIGINT SIGHUP SIGPIPE SIGTERM SIG_BLOCK SIG_UNBLOCK strftime isatty); @@ -46,6 +47,7 @@ use Socket; use Time::HiRes; use Math::BigInt; use Text::ParseWords; +use Digest::SHA; use version; ## no critic (InputOutput::RequireBriefOpen) @@ -66,13 +68,16 @@ use version; *MS_BIND = \0x1000; *MS_REC = \0x4000; *MNT_DETACH = \2; +# uuid_t NameSpace_DNS in rfc4122 +*UUID_NS_DNS = \'6ba7b810-9dad-11d1-80b4-00c04fd430c8'; our ( $CLONE_NEWNS, $CLONE_NEWUTS, $CLONE_NEWIPC, $CLONE_NEWUSER, $CLONE_NEWPID, $CLONE_NEWNET, $_LINUX_CAPABILITY_VERSION_3, $CAP_SYS_ADMIN, $PR_CAPBSET_READ, $MS_BIND, - $MS_REC, $MNT_DETACH + $MS_REC, $MNT_DETACH, + $UUID_NS_DNS ); #<<< @@ -215,11 +220,12 @@ sub minor { } sub can_execute { - my $tool = shift; - my $pid = open my $fh, '-|' // return 0; + my $tool = shift; + my $verbose = shift // '--version'; + my $pid = open my $fh, '-|' // return 0; if ($pid == 0) { open(STDERR, '>&', STDOUT) or die; - exec {$tool} $tool, '--version' or die; + exec {$tool} $tool, $verbose or die; } chomp( my $content = do { local $/; <$fh> } @@ -303,6 +309,28 @@ sub shellescape { return "'$string'"; } +sub create_v5_uuid { + use bytes; + my $ns_uuid = shift; + my $name = shift; + my $version = 0x50; + # convert the namespace uuid to binary + $ns_uuid =~ tr/-//d; + $ns_uuid = pack 'H*', $ns_uuid; + # concatenate namespace and name and take sha1 + my $digest = Digest::SHA->new(1); + $digest->add($ns_uuid); + $digest->add($name); + # only the first 16 bytes matter + my $uuid = substr($digest->digest(), 0, 16); + # set the version + substr $uuid, 6, 1, chr(ord(substr($uuid, 6, 1)) & 0x0f | $version); + substr $uuid, 8, 1, chr(ord(substr $uuid, 8, 1) & 0x3f | 0x80); + # convert binary back to uuid formatting + return join '-', map { unpack 'H*', $_ } + map { substr $uuid, 0, $_, '' } (4, 2, 2, 2, 6); +} + sub test_unshare_userns { my $verbose = shift; @@ -1150,6 +1178,8 @@ sub setup_mounts { eval { if (any { $_ eq $options->{mode} } ('root', 'unshare')) { + 0 == system('mount', "--make-rprivate", "/") + or warning("mount --make-rprivate / failed: $?"); # if more than essential should be installed, make the system look # more like a real one by creating or bind-mounting the device # nodes @@ -2040,19 +2070,16 @@ sub run_setup() { # for authentication, use the keyrings from the host print $conf "Dir::Etc::Trusted \"$options->{apttrusted}\";\n"; print $conf "Dir::Etc::TrustedParts \"$options->{apttrustedparts}\";\n"; - if ($options->{variant} ne 'apt') { - # apt considers itself essential. Thus, when generating an EDSP - # document for an external solver, it will add the Essential:yes field - # to the apt package stanza. This is unnecessary for any other variant - # than 'apt' because in all other variants we compile the set of - # packages we consider essential ourselves and for the 'essential' - # variant it would even be wrong to add apt. This workaround is only - # needed when apt is used with an external solver but doesn't hurt - # otherwise and we don't have a good way to figure out whether apt is - # using an external solver or not short of parsing the --aptopt - # options. - print $conf "pkgCacheGen::ForceEssential \",\";\n"; - } + # apt considers itself essential. Thus, when generating an EDSP document + # for an external solver, it will add the Essential:yes field to the apt + # package stanza. This is unnecessary because we compile the set of + # packages we consider essential ourselves and for the 'essential' variant + # it would even be wrong to add apt. This workaround is only needed when + # apt is used with an external solver but doesn't hurt otherwise and we + # don't have a good way to figure out whether apt is using an external + # solver or not short of parsing the --aptopt options. + print $conf "pkgCacheGen::ForceEssential \",\";\n"; + close $conf; # We put certain configuration items in their own configuration file @@ -2458,27 +2485,8 @@ sub run_download() { dryrun => $options->{dryrun}, }, ); - } elsif ($options->{variant} eq 'apt') { - # if we just want to install Essential:yes packages, apt and their - # dependencies then we can make use of libapt treating apt as - # implicitly essential. An upgrade with the (currently) empty status - # file will trigger an installation of the essential packages plus apt. - # - # 2018-09-02, #debian-dpkg on OFTC, times in UTC+2 - # 23:39 < josch> I'll just put it in my script and if it starts - # breaking some time I just say it's apt's fault. :P - # 23:42 < DonKult> that is how it usually works, so yes, do that :P (<- - # and please add that line next to it so you can - # remind me in 5+ years that I said that after I wrote - # in the bugreport: "Are you crazy?!? Nobody in his - # right mind would even suggest depending on it!") - @dl_debs = run_apt_download_progress({ - APT_ARGV => ['dist-upgrade'], - dryrun => $options->{dryrun}, - }, - ); } elsif (any { $_ eq $options->{variant} } - ('essential', 'standard', 'important', 'required', 'buildd')) { + ('essential', 'apt', 'standard', 'important', 'required', 'buildd')) { # 2021-06-07, #debian-apt on OFTC, times in UTC+2 # 17:27 < DonKult> (?essential includes 'apt' through) # 17:30 < josch> DonKult: no, because pkgCacheGen::ForceEssential ","; @@ -2952,8 +2960,14 @@ sub run_install() { my $options = shift; my @pkgs_to_install = (@{ $options->{include} }); + if ($options->{variant} eq 'extract') { + error "must not be called with variant extract"; + } + if (none { $_ eq $options->{variant} } ('custom', 'essential')) { + push @pkgs_to_install, 'apt'; + } if ($options->{variant} eq 'buildd') { - push @pkgs_to_install, 'build-essential', 'apt'; + push @pkgs_to_install, 'build-essential'; } if (any { $_ eq $options->{variant} } ('required', 'important', 'standard')) { @@ -4380,15 +4394,16 @@ sub guess_sources_format { } sub approx_disk_usage { - my $directory = shift; + my $directory = shift; + my $block_size = shift; info "approximating disk usage..."; # the "du" utility reports different results depending on the underlying # filesystem, see https://bugs.debian.org/650077 for a discussion # # we use code similar to the one used by dpkg-gencontrol instead # - # Regular files are measured in number of 1024 byte blocks. All other - # entries are assumed to take one block of space. + # Regular files are measured in number of $block_size byte blocks. All + # other entries are assumed to take one block of space. # # We ignore /dev because depending on the mode, the directory might be # populated or not and we want consistent disk usage results independent @@ -4412,8 +4427,8 @@ sub approx_disk_usage { return if exists $hardlink{"$dev:$ino"}; # Track hardlinks to avoid repeated additions. $hardlink{"$dev:$ino"} = 1 if $nlink > 1; - # add file size in 1024 byte blocks, rounded up - $installed_size += int(((-s _) + 1024) / 1024); + # add file size in $block_size byte blocks, rounded up + $installed_size += int(((-s _) + $block_size) / $block_size); } else { # all other entries are assumed to only take up one block $installed_size += 1; @@ -4424,8 +4439,9 @@ sub approx_disk_usage { # image. See https://bugs.debian.org/1005857 find({ wanted => $scan_installed_size, no_chdir => 1 }, $directory); - # because the above is only a heuristic we add 10% extra for good measure - return int($installed_size * 1.1); + # the above is only a heuristic and especially ext4 will consume quite a + # few more blocks than the heuristic above is going to compute + return int($installed_size * 1.2); } sub main() { @@ -4805,7 +4821,7 @@ sub main() { $options->{format} = 'directory'; } my @valid_formats - = ('auto', 'directory', 'tar', 'squashfs', 'ext2', 'null'); + = ('auto', 'directory', 'tar', 'squashfs', 'ext2', 'ext4', 'null'); if (none { $_ eq $options->{format} } @valid_formats) { error "invalid format. Choose from " . (join ', ', @valid_formats); } @@ -5670,6 +5686,30 @@ sub main() { if ($exitstatus != 0) { error "genext2fs failed with exit status: $exitstatus"; } + } elsif ($options->{target} =~ /\.ext4$/) { + $options->{format} = 'ext4'; + # check if the installed version of e2fsprogs supports tarballs on + # stdin + (undef, my $filename) = tempfile( + "mmdebstrap.ext4.XXXXXXXXXXXX", + OPEN => 0, + TMPDIR => 1 + ); + # creating file to suppress message "Creating regular file ..." + { open my $fh, '>', $filename; } + open my $fh, '|-', 'mke2fs', '-q', '-F', '-o', 'Linux', '-T', + 'ext4', '-b', '4096', '-d', '-', $filename, + '16384' // error "failed to fork(): $!"; + # write 10240 null-bytes to mke2fs -- this represents an empty + # tar archive + print $fh ("\0" x 10240) + or error "cannot write to mke2fs process"; + close $fh; + my $exitstatus = $?; + unlink $filename // die "cannot unlink $filename"; + if ($exitstatus != 0) { + error "mke2fs failed with exit status: $exitstatus"; + } } else { $options->{format} = 'directory'; } @@ -5687,21 +5727,31 @@ sub main() { info "ignoring target $options->{target} with null format"; } + my $blocksize = -1; if ($options->{format} eq 'ext2') { if (!can_execute 'genext2fs') { error "need genext2fs for ext2 format"; } + $blocksize = 1024; + } elsif ($options->{format} eq 'ext4') { + if (!can_execute 'mke2fs', '-V') { + error "need mke2fs for ext4 format"; + } + $blocksize = 4096; } elsif ($options->{format} eq 'squashfs') { if (!can_execute 'tar2sqfs') { error "need tar2sqfs binary from the squashfs-tools-ng package"; } + $blocksize = 1048576; } - if (any { $_ eq $options->{format} } ('tar', 'squashfs', 'ext2', 'null')) { + my $rootdir_handle; + if (any { $_ eq $options->{format} } + ('tar', 'squashfs', 'ext2', 'ext4', 'null')) { if ($options->{format} ne 'null') { if (any { $_ eq $options->{variant} } ('extract', 'custom') and $options->{mode} eq 'fakechroot') { - info "creating a tarball or squashfs image or ext2 image in" + info "creating a tarball, squashfs, ext2 or ext4 image in" . " fakechroot mode might fail in extract and" . " custom variants because there might be no tar inside the" . " chroot"; @@ -5726,6 +5776,13 @@ sub main() { # directory $options->{root} = tempdir('mmdebstrap.XXXXXXXXXX', TMPDIR => 1); info "using $options->{root} as tempdir"; + # add an flock on the temporary directory to prevent cleanup by systemd + # see section Age in tmpfiles.d(5) + sysopen($rootdir_handle, $options->{root}, O_RDONLY | O_DIRECTORY) + or error "Failed to sysopen $options->{root}: $!\n"; + flock($rootdir_handle, LOCK_EX) + or error "Unable to flock $options->{root}: $!\n"; + # in unshare and root mode, other users than the current user need to # access the rootfs, most prominently, the _apt user. Thus, make the # temporary directory world readable. @@ -5885,7 +5942,7 @@ sub main() { # If both the above assertion change, we can stop creating /dev entries as # well. my $devtar = ''; - if (any { $_ eq $options->{format} } ('tar', 'squashfs', 'ext2')) { + if (any { $_ eq $options->{format} } ('tar', 'squashfs', 'ext2', 'ext4')) { foreach my $file (@devfiles) { my ($fname, $mode, $type, $linkname, $devmajor, $devminor) = @{$file}; @@ -5954,6 +6011,9 @@ sub main() { push @taropts, '--xattrs', '--xattrs-exclude=system.*'; } elsif ($options->{format} eq "ext2") { warning "genext2fs does not support extended attributes"; + warning "ext2 does not support sub-second precision timestamps"; + warning "ext2 does not support timestamps beyond 2038 January 18"; + warning "ext2 inode size of 128 prevents removing these limitations"; } else { push @taropts, '--xattrs'; } @@ -5963,8 +6023,6 @@ sub main() { my $sigset = POSIX::SigSet->new(SIGINT, SIGHUP, SIGPIPE, SIGTERM); POSIX::sigprocmask(SIG_BLOCK, $sigset) or error "Can't block signals: $!"; - my $pid; - # a pipe to transfer the final tarball from the child to the parent pipe my $rfh, my $wfh; @@ -5979,163 +6037,108 @@ sub main() { # b) it puts code writing the protocol outside of the helper/listener # c) the forked listener process cannot communicate to its parent pipe my $nblkreader, my $nblkwriter or error "pipe failed: $!"; - if ($options->{mode} eq 'unshare') { - $pid = get_unshare_cmd( - sub { - # child - local $SIG{'INT'} = 'DEFAULT'; - local $SIG{'HUP'} = 'DEFAULT'; - local $SIG{'PIPE'} = 'DEFAULT'; - local $SIG{'TERM'} = 'DEFAULT'; - # unblock all delayed signals (and possibly handle them) - POSIX::sigprocmask(SIG_UNBLOCK, $sigset) - or error "Can't unblock signals: $!"; + my $worker = sub { + # child + local $SIG{'INT'} = 'DEFAULT'; + local $SIG{'HUP'} = 'DEFAULT'; + local $SIG{'PIPE'} = 'DEFAULT'; + local $SIG{'TERM'} = 'DEFAULT'; - close $rfh; - close $parentsock; - open(STDOUT, '>&', STDERR) or error "cannot open STDOUT: $!"; + # unblock all delayed signals (and possibly handle them) + POSIX::sigprocmask(SIG_UNBLOCK, $sigset) + or error "Can't unblock signals: $!"; - setup($options); + close $rfh; + close $parentsock; + open(STDOUT, '>&', STDERR) or error "cannot open STDOUT: $!"; - print $childsock (pack('n', 0) . 'adios'); - $childsock->flush(); + setup($options); - close $childsock; + print $childsock (pack('n', 0) . 'adios'); + $childsock->flush(); - close $nblkreader; - if (!$options->{dryrun} && $options->{format} eq 'ext2') { - my $numblocks = approx_disk_usage($options->{root}); - print $nblkwriter "$numblocks\n"; - $nblkwriter->flush(); - } - close $nblkwriter; + close $childsock; - if ($options->{dryrun}) { - info "simulate creating tarball..."; - } elsif (any { $_ eq $options->{format} } - ('tar', 'squashfs', 'ext2')) { - info "creating tarball..."; - - # redirect tar output to the writing end of the pipe so - # that the parent process can capture the output - open(STDOUT, '>&', $wfh) or error "cannot open STDOUT: $!"; - - # Add ./dev as the first entries of the tar file. - # We cannot add them after calling tar, because there is no - # way to prevent tar from writing NULL entries at the end. - if (any { $_ eq 'output/dev' } @{ $options->{skip} }) { - info "skipping output/dev as requested"; - } else { - print $devtar; - } + close $nblkreader; + if (!$options->{dryrun} && any { $_ eq $options->{format} } + ('ext2', 'ext4')) { + my $numblocks = approx_disk_usage($options->{root}, $blocksize); + print $nblkwriter "$numblocks\n"; + $nblkwriter->flush(); + } + close $nblkwriter; - # pack everything except ./dev - 0 == system('tar', @taropts, '-C', $options->{root}, '.') - or error "tar failed: $?"; + if ($options->{dryrun}) { + info "simulate creating tarball..."; + } elsif (any { $_ eq $options->{format} } + ('tar', 'squashfs', 'ext2', 'ext4')) { + info "creating tarball..."; + + # redirect tar output to the writing end of the pipe so + # that the parent process can capture the output + open(STDOUT, '>&', $wfh) or error "cannot open STDOUT: $!"; + + # Add ./dev as the first entries of the tar file. + # We cannot add them after calling tar, because there is no + # way to prevent tar from writing NULL entries at the end. + if (any { $_ eq 'output/dev' } @{ $options->{skip} }) { + info "skipping output/dev as requested"; + } else { + print $devtar; + } - info "done"; - } elsif (any { $_ eq $options->{format} } - ('directory', 'null')) { - # nothing to do - } else { - error "unknown format: $options->{format}"; + if ($options->{mode} eq 'unshare') { + # pack everything except ./dev + 0 == system('tar', @taropts, '-C', $options->{root}, '.') + or error "tar failed: $?"; + } elsif ($options->{mode} eq 'fakechroot') { + # By default, FAKECHROOT_EXCLUDE_PATH includes /proc and /sys + # which means that the resulting tarball will contain the + # permission and ownership information of /proc and /sys from + # the outside, which we want to avoid. + ## no critic (Variables::RequireLocalizedPunctuationVars) + $ENV{FAKECHROOT_EXCLUDE_PATH} = "/dev"; + # Fakechroot requires tar to run inside the chroot or otherwise + # absolute symlinks will include the path to the root directory + 0 == system('chroot', $options->{root}, 'tar', + @taropts, '-C', '/', '.') + or error "tar failed: $?"; + } elsif (any { $_ eq $options->{mode} } ('root', 'chrootless')) { + # If the chroot directory is not owned by the root user, then + # we assume that no measure was taken to fake root permissions. + # Since the final tarball should contain entries with root + # ownership, we instruct tar to do so. + my @owneropts = (); + if ((stat $options->{root})[4] != 0) { + push @owneropts, '--owner=0', '--group=0', + '--numeric-owner'; } + 0 == system('tar', @taropts, @owneropts, '-C', + $options->{root}, '.') + or error "tar failed: $?"; + } else { + error "unknown mode: $options->{mode}"; + } - exit 0; - }, - \@idmap - ); + info "done"; + } elsif (any { $_ eq $options->{format} } ('directory', 'null')) { + # nothing to do + } else { + error "unknown format: $options->{format}"; + } + + exit 0; + }; + + my $pid; + if ($options->{mode} eq 'unshare') { + $pid = get_unshare_cmd($worker, \@idmap); } elsif (any { $_ eq $options->{mode} } ('root', 'fakechroot', 'chrootless')) { $pid = fork() // error "fork() failed: $!"; if ($pid == 0) { - local $SIG{'INT'} = 'DEFAULT'; - local $SIG{'HUP'} = 'DEFAULT'; - local $SIG{'PIPE'} = 'DEFAULT'; - local $SIG{'TERM'} = 'DEFAULT'; - - # unblock all delayed signals (and possibly handle them) - POSIX::sigprocmask(SIG_UNBLOCK, $sigset) - or error "Can't unblock signals: $!"; - - close $rfh; - close $parentsock; - open(STDOUT, '>&', STDERR) or error "cannot open STDOUT: $!"; - - setup($options); - - print $childsock (pack('n', 0) . 'adios'); - $childsock->flush(); - - close $childsock; - - close $nblkreader; - if (!$options->{dryrun} && $options->{format} eq 'ext2') { - my $numblocks = approx_disk_usage($options->{root}); - print $nblkwriter $numblocks; - $nblkwriter->flush(); - } - close $nblkwriter; - - if ($options->{dryrun}) { - info "simulate creating tarball..."; - } elsif (any { $_ eq $options->{format} } - ('tar', 'squashfs', 'ext2')) { - info "creating tarball..."; - - # redirect tar output to the writing end of the pipe so that - # the parent process can capture the output - open(STDOUT, '>&', $wfh) or error "cannot open STDOUT: $!"; - - # Add ./dev as the first entries of the tar file. - # We cannot add them after calling tar, because there is no way - # to prevent tar from writing NULL entries at the end. - if (any { $_ eq 'output/dev' } @{ $options->{skip} }) { - info "skipping output/dev as requested"; - } else { - print $devtar; - } - - if ($options->{mode} eq 'fakechroot') { - # By default, FAKECHROOT_EXCLUDE_PATH includes /proc and - # /sys which means that the resulting tarball will contain - # the permission and ownership information of /proc and - # /sys from the outside, which we want to avoid. - ## no critic (Variables::RequireLocalizedPunctuationVars) - $ENV{FAKECHROOT_EXCLUDE_PATH} = "/dev"; - # Fakechroot requires tar to run inside the chroot or - # otherwise absolute symlinks will include the path to the - # root directory - 0 == system('chroot', $options->{root}, 'tar', - @taropts, '-C', '/', '.') - or error "tar failed: $?"; - } elsif (any { $_ eq $options->{mode} } ('root', 'chrootless')) - { - # If the chroot directory is not owned by the root user, - # then we assume that no measure was taken to fake root - # permissions. Since the final tarball should contain - # entries with root ownership, we instruct tar to do so. - my @owneropts = (); - if ((stat $options->{root})[4] != 0) { - push @owneropts, '--owner=0', '--group=0', - '--numeric-owner'; - } - 0 == system('tar', @taropts, @owneropts, '-C', - $options->{root}, '.') - or error "tar failed: $?"; - } else { - error "unknown mode: $options->{mode}"; - } - - info "done"; - } elsif (any { $_ eq $options->{format} } ('directory', 'null')) { - # nothing to do - } else { - error "unknown format: $options->{format}"; - } - - exit 0; + $worker->(); } } else { error "unknown mode: $options->{mode}"; @@ -6190,7 +6193,8 @@ sub main() { my $numblocks = 0; close $nblkwriter; - if (!$options->{dryrun} && $options->{format} eq 'ext2') { + if (!$options->{dryrun} && any { $_ eq $options->{format} } + ('ext2', 'ext4')) { $numblocks = <$nblkreader>; if (defined $numblocks) { chomp $numblocks; @@ -6209,9 +6213,11 @@ sub main() { # nothing to do } elsif (any { $_ eq $options->{format} } ('directory', 'null')) { # nothing to do - } elsif ($options->{format} eq 'ext2' && $numblocks <= 0) { + } elsif ((any { $_ eq $options->{format} } ('ext2', 'ext4')) + && $numblocks <= 0) { # nothing to do because of invalid $numblocks - } elsif (any { $_ eq $options->{format} } ('tar', 'squashfs', 'ext2')) { + } elsif (any { $_ eq $options->{format} } + ('tar', 'squashfs', 'ext2', 'ext4')) { # we use eval() so that error() doesn't take this process down and # thus leaves the setup() process without a parent eval { @@ -6220,16 +6226,16 @@ sub main() { error "cannot copy to standard output: $!"; } } else { - if ( $options->{format} eq 'squashfs' - or $options->{format} eq 'ext2' - or defined $tar_compressor) { + if (any { $_ eq $options->{format} } + ('squashfs', 'ext2', 'ext4') + or defined $tar_compressor) { my @argv = (); if ($options->{format} eq 'squashfs') { push @argv, 'tar2sqfs', '--quiet', '--no-skip', '--force', '--exportable', '--compressor', 'xz', - '--block-size', '1048576', + '--block-size', $blocksize, $options->{target}; } elsif ($options->{format} eq 'ext2') { if ($numblocks <= 0) { @@ -6237,6 +6243,26 @@ sub main() { } push @argv, 'genext2fs', '-B', 1024, '-b', $numblocks, '-i', '16384', '-a', '-', $options->{target}; + } elsif ($options->{format} eq 'ext4') { + if ($numblocks <= 0) { + error "invalid number of blocks: $numblocks"; + } + push @argv, 'mke2fs', '-q', '-F', '-o', 'Linux', '-T', + 'ext4'; + if (exists $ENV{SOURCE_DATE_EPOCH}) { + # if SOURCE_DATE_EPOCH was set, make the image + # reproducible by setting a fixed uuid and + # hash_seed + my $uuid = create_v5_uuid( + create_v5_uuid( + $UUID_NS_DNS, "mister-muffin.de" + ), + $mtime + ); + push @argv, '-U', $uuid, '-E', "hash_seed=$uuid"; + } + push @argv, '-b', $blocksize, '-d', '-', + $options->{target}, $numblocks; } elsif ($options->{format} eq 'tar') { push @argv, @{$tar_compressor}; } else { @@ -6258,8 +6284,8 @@ sub main() { or error "Can't unblock signals: $!"; # redirect stdout to file or /dev/null - if ( $options->{format} eq 'squashfs' - or $options->{format} eq 'ext2') { + if (any { $_ eq $options->{format} } + ('squashfs', 'ext2', 'ext4')) { open(STDOUT, '>', '/dev/null') or error "cannot open /dev/null for writing: $!"; } elsif ($options->{format} eq 'tar') { @@ -6340,7 +6366,7 @@ sub main() { if (any { $_ eq $options->{format} } ('directory')) { # nothing to do } elsif (any { $_ eq $options->{format} } - ('tar', 'squashfs', 'ext2', 'null')) { + ('tar', 'squashfs', 'ext2', 'ext4', 'null')) { if (!-e $options->{root}) { error "$options->{root} does not exist"; } @@ -6439,12 +6465,12 @@ can be disabled by choosing the empty string for I<SUITE>. See the section B<VARIANTS> for more information. The I<TARGET> option may either be the path to a directory, the path to a -tarball filename, the path to a squashfs image, the path to an ext2 image, a -FIFO, a character special device, or C<->. The I<TARGET> option is optional if -no I<MIRROR> option is provided. If I<TARGET> is missing or if I<TARGET> is -C<->, an uncompressed tarball will be sent to standard output. Without the -B<--format> option, I<TARGET> will be used to choose the format. See the -section B<FORMATS> for more information. +tarball filename, the path to a squashfs image, the path to an ext2 or ext4 +image, a FIFO, a character special device, or C<->. The I<TARGET> option is +optional if no I<MIRROR> option is provided. If I<TARGET> is missing or if +I<TARGET> is C<->, an uncompressed tarball will be sent to standard output. +Without the B<--format> option, I<TARGET> will be used to choose the format. +See the section B<FORMATS> for more information. The I<MIRROR> option may either be provided as a URI, in apt one-line format, as a path to a file in apt's one-line or deb822-format, or C<->. If no @@ -6521,8 +6547,8 @@ information. =item B<--format>=I<name> Choose the output format. Valid format I<name>s are B<auto>, B<directory>, -B<tar>, B<squashfs>, B<ext2> and B<null>. The default format is B<auto>. See -the section B<FORMATS> for more information. +B<tar>, B<squashfs>, B<ext2>, B<ext4> and B<null>. The default format is +B<auto>. See the section B<FORMATS> for more information. =item B<--aptopt>=I<option>|I<file> @@ -6935,7 +6961,24 @@ Or without LXC: $ mmdebstrap --unshare-helper /usr/sbin/chroot ./debian-rootfs /bin/bash -Or, if you don't mind using superuser privileges and have systemd-nspawn +Or without mmdebstrap: + + $ unshare --map-auto --map-user=65536 --map-group=65536 --keep-caps -- \ + > /usr/sbin/chroot ./debian-rootfs /bin/bash + +The above uses C<--map-auto> to map the block of user/group ids for the +effective user/group to a block starting at user/group ID 0. We also want to +map the current effective user/group ID into the subuid/subgid range using +C<--map-user> and C<--map-group>, respectively. But if that uid/gid overlaps +with the respective range, a "hole" will be removed from the mapping and the +remaining uid/gid values will get shifted. Thus, we map the current effective +user/group ID to the highest possible uid/gid, putting them at the end. Since +that means that the user/group will be "nobody" and not "root" inside the +namespace, C<--keep-caps> propagate permitted capabilities into the ambient set +and thus give the user C<CAP_DAC_OVERRIDE> and other capabilities that it +would've had. + +Lastly, if you don't mind using superuser privileges and have systemd-nspawn available and you know your subuid/subgid offset (100000 in this example): $ sudo systemd-nspawn --private-users=100000 \ @@ -6946,6 +6989,11 @@ Instead, use something like this: $ unshare --map-root-user --map-auto rm -rf ./debian-rootfs +The above L<unshare(1)> command will map user and group ids into different +ranges compared to the mapping used by B<mmdebstrap> (effectively shifting them +one up) but it will provide the required capabilities for the removal +operation. + If this mode is used as the root user, the user namespace is not unshared (but the mount namespace and other still are) and created directories will have correct ownership information. This is also useful in cases where the root user @@ -7020,14 +7068,10 @@ considered for selection of C<Essential:yes> packages. =item B<apt> -The B<essential> set plus apt. This variant uses the fact that B<apt> treats -itself as essential and thus running C<apt-get dist-upgrade> without any -packages installed will install the B<essential> set plus B<apt>. If you just -want B<essential> and B<apt>, then this variant is faster than using the -B<essential> variant and adding B<apt> via C<--include> because all packages -get installed at once. The downside of this variant is, that if it should -happen that an B<essential> package is not installable, then it will just get -ignored without throwing an error. +The B<essential> set plus apt. +It is roughly equivalent to running mmdebstrap with + + --variant=essential --include="apt" =item B<buildd> @@ -7079,6 +7123,7 @@ I<TARGET> equals C<->, or if I<TARGET> is a named pipe (fifo) or if I<TARGET> is a character special file, then the B<tar> format will be chosen. If I<TARGET> ends with C<.squashfs> or C<.sqfs>, then the B<squashfs> format will be chosen. If I<TARGET> ends with C<.ext2> then the B<ext2> format will be +chosen. If I<TARGET> ends with C<.ext4> then the B<ext4> format will be chosen. If none of these conditions apply, the B<directory> format will be chosen. @@ -7128,8 +7173,24 @@ with this format because C<genext2fs> can only write to a regular file. If you need your ext2 image be named C<->, then just explicitly pass the relative path to it like F<./->. To convert the result to an ext3 image, use C<tune2fs -O has_journal TARGET> and to convert it to ext4, use C<tune2fs -O -extents,uninit_bg,dir_index,has_journal TARGET>. Since C<genext2fs> does not -support extended attributes, the resulting image will not contain them. +extents,uninit_bg,dir_index,has_journal TARGET>. + +B<CAUTION>: the ext2 format does not support timestamps beyond 2038 January 19, +does not support sub-second precision timestamps and does not support extended +attributes. Its inode size of 128 prevents adding these features with tune2fs +later on. + +=item B<ext4> + +A temporary chroot directory will be created in C<$TMPDIR> or F</tmp> if +C<$TMPDIR> is not set. A tarball of that directory will be piped to the +C<mke2fs> utility, which will create an ext4 image that will be approximately +90% full in I<TARGET>. The special I<TARGET> C<-> does not work with this +format because C<mke2fs> can only write to a regular file. If you need your +ext4 image be named C<->, then just explicitly pass the relative path to it +like F<./->. If C<SOURCE_DATE_EPOCH> is set, the filesystem UUID and hash_seed +will be set to a UUID derived from SOURCE_DATE_EPOCH to create reproducible +images. =item B<null> @@ -7226,7 +7287,12 @@ B<--skip=tar-in/mknod>. =item B<tar-out> I<pathinside> I<outside.tar> Packs the path I<pathinside> from inside the chroot into a tarball, placing it -into a certain location I<outside.tar> outside the chroot. +into a certain location I<outside.tar> outside the chroot. To emulate behaviour +of C<cp> and to provide control over the path which gets put into the tarball, +a C<chdir()> is performed to the C<dirname()> of I<pathinside> and then the +C<basename()> of I<pathinside> is packaged as a tarball. For example, if +I<pathinside> is C</boot/.> then first a C<chdir()> into C</boot> will be +performed before packing up the contents of C<.> inside C</boot>. =item B<download> I<fileinside> I<fileoutside> @@ -7334,15 +7400,11 @@ the B<setup> step. This can be disabled using B<--skip=update>. =item B<download> In the B<extract> and B<custom> variants, C<apt-get install> is used to -download all the packages requested via the B<--include> option. The B<apt> -variant uses the fact that libapt treats the C<apt> packages as implicitly -essential to download only all C<Essential:yes> packages plus apt using -C<apt-get dist-upgrade>. In the remaining variants, all Packages files -downloaded by the B<update> step are inspected to find the C<Essential:yes> -package set as well as all packages of the required priority. If I<SUITE> is a -non-empty string, then only packages from the archive with suite or codename -matching I<SUITE> will be considered for selection of C<Essential:yes> -packages. +download all the packages requested via the B<--include> option. In the +remaining variants, apt patterns are used to find the C<Essential:yes> packages +from the native architecture. If I<SUITE> is a non-empty string, then only +packages from the archive with suite or codename matching I<SUITE> will be +considered for selection of C<Essential:yes> packages. =item B<mount> @@ -7438,8 +7500,8 @@ Performs cleanup tasks, unless B<--skip=cleanup> is used: =item B<output> For formats other than B<directory>, pack up the temporary chroot directory -into a tarball, ext2 image or squashfs image and delete the temporary chroot -directory. +into a tarball, ext2 image, ext4 image or squashfs image and delete the +temporary chroot directory. If B<--skip=output/dev> is added, the resulting chroot will not contain the device nodes, directories and symlinks that B<debootstrap> creates but just @@ -7525,7 +7587,7 @@ translated manual packages (but not the untranslated ones), and documentation Create a bootable USB Stick that boots into a full Debian desktop: $ mmdebstrap --aptopt='Apt::Install-Recommends "true"' --customize-hook \ - 'chroot "$1" adduser --gecos user --disabled-password user' \ + 'chroot "$1" adduser --comment user --disabled-password user' \ --customize-hook='echo 'user:live' | chroot "$1" chpasswd' \ --customize-hook='echo host > "$1/etc/hostname"' \ --customize-hook='echo "127.0.0.1 localhost host" > "$1/etc/hosts"' \ @@ -7774,7 +7836,7 @@ Limitations in comparison to debootstrap: =item * Some debootstrap options don't exist, namely: I<--second-stage>, I<--exclude>, I<--resolve-deps>, I<--force-check-gpg>, -I<--merged-usr> and I<--no-merged-usr> +I<--merged-usr>, I<--no-merged-usr> and I<--cache-dir>. =back diff --git a/mmdebstrap-autopkgtest-build-qemu b/mmdebstrap-autopkgtest-build-qemu index 19175e5..2d1b803 100755 --- a/mmdebstrap-autopkgtest-build-qemu +++ b/mmdebstrap-autopkgtest-build-qemu @@ -261,9 +261,21 @@ case "$ARCHITECTURE" in ;; esac +test_installed() { + pkg="$1" + if [ "$(dpkg-query -f '${db:Status-Status}' -W "$pkg")" != installed ]; then + die "please install $pkg" + fi +} + +for pkg in autopkgtest dosfstools e2fsprogs fdisk mount mtools passwd uidmap; do + test_installed "$pkg" +done + if test "$(dpkg-query -f '${db:Status-Status}' -W binutils-multiarch)" = installed; then GNU_PREFIX= else + test_installed dpkg-dev GNU_ARCHITECTURE="$(dpkg-architecture "-a$ARCHITECTURE" -qDEB_HOST_GNU_TYPE)" GNU_PREFIX="$GNU_ARCHITECTURE-" GNU_SUFFIX="-$(echo "$GNU_ARCHITECTURE" | tr _ -)" @@ -277,11 +289,7 @@ case $arches in *) die "enable $ARCHITECTURE by running: sudo dpkg --add-architecture $ARCHITECTURE && sudo apt update" ;; esac -for pkg in autopkgtest dosfstools e2fsprogs fdisk mount mtools passwd "systemd-boot-efi:$ARCHITECTURE" uidmap; do - if [ "$(dpkg-query -f '${db:Status-Status}' -W "$pkg")" != installed ]; then - die "please install $pkg" - fi -done +test_installed "systemd-boot-efi:$ARCHITECTURE" BOOTSTUB="/usr/lib/systemd/boot/efi/linux${EFIIMG#boot}.stub" @@ -308,12 +316,15 @@ FAT_SIZE_SECTORS=$((1024*254)) # - users who prefer qcow2 get to choose to run it themselves with their own # custom options like compression # +# --map-users=auto --map-user=0 => 0:$UID:1 + 1:$SUBUIDBASE:65535 +# --map-users=auto --map-user=65536 => 0:$SUBUIDBASE:65536 + 65536:$UID:1 +# # Make the image writeable to the first subgid. mmdebstrap will map this gid to # the root group. unshare instead will map the current gid to 0 and the first # subgid to 1. Therefore mmdebstrap will be able to write to the image. rm -f "$IMAGE" : >"$IMAGE" -unshare -U -r --map-groups=auto chown 0:1 "$IMAGE" +unshare --map-user=0 --map-group=0 --map-groups=auto chown 0:1 "$IMAGE" chmod 0660 "$IMAGE" # Make sure that the unshared user is able to access the file. @@ -350,13 +361,17 @@ fi EXT4_OFFSET_BYTES=$(( (FAT_OFFSET_SECTORS + FAT_SIZE_SECTORS) * 512)) EXT4_OPTIONS="offset=$EXT4_OFFSET_BYTES,assume_storage_prezeroed=1" + +# the --no-mtab option to mount is a workaround for https://github.com/util-linux/util-linux/issues/2981 +# revert 8c0ddc32660ca4e98c988966251f9c05d6bcccef once it is no longer needed set -- "$@" \ "--customize-hook=download vmlinuz '$WORKDIR/kernel'" \ "--customize-hook=download initrd.img '$WORKDIR/initrd'" \ - '--customize-hook=mount --bind "$1" "$1/mnt"' \ - '--customize-hook=mount --bind "$1/mnt/mnt" "$1/mnt/dev"' \ + '--customize-hook=mount --no-mtab --bind "$1" "$1/mnt"' \ + '--customize-hook=mount --no-mtab --bind "$1/mnt/mnt" "$1/mnt/dev"' \ '--customize-hook=/sbin/mkfs.ext4 -d "$1/mnt" -L autopkgtestvm -E '"'$EXT4_OPTIONS' '$IMAGE' '$SIZE'" \ - '--customize-hook=umount --lazy "$1/mnt"' \ + '--customize-hook=umount --lazy --no-mtab "$1/mnt/dev"' \ + '--customize-hook=umount --lazy --no-mtab "$1/mnt"' \ "$RELEASE" \ /dev/null diff --git a/run_qemu.sh b/run_qemu.sh index fc00ed9..a51493b 100755 --- a/run_qemu.sh +++ b/run_qemu.sh @@ -4,6 +4,7 @@ set -eu : "${DEFAULT_DIST:=unstable}" : "${cachedir:=./shared/cache}" +: "${MMDEBSTRAP_TESTS_DEBUG:=no}" tmpdir="$(mktemp -d)" cleanup() { @@ -29,20 +30,42 @@ fi touch shared/output.txt setpriv --pdeathsig TERM tail -f shared/output.txt & -# to connect to serial use: -# minicom -D 'unix#/tmp/ttyS0' -# -# or this (quit with ctrl+q): -# socat stdin,raw,echo=0,escape=0x11 unix-connect:/tmp/ttyS0 -ret=0 -timeout --foreground 40m debvm-run --image="$(realpath "$cachedir")/debian-$DEFAULT_DIST.ext4" -- \ - -nic none \ - -m 4G -snapshot \ +set -- timeout --foreground 40m \ + debvm-run --image="$(realpath "$cachedir")/debian-$DEFAULT_DIST.ext4" \ + -- +cpuname=$(lscpu | awk '/Model name:/ {print $3}' | tr '\n' '+') +ncpu=$(lscpu | awk '/Core\(s\) per socket:/ {print $4}' | tr '\n' '+') +if [ "$cpuname" = "Cortex-A53+Cortex-A73+" ] && [ "$ncpu" = "2+4+" ]; then + # crude detection of the big.LITTLE heterogeneous setup of cores on the + # amlogic a311d bananapi + # + # https://lists.nongnu.org/archive/html/qemu-devel/2020-10/msg08494.html + # https://gitlab.com/qemu-project/qemu/-/issues/239 + # https://segments.zhan.science/posts/kvm_on_pinehone_pro/#trouble-with-heterogeneous-architecture + set -- taskset --cpu-list 2,3,4,5 "$@" -smp 4 +fi + +set -- "$@" -nic none -m 4G -snapshot + +if [ "$MMDEBSTRAP_TESTS_DEBUG" = "no" ]; then + # to connect to serial use: + # minicom -D 'unix#/tmp/ttyS0' + # or this (quit with ctrl+q): + # socat stdin,raw,echo=0,escape=0x11 unix-connect:/tmp/ttyS0 + set -- "$@" \ -monitor unix:/tmp/monitor,server,nowait \ -serial unix:/tmp/ttyS0,server,nowait \ - -serial unix:/tmp/ttyS1,server,nowait \ - -virtfs local,id=mmdebstrap,path="$(pwd)/shared",security_model=none,mount_tag=mmdebstrap \ - >"$tmpdir/log" 2>&1 || ret=$? + -serial unix:/tmp/ttyS1,server,nowait +fi + +set -- "$@" -virtfs local,id=mmdebstrap,path="$(pwd)/shared",security_model=none,mount_tag=mmdebstrap + +ret=0 +if [ "$MMDEBSTRAP_TESTS_DEBUG" = "no" ]; then + "$@" >"$tmpdir/log" 2>&1 || ret=$? +else + "$@" 2>&1 | tee "$tmpdir/log" || ret=$? +fi if [ "$ret" -ne 0 ]; then cat "$tmpdir/log" exit $ret diff --git a/tests/apt-patterns b/tests/apt-patterns index c87e932..a4f03e2 100644 --- a/tests/apt-patterns +++ b/tests/apt-patterns @@ -5,4 +5,4 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM {{ CMD }} --mode={{ MODE }} --variant=essential \ --include '?or(?exact-name(dummy-does-not-exist),?exact-name(apt))' \ {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }} -tar -tf /tmp/debian-chroot.tar | sort | grep -v ./var/lib/apt/extended_states | diff -u tar1.txt - +tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt - diff --git a/tests/apt-patterns-custom b/tests/apt-patterns-custom index 2348a76..bd78ed1 100644 --- a/tests/apt-patterns-custom +++ b/tests/apt-patterns-custom @@ -6,4 +6,4 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM --include '?narrow(?archive(^{{ DIST }}$),?essential)' \ --include apt \ {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }} -tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt - +{ tar -tf /tmp/debian-chroot.tar; echo ./var/lib/apt/extended_states; } | sort | diff -u tar1.txt - diff --git a/tests/aspcud-apt-solver b/tests/aspcud-apt-solver index bc0fbc3..24ab3f8 100644 --- a/tests/aspcud-apt-solver +++ b/tests/aspcud-apt-solver @@ -6,6 +6,6 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM --include "$(tr '\n' ',' < pkglist.txt)" \ --aptopt='APT::Solver "aspcud"' \ {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }} -tar -tf /tmp/debian-chroot.tar | sort \ +{ tar -tf /tmp/debian-chroot.tar; echo ./var/lib/apt/extended_states; } | sort \ | grep -v '^./etc/apt/apt.conf.d/99mmdebstrap$' \ | diff -u tar1.txt - diff --git a/tests/essential-hook b/tests/essential-hook index dc2b01f..0013a1a 100644 --- a/tests/essential-hook +++ b/tests/essential-hook @@ -17,5 +17,4 @@ tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort \ | grep -v '^./usr/share/lintian/overrides/tzdata' \ | grep -v '^./usr/share/zoneinfo' \ | grep -v '^./var/lib/dpkg/info/tzdata.' \ - | grep -v '^./var/lib/apt/extended_states$' \ | diff -u tar1.txt - diff --git a/tests/include b/tests/include index e284b7d..42d6936 100644 --- a/tests/include +++ b/tests/include @@ -6,7 +6,6 @@ trap "rm -rf /tmp/debian-chroot" EXIT INT TERM rm /tmp/debian-chroot/usr/share/doc-base/doc-debian.debian-* rm -r /tmp/debian-chroot/usr/share/doc/debian rm -r /tmp/debian-chroot/usr/share/doc/doc-debian -rm /tmp/debian-chroot/var/lib/apt/extended_states rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.list rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.md5sums tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt - diff --git a/tests/include-foreign-libmagic-mgc b/tests/include-foreign-libmagic-mgc index 127a84e..6b0bd7d 100644 --- a/tests/include-foreign-libmagic-mgc +++ b/tests/include-foreign-libmagic-mgc @@ -30,13 +30,12 @@ export LC_ALL=C.UTF-8 { echo "$native_arch"; echo "$foreign_arch"; } | cmp /tmp/debian-chroot/var/lib/dpkg/arch - rm /tmp/debian-chroot/usr/lib/file/magic.mgc rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/README.Debian -rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/"changelog.Debian.$foreign_arch.gz" +rm -f /tmp/debian-chroot/usr/share/doc/libmagic-mgc/"changelog.Debian.$foreign_arch.gz" rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.Debian.gz rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.gz rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/copyright rm /tmp/debian-chroot/usr/share/file/magic.mgc rm /tmp/debian-chroot/usr/share/misc/magic.mgc -rm /tmp/debian-chroot/var/lib/apt/extended_states rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums rmdir /tmp/debian-chroot/usr/share/doc/libmagic-mgc/ diff --git a/tests/include-foreign-libmagic-mgc-with-multiple-arch-options b/tests/include-foreign-libmagic-mgc-with-multiple-arch-options index 3108134..6d878e0 100644 --- a/tests/include-foreign-libmagic-mgc-with-multiple-arch-options +++ b/tests/include-foreign-libmagic-mgc-with-multiple-arch-options @@ -31,13 +31,12 @@ export LC_ALL=C.UTF-8 { echo "$native_arch"; echo "$foreign_arch"; } | cmp /tmp/debian-chroot/var/lib/dpkg/arch - rm /tmp/debian-chroot/usr/lib/file/magic.mgc rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/README.Debian -rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/"changelog.Debian.$foreign_arch.gz" +rm -f /tmp/debian-chroot/usr/share/doc/libmagic-mgc/"changelog.Debian.$foreign_arch.gz" rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.Debian.gz rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.gz rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/copyright rm /tmp/debian-chroot/usr/share/file/magic.mgc rm /tmp/debian-chroot/usr/share/misc/magic.mgc -rm /tmp/debian-chroot/var/lib/apt/extended_states rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums rmdir /tmp/debian-chroot/usr/share/doc/libmagic-mgc/ diff --git a/tests/install-libmagic-mgc-on-foreign b/tests/install-libmagic-mgc-on-foreign index 918224b..3254ff6 100644 --- a/tests/install-libmagic-mgc-on-foreign +++ b/tests/install-libmagic-mgc-on-foreign @@ -37,7 +37,7 @@ rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/README.Debian rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.Debian.gz rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.gz rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/copyright -rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/"changelog.Debian.$foreign_arch.gz" +rm -f /tmp/debian-chroot/usr/share/doc/libmagic-mgc/"changelog.Debian.$foreign_arch.gz" rm /tmp/debian-chroot/usr/share/file/magic.mgc rm /tmp/debian-chroot/usr/share/misc/magic.mgc # delete real files diff --git a/tests/logfile b/tests/logfile index 5e2dbeb..eff6504 100644 --- a/tests/logfile +++ b/tests/logfile @@ -16,6 +16,7 @@ I: running apt-get update... I: downloading packages with apt... I: extracting archives... I: installing essential packages... +I: installing remaining packages inside the chroot... I: cleaning package lists and apt cache... LOG tail --lines=1 /tmp/log | grep '^I: success in .* seconds$' diff --git a/tests/missing-dev-sys-proc-inside-the-chroot b/tests/missing-dev-sys-proc-inside-the-chroot index d127911..9c9b1c0 100644 --- a/tests/missing-dev-sys-proc-inside-the-chroot +++ b/tests/missing-dev-sys-proc-inside-the-chroot @@ -17,4 +17,8 @@ if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto prefix="runuser -u ${SUDO_USER:-user} --" fi -$prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} --include=dpkg,dash,diffutils,coreutils,libc-bin,sed {{ DIST }} /dev/null {{ MIRROR }} +# creating /sbin manually because of #1071078 +$prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} \ + --setup-hook='mkdir "$1/sbin"' \ + --include=dpkg,dash,diffutils,coreutils,libc-bin,sed \ + {{ DIST }} /dev/null {{ MIRROR }} diff --git a/tests/mmdebstrap b/tests/mmdebstrap index 3327fc6..f383ec4 100644 --- a/tests/mmdebstrap +++ b/tests/mmdebstrap @@ -5,15 +5,26 @@ export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }} [ "$(id -u)" -eq 0 ] [ {{ MODE }} = "root" ] -case {{ FORMAT }} in tar|squashfs|ext2) : ;; *) exit 1;; esac +case {{ FORMAT }} in tar|squashfs|ext2|ext4) : ;; *) exit 1;; esac -{{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} {{ DIST }} ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} {{ MIRROR }} +{{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} {{ DIST }} /tmp/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} {{ MIRROR }} +# creating an ext4 image on a 9p filesystem produces different results compared +# to creating it on a tmpfs or ext4 fs because 9p does not support discards and +# even when running with -E nodiscard, the number of written bytes will differ +# https://lore.kernel.org/linux-ext4/171484520952.2626447.2160419274451668597@localhost/T/#t +mv /tmp/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} if [ "{{ FORMAT }}" = tar ]; then printf 'ustar ' | cmp --bytes=6 --ignore-initial=257:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.tar - elif [ "{{ FORMAT }}" = squashfs ]; then printf 'hsqs' | cmp --bytes=4 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.squashfs - elif [ "{{ FORMAT }}" = ext2 ]; then printf '\123\357' | cmp --bytes=2 --ignore-initial=1080:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext2 - + printf '\000\000\000\000\000\000\000\000\000\000\000\000' | cmp --bytes=12 --ignore-initial=1116:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext2 - +elif [ "{{ FORMAT }}" = ext4 ]; then + printf '\123\357' | cmp --bytes=2 --ignore-initial=1080:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext4 - + printf '\074\020\000\000\302\042\000\000\153\004\000\000' | cmp --bytes=12 --ignore-initial=1116:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext4 - + + [ "$(/sbin/blkid --match-tag UUID --output value ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext4)" = "$(uuidgen --sha1 --namespace="$(uuidgen --sha1 --namespace='@dns' --name mister-muffin.de)" --name $SOURCE_DATE_EPOCH)" ] else echo "unknown format: {{ FORMAT }}" >&2 exit 1 diff --git a/tests/multiple-include b/tests/multiple-include index 36f53ec..eaef935 100644 --- a/tests/multiple-include +++ b/tests/multiple-include @@ -11,7 +11,6 @@ rm /tmp/debian-chroot/etc/localtime rm /tmp/debian-chroot/etc/timezone rm -r /tmp/debian-chroot/usr/share/doc/tzdata rm -r /tmp/debian-chroot/usr/share/zoneinfo -rm /tmp/debian-chroot/var/lib/apt/extended_states for p in doc-debian tzdata; do for f in list md5sums config postinst postrm templates preinst prerm; do [ -e "/tmp/debian-chroot/var/lib/dpkg/info/$p.$f" ] || continue diff --git a/tests/not-having-to-install-apt-in-include-because-a-hook-did-it-before b/tests/not-having-to-install-apt-in-include-because-a-hook-did-it-before index 9a36307..c745cf3 100644 --- a/tests/not-having-to-install-apt-in-include-because-a-hook-did-it-before +++ b/tests/not-having-to-install-apt-in-include-because-a-hook-did-it-before @@ -6,4 +6,4 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM --essential-hook='APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get update' \ --essential-hook='APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install apt' \ {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }} -tar -tf /tmp/debian-chroot.tar | sort | grep -v ./var/lib/apt/extended_states | diff -u tar1.txt - +tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt - |