diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-13 14:14:39 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-13 14:14:39 +0000 |
commit | ee17e45964b786b48b455959dfe68715971893fb (patch) | |
tree | 118f40aa65dc838499053413b05adfd00f839c62 /make_mirror.sh | |
parent | Initial commit. (diff) | |
download | mmdebstrap-ee17e45964b786b48b455959dfe68715971893fb.tar.xz mmdebstrap-ee17e45964b786b48b455959dfe68715971893fb.zip |
Adding upstream version 1.4.3.upstream/1.4.3
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'make_mirror.sh')
-rwxr-xr-x | make_mirror.sh | 568 |
1 files changed, 568 insertions, 0 deletions
diff --git a/make_mirror.sh b/make_mirror.sh new file mode 100755 index 0000000..8849ee3 --- /dev/null +++ b/make_mirror.sh @@ -0,0 +1,568 @@ +#!/bin/sh + +set -eu + +# This script fills either cache.A or cache.B with new content and then +# atomically switches the cache symlink from one to the other at the end. +# This way, at no point will the cache be in an non-working state, even +# when this script got canceled at any point. +# Working with two directories also automatically prunes old packages in +# the local repository. + +deletecache() { + dir="$1" + echo "running deletecache $dir">&2 + if [ ! -e "$dir" ]; then + return + fi + if [ ! -e "$dir/mmdebstrapcache" ]; then + echo "$dir cannot be the mmdebstrap cache" >&2 + return 1 + fi + # be very careful with removing the old directory + # experimental is pulled in with USE_HOST_APT_CONFIG=yes on debci + # when testing a package from experimental + for dist in oldstable stable testing unstable experimental; do + # deleting artifacts from test "debootstrap" + for variant in minbase buildd -; do + if [ -e "$dir/debian-$dist-$variant.tar" ]; then + rm "$dir/debian-$dist-$variant.tar" + else + echo "does not exist: $dir/debian-$dist-$variant.tar" >&2 + fi + done + # deleting artifacts from test "mmdebstrap" + for variant in essential apt minbase buildd - standard; do + for format in tar ext2 squashfs; do + if [ -e "$dir/mmdebstrap-$dist-$variant.$format" ]; then + # attempt to delete for all dists because DEFAULT_DIST might've been different the last time + rm "$dir/mmdebstrap-$dist-$variant.$format" + elif [ "$dist" = "$DEFAULT_DIST" ]; then + # only warn about non-existance when it's expected to exist + echo "does not exist: $dir/mmdebstrap-$dist-$variant.$format" >&2 + fi + done + done + if [ -e "$dir/debian/dists/$dist" ]; then + rm --one-file-system --recursive "$dir/debian/dists/$dist" + else + echo "does not exist: $dir/debian/dists/$dist" >&2 + fi + case "$dist" in oldstable|stable) + if [ -e "$dir/debian/dists/$dist-updates" ]; then + rm --one-file-system --recursive "$dir/debian/dists/$dist-updates" + else + echo "does not exist: $dir/debian/dists/$dist-updates" >&2 + fi + ;; + esac + case "$dist" in oldstable|stable) + if [ -e "$dir/debian-security/dists/$dist-security" ]; then + rm --one-file-system --recursive "$dir/debian-security/dists/$dist-security" + else + echo "does not exist: $dir/debian-security/dists/$dist-security" >&2 + fi + ;; + esac + done + for f in "$dir/debian-"*.ext4; do + if [ -e "$f" ]; then + rm --one-file-system "$f" + fi + done + # on i386 and amd64, the intel-microcode and amd64-microcode packages + # from non-free-firwame get pulled in because they are + # priority:standard with USE_HOST_APT_CONFIG=yes + for c in main non-free-firmware; do + if [ -e "$dir/debian/pool/$c" ]; then + rm --one-file-system --recursive "$dir/debian/pool/$c" + else + echo "does not exist: $dir/debian/pool/$c" >&2 + fi + done + if [ -e "$dir/debian-security/pool/updates/main" ]; then + rm --one-file-system --recursive "$dir/debian-security/pool/updates/main" + else + echo "does not exist: $dir/debian-security/pool/updates/main" >&2 + fi + for i in $(seq 1 6); do + if [ ! -e "$dir/debian$i" ]; then + continue + fi + rm "$dir/debian$i" + done + rm "$dir/mmdebstrapcache" + # remove all symlinks + find "$dir" -type l -delete + + # now the rest should only be empty directories + if [ -e "$dir" ]; then + find "$dir" -depth -print0 | xargs -0 --no-run-if-empty rmdir + else + echo "does not exist: $dir" >&2 + fi +} + +cleanup_newcachedir() { + echo "running cleanup_newcachedir" + deletecache "$newcachedir" +} + +cleanupapt() { + echo "running cleanupapt" >&2 + if [ ! -e "$rootdir" ]; then + return + fi + for f in \ + "$rootdir/var/cache/apt/archives/"*.deb \ + "$rootdir/var/cache/apt/archives/partial/"*.deb \ + "$rootdir/var/cache/apt/"*.bin \ + "$rootdir/var/lib/apt/lists/"* \ + "$rootdir/var/lib/dpkg/status" \ + "$rootdir/var/lib/dpkg/lock-frontend" \ + "$rootdir/var/lib/dpkg/lock" \ + "$rootdir/var/lib/apt/lists/lock" \ + "$rootdir/etc/apt/apt.conf" \ + "$rootdir/etc/apt/sources.list.d/"* \ + "$rootdir/etc/apt/preferences.d/"* \ + "$rootdir/etc/apt/sources.list" \ + "$rootdir/var/cache/apt/archives/lock"; do + if [ ! -e "$f" ]; then + echo "does not exist: $f" >&2 + continue + fi + if [ -d "$f" ]; then + rmdir "$f" + else + rm "$f" + fi + done + find "$rootdir" -depth -print0 | xargs -0 --no-run-if-empty rmdir +} + +# note: this function uses brackets instead of curly braces, so that it's run +# in its own process and we can handle traps independent from the outside +update_cache() ( + dist="$1" + nativearch="$2" + + # use a subdirectory of $newcachedir so that we can use + # hardlinks + rootdir="$newcachedir/apt" + mkdir -p "$rootdir" + + # we only set this trap here and overwrite the previous trap, because + # the update_cache function is run as part of a pipe and thus in its + # own process which will EXIT after it finished + trap 'kill "$PROXYPID" || :;cleanupapt' EXIT INT TERM + + for p in /etc/apt/apt.conf.d /etc/apt/sources.list.d /etc/apt/preferences.d /var/cache/apt/archives /var/lib/apt/lists/partial /var/lib/dpkg; do + mkdir -p "$rootdir/$p" + done + + # read sources.list content from stdin + cat > "$rootdir/etc/apt/sources.list" + + cat << END > "$rootdir/etc/apt/apt.conf" +Apt::Architecture "$nativearch"; +Apt::Architectures "$nativearch"; +Dir::Etc "$rootdir/etc/apt"; +Dir::State "$rootdir/var/lib/apt"; +Dir::Cache "$rootdir/var/cache/apt"; +Apt::Install-Recommends false; +Apt::Get::Download-Only true; +Acquire::Languages "none"; +Dir::Etc::Trusted "/etc/apt/trusted.gpg"; +Dir::Etc::TrustedParts "/etc/apt/trusted.gpg.d"; +Acquire::http::Proxy "http://127.0.0.1:8080/"; +END + + : > "$rootdir/var/lib/dpkg/status" + + if [ "$dist" = "$DEFAULT_DIST" ] && [ "$nativearch" = "$HOSTARCH" ] && [ "$USE_HOST_APT_CONFIG" = "yes" ]; then + # we append sources and settings instead of overwriting after + # an empty line + for f in /etc/apt/sources.list /etc/apt/sources.list.d/*; do + [ -e "$f" ] || continue + [ -e "$rootdir/$f" ] && echo >> "$rootdir/$f" + # Filter out file:// repositories as they are added + # to each mmdebstrap call verbatim by + # debian/tests/copy_host_apt_config + # Also filter out all mirrors that are not of suite + # $DEFAULT_DIST, except experimental if the suite + # is unstable. This prevents packages from + # unstable entering a testing mirror. + if [ "$dist" = unstable ]; then + grep -v ' file://' "$f" \ + | grep -E " (unstable|experimental) " \ + >> "$rootdir/$f" || : + else + grep -v ' file://' "$f" \ + | grep " $DEFAULT_DIST " \ + >> "$rootdir/$f" || : + fi + done + for f in /etc/apt/preferences.d/*; do + [ -e "$f" ] || continue + [ -e "$rootdir/$f" ] && echo >> "$rootdir/$f" + cat "$f" >> "$rootdir/$f" + done + fi + + echo "creating mirror for $dist" >&2 + for f in /etc/apt/sources.list /etc/apt/sources.list.d/* /etc/apt/preferences.d/*; do + [ -e "$rootdir/$f" ] || continue + echo "contents of $f:" >&2 + cat "$rootdir/$f" >&2 + done + + APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get update --error-on=any + + pkgs=$(APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get indextargets \ + --format '$(FILENAME)' 'Created-By: Packages' "Architecture: $nativearch" \ + | xargs --delimiter='\n' /usr/lib/apt/apt-helper cat-file \ + | grep-dctrl --no-field-names --show-field=Package --exact-match \ + \( --field=Essential yes --or --field=Priority required \ + --or --field=Priority important --or --field=Priority standard \ + \)) + + pkgs="$pkgs build-essential busybox gpg eatmydata fakechroot fakeroot" + + # we need usr-is-merged to simulate debootstrap behaviour for all dists + # starting from Debian 12 (Bullseye) + case "$dist" in + oldstable) : ;; + *) pkgs="$pkgs usr-is-merged usrmerge" ;; + esac + + # shellcheck disable=SC2086 + APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install $pkgs + + rm "$rootdir/var/cache/apt/archives/lock" + rmdir "$rootdir/var/cache/apt/archives/partial" + APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --option Dir::Etc::SourceList=/dev/null update + APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get clean + + cleanupapt + + # this function is run in its own process, so we unset all traps before + # returning + trap "-" EXIT INT TERM +) + +check_proxy_running() { + if timeout 1 bash -c 'exec 3<>/dev/tcp/127.0.0.1/8080 && printf "GET http://deb.debian.org/debian/dists/'"$DEFAULT_DIST"'/InRelease HTTP/1.1\nHost: deb.debian.org\n\n" >&3 && grep "Suite: '"$DEFAULT_DIST"'" <&3 >/dev/null' 2>/dev/null; then + return 0 + elif timeout 1 env http_proxy="http://127.0.0.1:8080/" wget --quiet -O - "http://deb.debian.org/debian/dists/$DEFAULT_DIST/InRelease" | grep "Suite: $DEFAULT_DIST" >/dev/null; then + return 0 + elif timeout 1 curl --proxy "http://127.0.0.1:8080/" --silent "http://deb.debian.org/debian/dists/$DEFAULT_DIST/InRelease" | grep "Suite: $DEFAULT_DIST" >/dev/null; then + return 0 + fi + return 1 +} + +if [ -e "./shared/cache.A" ] && [ -e "./shared/cache.B" ]; then + echo "both ./shared/cache.A and ./shared/cache.B exist" >&2 + echo "was a former run of the script aborted?" >&2 + if [ -e ./shared/cache ]; then + echo "cache symlink points to $(readlink ./shared/cache)" >&2 + case "$(readlink ./shared/cache)" in + cache.A) + echo "removing ./shared/cache.B" >&2 + rm -r ./shared/cache.B + ;; + cache.B) + echo "removing ./shared/cache.A" >&2 + rm -r ./shared/cache.A + ;; + *) + echo "unexpected" >&2 + exit 1 + ;; + esac + else + echo "./shared/cache doesn't exist" >&2 + exit 1 + fi +fi + +if [ -e "./shared/cache.A" ]; then + oldcache=cache.A + newcache=cache.B +else + oldcache=cache.B + newcache=cache.A +fi + +oldcachedir="./shared/$oldcache" +newcachedir="./shared/$newcache" + +oldmirrordir="$oldcachedir/debian" +newmirrordir="$newcachedir/debian" + +mirror="http://deb.debian.org/debian" +security_mirror="http://security.debian.org/debian-security" +components=main + +: "${DEFAULT_DIST:=unstable}" +: "${ONLY_DEFAULT_DIST:=no}" +: "${ONLY_HOSTARCH:=no}" +: "${HAVE_QEMU:=yes}" +: "${RUN_MA_SAME_TESTS:=yes}" +# by default, use the mmdebstrap executable in the current directory +: "${CMD:=./mmdebstrap}" +: "${USE_HOST_APT_CONFIG:=no}" +: "${FORCE_UPDATE:=no}" + +if [ "$FORCE_UPDATE" != "yes" ] && [ -e "$oldmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then + http_code=$(curl --output /dev/null --silent --location --head --time-cond "$oldmirrordir/dists/$DEFAULT_DIST/InRelease" --write-out '%{http_code}' "$mirror/dists/$DEFAULT_DIST/InRelease") + case "$http_code" in + 200) ;; # need update + 304) echo up-to-date; exit 0;; + *) echo "unexpected status: $http_code"; exit 1;; + esac +fi + +./caching_proxy.py "$oldcachedir" "$newcachedir" & +PROXYPID=$! +trap 'kill "$PROXYPID" || :' EXIT INT TERM + +for i in $(seq 10); do + check_proxy_running && break + sleep 1 +done +if [ ! -s "$newmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then + echo "failed to start proxy" >&2 + kill $PROXYPID + exit 1 +fi + +trap 'kill "$PROXYPID" || :;cleanup_newcachedir' EXIT INT TERM + +mkdir -p "$newcachedir" +touch "$newcachedir/mmdebstrapcache" + +HOSTARCH=$(dpkg --print-architecture) +arches="$HOSTARCH" +if [ "$HOSTARCH" = amd64 ]; then + arches="$arches arm64 i386" +elif [ "$HOSTARCH" = arm64 ]; then + arches="$arches amd64 armhf" +fi + +# we need the split_inline_sig() function +# shellcheck disable=SC1091 +. /usr/share/debootstrap/functions + +for dist in oldstable stable testing unstable; do + for nativearch in $arches; do + # non-host architectures are only downloaded for $DEFAULT_DIST + if [ "$nativearch" != "$HOSTARCH" ] && [ "$DEFAULT_DIST" != "$dist" ]; then + continue + fi + # if ONLY_DEFAULT_DIST is set, only download DEFAULT_DIST + if [ "$ONLY_DEFAULT_DIST" = "yes" ] && [ "$DEFAULT_DIST" != "$dist" ]; then + continue + fi + if [ "$ONLY_HOSTARCH" = "yes" ] && [ "$nativearch" != "$HOSTARCH" ]; then + continue + fi + # we need a first pass without updates and security patches + # because otherwise, old package versions needed by + # debootstrap will not get included + echo "deb [arch=$nativearch] $mirror $dist $components" | update_cache "$dist" "$nativearch" + # we need to include the base mirror again or otherwise + # packages like build-essential will be missing + case "$dist" in oldstable|stable) + cat << END | update_cache "$dist" "$nativearch" +deb [arch=$nativearch] $mirror $dist $components +deb [arch=$nativearch] $mirror $dist-updates main +deb [arch=$nativearch] $security_mirror $dist-security main +END + ;; + esac + done + codename=$(awk '/^Codename: / { print $2; }' < "$newmirrordir/dists/$dist/InRelease") + ln -s "$dist" "$newmirrordir/dists/$codename" + + # split the InRelease file into Release and Release.gpg not because apt + # or debootstrap need it that way but because grep-dctrl does + split_inline_sig \ + "$newmirrordir/dists/$dist/InRelease" \ + "$newmirrordir/dists/$dist/Release" \ + "$newmirrordir/dists/$dist/Release.gpg" + touch --reference="$newmirrordir/dists/$dist/InRelease" "$newmirrordir/dists/$dist/Release" "$newmirrordir/dists/$dist/Release.gpg" +done + +kill $PROXYPID + +# Create some symlinks so that we can trick apt into accepting multiple apt +# lines that point to the same repository but look different. This is to +# avoid the warning: +# W: Target Packages (main/binary-all/Packages) is configured multiple times... +for i in $(seq 1 6); do + ln -s debian "$newcachedir/debian$i" +done + +tmpdir="" + +cleanuptmpdir() { + if [ -z "$tmpdir" ]; then + return + fi + if [ ! -e "$tmpdir" ]; then + return + fi + for f in "$tmpdir/worker.sh" "$tmpdir/mmdebstrap.service"; do + if [ ! -e "$f" ]; then + echo "does not exist: $f" >&2 + continue + fi + rm "$f" + done + rmdir "$tmpdir" +} + +SOURCE_DATE_EPOCH="$(date --date="$(grep-dctrl -s Date -n '' "$newmirrordir/dists/$DEFAULT_DIST/Release")" +%s)" +export SOURCE_DATE_EPOCH + +if [ "$HAVE_QEMU" = "yes" ]; then + # we use the caching proxy again when building the qemu image + # - we can re-use the packages that were already downloaded earlier + # - we make sure that the qemu image uses the same Release file even + # if a mirror push happened between now and earlier + # - we avoid polluting the mirror with the additional packages by + # using --readonly + ./caching_proxy.py --readonly "$oldcachedir" "$newcachedir" & + PROXYPID=$! + + for i in $(seq 10); do + check_proxy_running && break + sleep 1 + done + if [ ! -s "$newmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then + echo "failed to start proxy" >&2 + kill $PROXYPID + exit 1 + fi + + tmpdir="$(mktemp -d)" + trap 'kill "$PROXYPID" || :;cleanuptmpdir; cleanup_newcachedir' EXIT INT TERM + + pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,squashfs-tools-ng,genext2fs,linux-image-generic,passwd + if [ ! -e ./mmdebstrap ]; then + pkgs="$pkgs,mmdebstrap" + fi + arches=$HOSTARCH + if [ "$RUN_MA_SAME_TESTS" = "yes" ]; then + case "$HOSTARCH" in + amd64) + arches=amd64,arm64 + pkgs="$pkgs,libfakechroot:arm64,libfakeroot:arm64" + ;; + arm64) + arches=arm64,amd64 + pkgs="$pkgs,libfakechroot:amd64,libfakeroot:amd64" + ;; + esac + fi + + cat << END > "$tmpdir/mmdebstrap.service" +[Unit] +Description=mmdebstrap worker script + +[Service] +Type=oneshot +ExecStart=/worker.sh + +[Install] +WantedBy=multi-user.target +END + # here is something crazy: + # as we run mmdebstrap, the process ends up being run by different users with + # different privileges (real or fake). But for being able to collect + # Devel::Cover data, they must all share a single directory. The only way that + # I found to make this work is to mount the database directory with a + # filesystem that doesn't support ownership information at all and a umask that + # gives read/write access to everybody. + # https://github.com/pjcj/Devel--Cover/issues/223 + cat << 'END' > "$tmpdir/worker.sh" +#!/bin/sh +echo 'root:root' | chpasswd +mount -t 9p -o trans=virtio,access=any,msize=128k mmdebstrap /mnt +# need to restart mini-httpd because we mounted different content into www-root +systemctl restart mini-httpd + +ip link set enp0s1 down || : + +handler () { + while IFS= read -r line || [ -n "$line" ]; do + printf "%s %s: %s\n" "$(date -u -d "0 $(date +%s.%3N) seconds - $2 seconds" +"%T.%3N")" "$1" "$line" + done +} + +( + cd /mnt; + if [ -e cover_db.img ]; then + mkdir -p cover_db + mount -o loop,umask=000 cover_db.img cover_db + fi + + now=$(date +%s.%3N) + ret=0 + { { { { { + sh -x ./test.sh 2>&1 1>&4 3>&- 4>&-; echo $? >&2; + } | handler E "$now" >&3; + } 4>&1 | handler O "$now" >&3; + } 2>&1; + } | { read xs; exit $xs; }; + } 3>&1 || ret=$? + echo $ret > /mnt/exitstatus.txt + if [ -e cover_db.img ]; then + df -h cover_db + umount cover_db + fi +) > /mnt/output.txt 2>&1 +umount /mnt +systemctl poweroff +END + chmod +x "$tmpdir/worker.sh" + if [ -z ${DISK_SIZE+x} ]; then + DISK_SIZE=10G + fi + # set PATH to pick up the correct mmdebstrap variant + env PATH="$(dirname "$(realpath --canonicalize-existing "$CMD")"):$PATH" \ + debvm-create --skip=usrmerge,systemdnetwork \ + --size="$DISK_SIZE" --release="$DEFAULT_DIST" \ + --output="$newcachedir/debian-$DEFAULT_DIST.ext4" -- \ + --architectures="$arches" --include="$pkgs" \ + --setup-hook='echo "Acquire::http::Proxy \"http://127.0.0.1:8080/\";" > "$1/etc/apt/apt.conf.d/00proxy"' \ + --hook-dir=/usr/share/mmdebstrap/hooks/maybe-merged-usr \ + --customize-hook='rm "$1/etc/apt/apt.conf.d/00proxy"' \ + --customize-hook='mkdir -p "$1/etc/systemd/system/multi-user.target.wants"' \ + --customize-hook='ln -s ../mmdebstrap.service "$1/etc/systemd/system/multi-user.target.wants/mmdebstrap.service"' \ + --customize-hook='touch "$1/mmdebstrap-testenv"' \ + --customize-hook='copy-in "'"$tmpdir"'/mmdebstrap.service" /etc/systemd/system/' \ + --customize-hook='copy-in "'"$tmpdir"'/worker.sh" /' \ + --customize-hook='echo 127.0.0.1 localhost > "$1/etc/hosts"' \ + --customize-hook='printf "START=1\nDAEMON_OPTS=\"-h 127.0.0.1 -p 80 -u nobody -dd /mnt/cache -i /var/run/mini-httpd.pid -T UTF-8\"\n" > "$1/etc/default/mini-httpd"' \ + "$mirror" + + kill $PROXYPID + cleanuptmpdir + trap "cleanup_newcachedir" EXIT INT TERM +fi + +# delete possibly leftover symlink +if [ -e ./shared/cache.tmp ]; then + rm ./shared/cache.tmp +fi +# now atomically switch the symlink to point to the other directory +ln -s $newcache ./shared/cache.tmp +mv --no-target-directory ./shared/cache.tmp ./shared/cache + +deletecache "$oldcachedir" + +trap - EXIT INT TERM + +echo "$0 finished successfully" >&2 |