summaryrefslogtreecommitdiffstats
path: root/convert-usrmerge
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-20 16:34:05 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-20 16:34:05 +0000
commit17910b92b762000663c665fedbab30f7b1d41bdf (patch)
tree2e8c4fbab094dac1e376dc9395e3cf1cc5e40639 /convert-usrmerge
parentInitial commit. (diff)
downloadusrmerge-17910b92b762000663c665fedbab30f7b1d41bdf.tar.xz
usrmerge-17910b92b762000663c665fedbab30f7b1d41bdf.zip
Adding upstream version 39.upstream/39upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'convert-usrmerge')
-rwxr-xr-xconvert-usrmerge617
1 files changed, 617 insertions, 0 deletions
diff --git a/convert-usrmerge b/convert-usrmerge
new file mode 100755
index 0000000..7567a01
--- /dev/null
+++ b/convert-usrmerge
@@ -0,0 +1,617 @@
+#!/usr/bin/perl
+# vim: shiftwidth=4 tabstop=4
+#
+# Copyright 2014-2022 by Marco d'Itri <md@Linux.IT>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+use warnings;
+use strict;
+use autodie;
+use v5.16;
+
+use File::Find::Rule;
+use Cwd qw(abs_path);
+use Errno;
+
+my $Debug = 0;
+my $Program_RC = 0;
+my $Is_Hurd = is_hurd();
+
+# If a file exists both in / and /usr then ignore (delete) the one in /
+# Justification for these entries:
+# /lib/udev/hwdb.bin -> systemd-hwdb --usr update [LP: #1930573, Ubuntu-specific]
+my %Ignore_In_Root = map { $_ => 1 } qw(
+ /lib/udev/hwdb.bin
+);
+
+$ENV{LC_ALL} = 'C';
+
+check_free_space();
+
+check_overlayfs();
+
+check_uml();
+
+go_faster();
+
+# print the long error message if something fails due to autodie
+$SIG{__DIE__} = sub { fatal($_[0]); };
+
+{
+ foreach my $name (early_conversion_files()) {
+ next if $name =~ m#^/usr/#; # already converted
+ convert_file($name);
+ }
+
+ my @dirs = directories_to_merge();
+
+ # create any directory which is in / but not in /usr
+ umask(0022);
+ foreach my $dir (@dirs) {
+ next if -e "/usr$dir";
+ next if not -d $dir; # but only if it actually exists in /
+ mkdir("/usr$dir");
+ restore_context("/usr$dir");
+ }
+
+ my @later;
+ my $rule = File::Find::Rule->mindepth(1)->maxdepth(1)->start(@dirs);
+ while (defined (my $name = $rule->match)) {
+ convert_file($name, \@later);
+ }
+
+ # symlinks must be converted after the rest to avoid races, because
+ # they may point to binaries which have not been converted yet
+ if ($Is_Hurd) {
+ while (@later) {
+ my @newlater;
+ convert_file($_, \@newlater) foreach @later;
+ @later = @newlater;
+ }
+ } else {
+ convert_file($_) foreach @later;
+ }
+
+ verify_links_only($_) foreach @dirs;
+
+ convert_directory($_) foreach @dirs;
+
+ exit($Program_RC) if $Program_RC;
+ print "The system has been successfully converted.\n";
+ exit;
+}
+
+##############################################################################
+sub convert_file {
+ my ($n, $later) = @_;
+ print "==== $n ====\n" if $Debug;
+
+ # the source is a broken link
+ if (-l $n and not -e $n and -e "/usr$n") {
+ warn "WARNING: $n is a broken symlink and has been renamed!\n";
+ rename($n, "$n.usrmerge-broken");
+ return;
+ }
+
+ # the destination is a broken link
+ if (-l "/usr$n" and not -e "/usr$n") {
+ warn "WARNING: /usr$n is a broken symlink and has been renamed!\n";
+ rename("/usr$n", "/usr$n.usrmerge-broken");
+ # continue and move the source as usual
+ }
+
+ # is a directory and the destination does not exist
+ if (-d $n and not -e "/usr$n") {
+ mv($n, "/usr$n"); # XXX race
+ symlink("/usr$n", $n);
+ return;
+ }
+
+ # is a link and the destination does not exist, but defer the conversion
+ if (-l $n and not -e "/usr$n" and $later and not $Is_Hurd) {
+ push(@$later, $n);
+ return;
+ }
+
+ # the same, but only tested on Hurd systems (see #1020463 for details)
+ if (-l $n and not -e "/usr$n" and $later and $Is_Hurd) {
+ my $l = readlink($n);
+ my ($basedir) = $n =~ m#^(.+)/[^/]+$#;
+ if ($l !~ m#^/# and not -e "/usr$basedir/$l") {
+ fatal("Converted link /usr$n would point to non-existing"
+ . " /usr$basedir/$l but we are not deferring conversion")
+ if not $later;
+ push(@$later, $n);
+ return;
+ }
+ }
+
+ # is a file or link and the destination does not exist
+ if (not -e "/usr$n") {
+ cp($n, "/usr$n");
+ symlink("/usr$n", "$n~~tmp~usrmerge~~");
+ rename("$n~~tmp~usrmerge~~", $n);
+ # XXX alternative implementation:
+ #mv($n, "/usr$n"); # XXX race
+ #symlink("/usr$n", $n);
+ return;
+ }
+
+ # The other cases are more complex and there are some corner cases that
+ # we do not try to resolve automatically.
+
+ # both source and dest are links
+ if (-l $n and -l "/usr$n") {
+ my $l1 = readlink($n);
+ my $l2 = readlink("/usr$n");
+ my ($basedir) = $n =~ m#^(.+)/[^/]+$#;
+ return if $l1 eq $l2; # and they point to the same file
+ return if "$basedir/$l1" eq $l2; # same (the / link is relative)
+ return if $l1 eq "/usr$basedir/$l2";# same (the /usr link is relative)
+ return if $l1 eq "/usr$n"; # and the / link points to the /usr link
+ if ($l2 eq $n) { # and the /usr link points to the / link
+ # the target of the new link will be an absolute path, so it
+ # may be different from the original one even if both point to
+ # the same file
+ symlink(abs_path($n), "/usr$n~~tmp~usrmerge~~");
+ rename("/usr$n~~tmp~usrmerge~~", "/usr$n");
+ # convert the /bin link too to make the program idempotent
+ symlink(abs_path($n), "$n~~tmp~usrmerge~~");
+ rename("$n~~tmp~usrmerge~~", "$n");
+ return;
+ }
+ fatal("Both $n and /usr$n exist");
+ }
+
+ # the source is a link
+ if (-l $n) {
+ my $l = readlink($n);
+ return if $l eq "/usr$n"; # and it points to dest
+ fatal("Both $n and /usr$n exist");
+ }
+
+ # the destination is a link
+ if (-l "/usr$n") {
+ my $l = readlink("/usr$n");
+ if ($l eq $n) { # and it points to source
+ cp($n, "/usr$n~~tmp~usrmerge~~");
+ rename("/usr$n~~tmp~usrmerge~~", "/usr$n");
+ symlink("/usr$n", "$n~~tmp~usrmerge~~");
+ rename("$n~~tmp~usrmerge~~", $n);
+ # XXX alternative implementation:
+ #mv($n, "/usr$n"); # XXX race
+ #symlink("/usr$n", $n);
+ return;
+ }
+ fatal("Both $n and /usr$n exist");
+ }
+
+ # both source and dest are directories
+ # this is the second most common case
+ if (-d $n and -d "/usr$n") {
+ # so they have to be merged recursively
+ my $rule = File::Find::Rule->mindepth(1)->maxdepth(1)->start($n);
+ while (defined (my $name = $rule->match)) {
+ convert_file($name, $later);
+ }
+ return;
+ }
+
+ fatal("$n is a directory and /usr$n is not")
+ if -d $n and -e "/usr$n";
+ fatal("/usr$n is a directory and $n is not")
+ if -d "/usr$n";
+ if (-e "/usr$n" and exists $Ignore_In_Root{$n}) {
+ rm($n);
+ return;
+ }
+ fatal("Both $n and /usr$n exist")
+ if -e "/usr$n";
+
+ fatal("The status of $n and /usr$n is really unexpected");
+}
+
+##############################################################################
+# To prevent a failure later, the regular files of the libraries used by
+# cp and mv must be converted before of the symlinks that point to them.
+sub early_conversion_files {
+ no autodie qw(close);
+
+ my $bin_cp = '/bin/cp';
+ $bin_cp = '/usr/bin/cp' unless -x $bin_cp;
+ open(my $fh, '-|', "ldd $bin_cp");
+ my @ldd = <$fh>;
+ close $fh or fatal("Failed to execute 'ldd $bin_cp'");
+
+ # the libraries
+ my @list = grep { $_ } map { /^\s+\S+ => (\/\S+) / and $1 } @ldd;
+ # the dynamic linker
+ push(@list, grep { $_ } map { /^\s+(\/\S+) \(/ and $1 } @ldd);
+
+ # this is the equivalent of readlink --canonicalize
+ my @newlist;
+ foreach my $name (@list) {
+ my $newname = -l $name ? readlink($name) : $name;
+ if ($newname !~ m#^/#) {
+ my $dir = $name;
+ $dir =~ s#[^/]+$##;
+ $newname = $dir . $newname;
+ }
+ my $topdir = $newname;
+ $topdir =~ s#^(/[^/]+).*#$1#;
+ # this is needed to be idempotent after a complete conversion
+ next if -l $topdir;
+ push(@newlist, $newname);
+ }
+
+ return @newlist;
+}
+
+##############################################################################
+# Safety check: verify that there are no regular files in the directories
+# that will be deleted by the final pass.
+sub verify_links_only {
+ my ($dir) = @_;
+
+ my $link_or_dir = File::Find::Rule->or(
+ File::Find::Rule->symlink,
+ File::Find::Rule->directory,
+ );
+
+ my $rule = File::Find::Rule->mindepth(1)->not($link_or_dir)->start($dir);
+ while (defined (my $name = $rule->match)) {
+ print STDERR "$name is not a symlink!\n";
+ print STDERR "\nSafety check: the conversion has failed!\n";
+ exit 1;
+ }
+}
+
+sub convert_directory {
+ my ($dir) = @_;
+
+ return if -l $dir;
+
+ if (not -d $dir) {
+ # do not create a broken symlink to /usr$dir if it does not exist
+ return if not -e "usr$dir";
+ symlink("usr$dir", $dir);
+ restore_context($dir);
+ return;
+ }
+
+ no autodie;
+ if (not rename($dir, "$dir~~delete~usrmerge~~")) { # XXX race
+ if ($!{EBUSY}) {
+ handle_ebusy($dir);
+ return;
+ }
+ die "Can't rename $dir: $!";
+ }
+ use autodie;
+ symlink("usr$dir", $dir);
+ restore_context($dir);
+
+ rm('-rf', "$dir~~delete~usrmerge~~");
+}
+
+sub restore_context {
+ my ($file) = @_;
+
+ return if not (-x '/sbin/restorecon' or -x '/usr/sbin/restorecon');
+
+ safe_system('restorecon', $file);
+}
+
+sub handle_ebusy {
+ my ($dir) = @_;
+
+ print STDERR <<END;
+
+WARNING: renaming $dir/ (for the purpose of replacing it with a symlink
+to /usr$dir/) has failed with the EBUSY error.
+This is probably caused by a systemd service started with the
+ProtectSystem option. Before running again this program you will need to
+stop the relevant daemon(s) or reboot the system.
+Do not install or update other Debian packages until the program
+has been run successfully. (Removing packages is not harmful.)
+END
+
+ # continue, but have the program eventually return an error
+ $Program_RC = 1;
+
+ # if systemd is running...
+ return if -d '/run/systemd/system';
+
+ # list the services with ProtectSystem enabled
+ my $cmd = q{
+ for service in $(systemctl --no-legend --full list-units \
+ --state=active --type=service | cut -d ' ' -f 1); do
+ if systemctl show $service | egrep -q "^ProtectSystem=(yes|full)"; then
+ echo $service
+ fi
+ done
+ };
+
+ my $cmd_output = qx{$cmd} || return;
+ print STDERR <<END;
+
+The following services are using the ProtectSystem feature:
+$cmd_output
+END
+}
+
+##############################################################################
+sub check_free_space {
+ no autodie qw(close);
+
+ my $fh;
+ open($fh, '-|', 'stat --dereference --file-system --format="%i" /');
+ my $root_id = <$fh>;
+ die "stat / failed" if not defined $root_id;
+ chomp $root_id;
+
+ # beware: df(1) reports the value of %a, not of %f
+ open($fh, '-|',
+ 'stat --dereference --file-system --format="%f %S %i" /usr/');
+ my $stat_output = <$fh>;
+ die "stat /usr failed" if not defined $stat_output;
+ chomp $stat_output;
+ my ($free_blocks, $bs, $usr_id) = split(/ /, $stat_output);
+
+ return if $root_id eq $usr_id;
+
+ my $free = $free_blocks * ($bs / 1024);
+ my @dirs = grep { -e $_ } directories_to_merge();
+
+ my $cmd = "du --summarize --no-dereference --total --block-size=1K @dirs";
+ open($fh, '-|', $cmd);
+ my $needed;
+ while (<$fh>) {
+ ($needed) = /^(\d+)\s+total$/;
+ }
+ close $fh or fatal("Failed to execute $cmd");
+ die "$cmd failed" if not defined $needed;
+
+ say "Free space in /usr: $free KB." if $Debug;
+ say "The origin directories (@dirs) require $needed KB." if $Debug;
+
+ die "$free KB in /usr but $needed KB are required!\n" if $needed > $free;
+}
+
+# check if we are running under overlayfs, and if so, try and see if directories
+# are movable - we might be running in a container, thus running under overlayfs
+# but still able to do the conversion because the chroot is not overlayed.
+sub check_overlayfs {
+ no autodie qw(close);
+
+ my $fh;
+ open($fh, '-|', "stat --file-system --format=%T /");
+ my $fs_type = <$fh>;
+ close $fh or fatal("Failed to execute stat --file-system --format=%T /");
+ die "stat / failed" if not defined $fs_type;
+ chomp $fs_type;
+ return if $fs_type ne "overlayfs";
+
+ # We have detected overlayfs on /. Let's see if we can move the directories
+ # with a (hopefully) quick back-and-forth. This is not risk-free, but it's
+ # the best we can do.
+ my @dirs = directories_to_merge();
+ foreach my $dir (@dirs) {
+ next if not -e "$dir";
+
+ no autodie qw(rename);
+ if ((not rename("$dir", "$dir~usrmerge~")) && $!{EXDEV}) {
+ die("Warning: overlayfs detected, /usr/lib/usrmerge/convert-usrmerge will not
+be run automatically. See #1008202 for details.
+
+If this is a container then it can be converted by unpacking the image,
+entering it with chroot(8), installing usrmerge and then repacking the
+image again.");
+ }
+ use autodie qw(rename);
+
+ rename("$dir~usrmerge~", "$dir");
+ }
+}
+
+sub is_hurd {
+ my $architecture = qx{dpkg --print-architecture};
+ return 1 if $architecture =~ /^hurd-/;
+ return 0;
+}
+
+sub is_mount_point {
+ my ($path) = @_;
+
+ my @cmd = ('mountpoint', '-q', $path);
+ my $rc = system(@cmd);
+ die "Failed to execute @cmd: $!\n" if $rc == -1;
+ return 0 if $rc;
+ return 1;
+}
+
+# Check if something is mounted on /lib/modules/ in User Mode Linux systems.
+# See #1021180 for details.
+sub check_uml {
+ return if not is_mount_point('/lib/modules/');
+
+ print STDERR <<END;
+
+FATAL ERROR:
+
+/lib/modules/ is a mount point.
+Probably this system is using User Mode Linux or some variant of Xen.
+
+To continue the conversion please unmount /lib/modules/ (try the command
+'umount -l /lib/modules/') and then try again.
+Do not forget to reboot after the conversion is complete to have it
+mounted again.
+
+END
+ exit(1);
+}
+
+##############################################################################
+# Try to avoid the inherent races by being as fast as possible.
+sub go_faster {
+ system('/usr/bin/ionice', '--class=realtime', "--pid=$$")
+ if -x '/usr/bin/ionice';
+ system('/usr/bin/chrt', '--rr', '-p', '99', '--pid', $$)
+ if -x '/usr/bin/chrt';
+}
+
+# We use cp from coreutils because it preserves extended attributes and
+# handles sparse files.
+sub cp {
+ my ($source, $dest) = @_;
+
+ safe_system('cp', '--no-dereference', '--preserve=all',
+ '--reflink=auto', '--sparse=always', $source, $dest);
+}
+
+# We use mv from coreutils because it supports moving directories across
+# filesystems, which would be inconvenient to implement here.
+sub mv {
+ my ($source, $dest) = @_;
+
+ safe_system('mv', '--no-clobber', $source, $dest);
+}
+
+# We use rm from coreutils because I cannot be bothered to implement -r.
+sub rm {
+ safe_system('rm', @_);
+}
+
+sub safe_system {
+ my (@cmd) = @_;
+
+ my $rc = system(@cmd);
+ die "Failed to execute @cmd: $!\n" if $rc == -1;
+ die "@cmd: rc=" . ($? >> 8) . "\n" if $rc;
+}
+
+sub fatal {
+ my ($msg) = @_;
+
+ $msg .= ".\n" if $msg !~ /\n$/;
+
+ print STDERR <<END;
+
+FATAL ERROR:
+$msg
+You can try correcting the errors reported and running again
+$0 until it will complete without errors.
+Do not install or update other Debian packages until the program
+has been run successfully.
+
+END
+ exit(1);
+}
+
+# Find out where the runtime dynamic linker and the shared libraries
+# can be installed on each architecture: native and multilib.
+sub directories_to_merge {
+ return qw(/bin /sbin /lib /lib32 /lib64 /libo32 /libx32);
+}
+
+# check if the argument is one of the architectures enabled on the system
+sub running_arch {
+ my ($wanted) = @_;
+
+ state @system_arch;
+ if (not @system_arch) {
+ push(@system_arch, `dpkg --print-architecture`);
+ push(@system_arch, `dpkg --print-foreign-architectures`);
+ chomp @system_arch;
+ }
+
+ return 1 if grep { $_ eq $wanted } @system_arch;
+ return 0;
+}
+
+##############################################################################
+__END__
+
+=head1 NAME
+
+convert-usrmerge - converts the system to everything-in-usr
+
+=head1 SYNOPSIS
+
+convert-usrmerge
+
+=head1 DESCRIPTION
+
+This program will automatically convert the system to the
+everything-in-usr directory scheme.
+
+There is no automatic method to restore the precedent configuration, so
+there is no going back once the program has been run.
+
+The program is idempotent, unless the system crashes at a really bad time.
+
+The conflicts of all files in the Debian archive can be solved
+automatically, but some corner cases of custom setups may require
+manual changes.
+
+=head1 CONFLICTS RESOLUTION MATRIX
+
+ s/d F D L B
+ F X X S Rd
+ D X D S Rd
+ L K K K? Rd
+ B Rs Rs Rs Rs
+
+I<Source> is / and I<destination> is /usr.
+
+Types of files:
+
+=over 4
+
+=item F: file
+
+=item D: directory
+
+=item L: symbolic link
+
+=item B: broken symbolic link
+
+=back
+
+Actions:
+
+=over 4
+
+=item X: unresolvable conflict
+
+=item D: recurse and compare the content of the directories
+
+=item K: keep the source (if the link matches the destination)
+
+=item S: swap source and destination (if the link matches the destination)
+
+=item R: rename (source or destination)
+
+=back
+
+=head1 BUGS
+
+Replacing a directory with a symlink is racy unless we do a complex dance
+of bind mounts. We should decide if this is really needed.
+
+Conflicting relative symbolic links are not handled automatically.
+
+The libc6-i386 and libc6-x32 packages require to convert the /lib32 and
+/libx32 directories as well, otherwise they would only contain the
+symlink to the dynamic linker specified by the architecture ABI.
+
+=head1 AUTHOR
+
+The program and this man page have been written by Marco d'Itri
+and are licensed under the terms of the GNU General Public License,
+version 2 or higher.
+