Adding upstream version 39.upstream/39upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 617 insertions, 0 deletions

diff --git a/convert-usrmerge b/convert-usrmerge
new file mode 100755
index 0000000..7567a01
--- /dev/null
+++ b/convert-usrmerge
@@ -0,0 +1,617 @@
+#!/usr/bin/perl
+# vim: shiftwidth=4 tabstop=4
+#
+# Copyright 2014-2022 by Marco d'Itri <md@Linux.IT>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+use warnings;
+use strict;
+use autodie;
+use v5.16;
+
+use File::Find::Rule;
+use Cwd qw(abs_path);
+use Errno;
+
+my $Debug = 0;
+my $Program_RC = 0;
+my $Is_Hurd = is_hurd();
+
+# If a file exists both in / and /usr then ignore (delete) the one in /
+# Justification for these entries:
+# /lib/udev/hwdb.bin -> systemd-hwdb --usr update [LP: #1930573, Ubuntu-specific]
+my %Ignore_In_Root = map { $_ => 1 } qw(
+	/lib/udev/hwdb.bin
+);
+
+$ENV{LC_ALL} = 'C';
+
+check_free_space();
+
+check_overlayfs();
+
+check_uml();
+
+go_faster();
+
+# print the long error message if something fails due to autodie
+$SIG{__DIE__} = sub { fatal($_[0]); };
+
+{
+	foreach my $name (early_conversion_files()) {
+		next if $name =~ m#^/usr/#;	# already converted
+		convert_file($name);
+	}
+
+	my @dirs = directories_to_merge();
+
+	# create any directory which is in / but not in /usr
+	umask(0022);
+	foreach my $dir (@dirs) {
+		next if -e "/usr$dir";
+		next if not -d $dir;	# but only if it actually exists in /
+		mkdir("/usr$dir");
+		restore_context("/usr$dir");
+	}
+
+	my @later;
+	my $rule = File::Find::Rule->mindepth(1)->maxdepth(1)->start(@dirs);
+	while (defined (my $name = $rule->match)) {
+		convert_file($name, \@later);
+	}
+
+	# symlinks must be converted after the rest to avoid races, because
+	# they may point to binaries which have not been converted yet
+	if ($Is_Hurd) {
+		while (@later) {
+			my @newlater;
+			convert_file($_, \@newlater) foreach @later;
+			@later = @newlater;
+		}
+	} else {
+		convert_file($_) foreach @later;
+	}
+
+	verify_links_only($_) foreach @dirs;
+
+	convert_directory($_) foreach @dirs;
+
+	exit($Program_RC) if $Program_RC;
+	print "The system has been successfully converted.\n";
+	exit;
+}
+
+##############################################################################
+sub convert_file {
+	my ($n, $later) = @_;
+	print "==== $n ====\n" if $Debug;
+
+	# the source is a broken link
+	if (-l $n and not -e $n and -e "/usr$n") {
+		warn "WARNING: $n is a broken symlink and has been renamed!\n";
+		rename($n, "$n.usrmerge-broken");
+		return;
+	}
+
+	# the destination is a broken link
+	if (-l "/usr$n" and not -e "/usr$n") {
+		warn "WARNING: /usr$n is a broken symlink and has been renamed!\n";
+		rename("/usr$n", "/usr$n.usrmerge-broken");
+		# continue and move the source as usual
+	}
+
+	# is a directory and the destination does not exist
+	if (-d $n and not -e "/usr$n") {
+		mv($n, "/usr$n");					# XXX race
+		symlink("/usr$n", $n);
+		return;
+	}
+
+	# is a link and the destination does not exist, but defer the conversion
+	if (-l $n and not -e "/usr$n" and $later and not $Is_Hurd) {
+		push(@$later, $n);
+		return;
+	}
+
+	# the same, but only tested on Hurd systems (see #1020463 for details)
+	if (-l $n and not -e "/usr$n" and $later and $Is_Hurd) {
+		my $l = readlink($n);
+		my ($basedir) = $n =~ m#^(.+)/[^/]+$#;
+		if ($l !~ m#^/# and not -e "/usr$basedir/$l") {
+			fatal("Converted link /usr$n would point to non-existing"
+					. " /usr$basedir/$l but we are not deferring conversion")
+				if not $later;
+			push(@$later, $n);
+			return;
+		}
+	}
+
+	# is a file or link and the destination does not exist
+	if (not -e "/usr$n") {
+		cp($n, "/usr$n");
+		symlink("/usr$n", "$n~~tmp~usrmerge~~");
+		rename("$n~~tmp~usrmerge~~", $n);
+		# XXX alternative implementation:
+		#mv($n, "/usr$n");					# XXX race
+		#symlink("/usr$n", $n);
+		return;
+	}
+
+	# The other cases are more complex and there are some corner cases that
+	# we do not try to resolve automatically.
+
+	# both source and dest are links
+	if (-l $n and -l "/usr$n") {
+		my $l1 = readlink($n);
+		my $l2 = readlink("/usr$n");
+		my ($basedir) = $n =~ m#^(.+)/[^/]+$#;
+		return if $l1 eq $l2;				# and they point to the same file
+		return if "$basedir/$l1" eq $l2;	# same (the / link is relative)
+		return if $l1 eq "/usr$basedir/$l2";# same (the /usr link is relative)
+		return if $l1 eq "/usr$n";	# and the / link points to the /usr link
+		if ($l2 eq $n) {			# and the /usr link points to the / link
+			# the target of the new link will be an absolute path, so it
+			# may be different from the original one even if both point to
+			# the same file
+			symlink(abs_path($n), "/usr$n~~tmp~usrmerge~~");
+			rename("/usr$n~~tmp~usrmerge~~", "/usr$n");
+			# convert the /bin link too to make the program idempotent
+			symlink(abs_path($n), "$n~~tmp~usrmerge~~");
+			rename("$n~~tmp~usrmerge~~", "$n");
+			return;
+		}
+		fatal("Both $n and /usr$n exist");
+	}
+
+	# the source is a link
+	if (-l $n) {
+		my $l = readlink($n);
+		return if $l eq "/usr$n";			# and it points to dest
+		fatal("Both $n and /usr$n exist");
+	}
+
+	# the destination is a link
+	if (-l "/usr$n") {
+		my $l = readlink("/usr$n");
+		if ($l eq $n) {						# and it points to source
+			cp($n, "/usr$n~~tmp~usrmerge~~");
+			rename("/usr$n~~tmp~usrmerge~~", "/usr$n");
+			symlink("/usr$n", "$n~~tmp~usrmerge~~");
+			rename("$n~~tmp~usrmerge~~", $n);
+			# XXX alternative implementation:
+			#mv($n, "/usr$n");				# XXX race
+			#symlink("/usr$n", $n);
+			return;
+		}
+		fatal("Both $n and /usr$n exist");
+	}
+
+	# both source and dest are directories
+	# this is the second most common case
+	if (-d $n and -d "/usr$n") {
+		# so they have to be merged recursively
+		my $rule = File::Find::Rule->mindepth(1)->maxdepth(1)->start($n);
+		while (defined (my $name = $rule->match)) {
+			convert_file($name, $later);
+		}
+		return;
+	}
+
+	fatal("$n is a directory and /usr$n is not")
+		if -d $n and -e "/usr$n";
+	fatal("/usr$n is a directory and $n is not")
+		if -d "/usr$n";
+	if (-e "/usr$n" and exists $Ignore_In_Root{$n}) {
+		rm($n);
+		return;
+	}
+	fatal("Both $n and /usr$n exist")
+		if -e "/usr$n";
+
+	fatal("The status of $n and /usr$n is really unexpected");
+}
+
+##############################################################################
+# To prevent a failure later, the regular files of the libraries used by
+# cp and mv must be converted before of the symlinks that point to them.
+sub early_conversion_files {
+	no autodie qw(close);
+
+	my $bin_cp = '/bin/cp';
+	$bin_cp = '/usr/bin/cp' unless -x $bin_cp;
+	open(my $fh, '-|', "ldd $bin_cp");
+	my @ldd = <$fh>;
+	close $fh or fatal("Failed to execute 'ldd $bin_cp'");
+
+	# the libraries
+	my @list = grep { $_ } map { /^\s+\S+ => (\/\S+) / and $1 } @ldd;
+	# the dynamic linker
+	push(@list, grep { $_ } map { /^\s+(\/\S+) \(/ and $1 } @ldd);
+
+	# this is the equivalent of readlink --canonicalize
+	my @newlist;
+	foreach my $name (@list) {
+		my $newname = -l $name ? readlink($name) : $name;
+		if ($newname !~ m#^/#) {
+			my $dir = $name;
+			$dir =~ s#[^/]+$##;
+			$newname = $dir . $newname;
+		}
+		my $topdir = $newname;
+		$topdir =~ s#^(/[^/]+).*#$1#;
+		# this is needed to be idempotent after a complete conversion
+		next if -l $topdir;
+		push(@newlist, $newname);
+	}
+
+	return @newlist;
+}
+
+##############################################################################
+# Safety check: verify that there are no regular files in the directories
+# that will be deleted by the final pass.
+sub verify_links_only {
+	my ($dir) = @_;
+
+	my $link_or_dir = File::Find::Rule->or(
+		File::Find::Rule->symlink,
+		File::Find::Rule->directory,
+	);
+
+	my $rule = File::Find::Rule->mindepth(1)->not($link_or_dir)->start($dir);
+	while (defined (my $name = $rule->match)) {
+		print STDERR "$name is not a symlink!\n";
+		print STDERR "\nSafety check: the conversion has failed!\n";
+		exit 1;
+	}
+}
+
+sub convert_directory {
+	my ($dir) = @_;
+
+	return if -l $dir;
+
+	if (not -d $dir) {
+		# do not create a broken symlink to /usr$dir if it does not exist
+		return if not -e "usr$dir";
+		symlink("usr$dir", $dir);
+		restore_context($dir);
+		return;
+	}
+
+	no autodie;
+	if (not rename($dir, "$dir~~delete~usrmerge~~")) {	# XXX race
+		if ($!{EBUSY}) {
+			handle_ebusy($dir);
+			return;
+		}
+		die "Can't rename $dir: $!";
+	}
+	use autodie;
+	symlink("usr$dir", $dir);
+	restore_context($dir);
+
+	rm('-rf', "$dir~~delete~usrmerge~~");
+}
+
+sub restore_context {
+	my ($file) = @_;
+
+	return if not (-x '/sbin/restorecon' or -x '/usr/sbin/restorecon');
+
+	safe_system('restorecon', $file);
+}
+
+sub handle_ebusy {
+	my ($dir) = @_;
+
+	print STDERR <<END;
+
+WARNING: renaming $dir/ (for the purpose of replacing it with a symlink
+to /usr$dir/) has failed with the EBUSY error.
+This is probably caused by a systemd service started with the
+ProtectSystem option. Before running again this program you will need to
+stop the relevant daemon(s) or reboot the system.
+Do not install or update other Debian packages until the program
+has been run successfully. (Removing packages is not harmful.)
+END
+
+	# continue, but have the program eventually return an error
+	$Program_RC = 1;
+
+	# if systemd is running...
+	return if -d '/run/systemd/system';
+
+	# list the services with ProtectSystem enabled
+	my $cmd = q{
+	for service in $(systemctl --no-legend --full list-units \
+		    --state=active --type=service | cut -d ' ' -f 1); do
+	  if systemctl show $service | egrep -q "^ProtectSystem=(yes|full)"; then
+		echo $service
+	  fi
+	done
+	};
+
+	my $cmd_output = qx{$cmd} || return;
+	print STDERR <<END;
+
+The following services are using the ProtectSystem feature:
+$cmd_output
+END
+}
+
+##############################################################################
+sub check_free_space {
+	no autodie qw(close);
+
+	my $fh;
+	open($fh, '-|', 'stat --dereference --file-system --format="%i" /');
+	my $root_id = <$fh>;
+	die "stat / failed" if not defined $root_id;
+	chomp $root_id;
+
+	# beware: df(1) reports the value of %a, not of %f
+	open($fh, '-|',
+		'stat --dereference --file-system --format="%f %S %i" /usr/');
+	my $stat_output = <$fh>;
+	die "stat /usr failed" if not defined $stat_output;
+	chomp $stat_output;
+	my ($free_blocks, $bs, $usr_id) = split(/ /, $stat_output);
+
+	return if $root_id eq $usr_id;
+
+	my $free = $free_blocks * ($bs / 1024);
+	my @dirs = grep { -e $_ } directories_to_merge();
+
+	my $cmd = "du --summarize --no-dereference --total --block-size=1K @dirs";
+	open($fh, '-|', $cmd);
+	my $needed;
+	while (<$fh>) {
+		($needed) = /^(\d+)\s+total$/;
+	}
+	close $fh or fatal("Failed to execute $cmd");
+	die "$cmd failed" if not defined $needed;
+
+	say "Free space in /usr: $free KB." if $Debug;
+	say "The origin directories (@dirs) require $needed KB." if $Debug;
+
+	die "$free KB in /usr but $needed KB are required!\n" if $needed > $free;
+}
+
+# check if we are running under overlayfs, and if so, try and see if directories
+# are movable - we might be running in a container, thus running under overlayfs
+# but still able to do the conversion because the chroot is not overlayed.
+sub check_overlayfs {
+	no autodie qw(close);
+
+	my $fh;
+	open($fh, '-|', "stat --file-system --format=%T /");
+	my $fs_type = <$fh>;
+	close $fh or fatal("Failed to execute stat --file-system --format=%T /");
+	die "stat / failed" if not defined $fs_type;
+	chomp $fs_type;
+	return if $fs_type ne "overlayfs";
+
+	# We have detected overlayfs on /. Let's see if we can move the directories
+	# with a (hopefully) quick back-and-forth. This is not risk-free, but it's
+	# the best we can do.
+	my @dirs = directories_to_merge();
+	foreach my $dir (@dirs) {
+		next if not -e "$dir";
+
+		no autodie qw(rename);
+		if ((not rename("$dir", "$dir~usrmerge~")) && $!{EXDEV}) {
+			die("Warning: overlayfs detected, /usr/lib/usrmerge/convert-usrmerge will not
+be run automatically. See #1008202 for details.
+
+If this is a container then it can be converted by unpacking the image,
+entering it with chroot(8), installing usrmerge and then repacking the
+image again.");
+		}
+		use autodie qw(rename);
+
+		rename("$dir~usrmerge~", "$dir");
+	}
+}
+
+sub is_hurd {
+	my $architecture = qx{dpkg --print-architecture};
+	return 1 if $architecture =~ /^hurd-/;
+	return 0;
+}
+
+sub is_mount_point {
+	my ($path) = @_;
+
+	my @cmd = ('mountpoint', '-q', $path);
+	my $rc = system(@cmd);
+	die "Failed to execute @cmd: $!\n" if $rc == -1;
+	return 0 if $rc;
+	return 1;
+}
+
+# Check if something is mounted on /lib/modules/ in User Mode Linux systems.
+# See #1021180 for details.
+sub check_uml {
+	return if not is_mount_point('/lib/modules/');
+
+	print STDERR <<END;
+
+FATAL ERROR:
+
+/lib/modules/ is a mount point.
+Probably this system is using User Mode Linux or some variant of Xen.
+
+To continue the conversion please unmount /lib/modules/ (try the command
+'umount -l /lib/modules/') and then try again.
+Do not forget to reboot after the conversion is complete to have it
+mounted again.
+
+END
+	exit(1);
+}
+
+##############################################################################
+# Try to avoid the inherent races by being as fast as possible.
+sub go_faster {
+	system('/usr/bin/ionice', '--class=realtime', "--pid=$$")
+		if -x '/usr/bin/ionice';
+	system('/usr/bin/chrt', '--rr', '-p', '99', '--pid', $$)
+		if -x '/usr/bin/chrt';
+}
+
+# We use cp from coreutils because it preserves extended attributes and
+# handles sparse files.
+sub cp {
+	my ($source, $dest) = @_;
+
+	safe_system('cp', '--no-dereference', '--preserve=all',
+		'--reflink=auto', '--sparse=always', $source, $dest);
+}
+
+# We use mv from coreutils because it supports moving directories across
+# filesystems, which would be inconvenient to implement here.
+sub mv {
+	my ($source, $dest) = @_;
+
+	safe_system('mv', '--no-clobber', $source, $dest);
+}
+
+# We use rm from coreutils because I cannot be bothered to implement -r.
+sub rm {
+	safe_system('rm', @_);
+}
+
+sub safe_system {
+	my (@cmd) = @_;
+
+	my $rc = system(@cmd);
+	die "Failed to execute @cmd: $!\n" if $rc == -1;
+	die "@cmd: rc=" . ($? >> 8) . "\n" if $rc;
+}
+
+sub fatal {
+	my ($msg) = @_;
+
+	$msg .= ".\n" if $msg !~ /\n$/;
+
+	print STDERR <<END;
+
+FATAL ERROR:
+$msg
+You can try correcting the errors reported and running again
+$0 until it will complete without errors.
+Do not install or update other Debian packages until the program
+has been run successfully.
+
+END
+	exit(1);
+}
+
+# Find out where the runtime dynamic linker and the shared libraries
+# can be installed on each architecture: native and multilib.
+sub directories_to_merge {
+	return qw(/bin /sbin /lib /lib32 /lib64 /libo32 /libx32);
+}
+
+# check if the argument is one of the architectures enabled on the system
+sub running_arch {
+	my ($wanted) = @_;
+
+	state @system_arch;
+	if (not @system_arch) {
+		push(@system_arch, `dpkg --print-architecture`);
+		push(@system_arch, `dpkg --print-foreign-architectures`);
+		chomp @system_arch;
+	}
+
+	return 1 if grep { $_ eq $wanted } @system_arch;
+	return 0;
+}
+
+##############################################################################
+__END__
+
+=head1 NAME
+
+convert-usrmerge - converts the system to everything-in-usr
+
+=head1 SYNOPSIS
+
+convert-usrmerge
+
+=head1 DESCRIPTION
+
+This program will automatically convert the system to the
+everything-in-usr directory scheme.
+
+There is no automatic method to restore the precedent configuration, so
+there is no going back once the program has been run.
+
+The program is idempotent, unless the system crashes at a really bad time.
+
+The conflicts of all files in the Debian archive can be solved
+automatically, but some corner cases of custom setups may require
+manual changes.
+
+=head1 CONFLICTS RESOLUTION MATRIX
+
+ s/d F   D   L   B
+ F   X   X   S   Rd
+ D   X   D   S   Rd
+ L   K   K   K?  Rd
+ B   Rs  Rs  Rs  Rs
+
+I<Source> is / and I<destination> is /usr.
+
+Types of files:
+
+=over 4
+
+=item F: file
+
+=item D: directory
+
+=item L: symbolic link
+
+=item B: broken symbolic link
+
+=back
+
+Actions:
+
+=over 4
+
+=item X: unresolvable conflict
+
+=item D: recurse and compare the content of the directories
+
+=item K: keep the source (if the link matches the destination)
+
+=item S: swap source and destination (if the link matches the destination)
+
+=item R: rename (source or destination)
+
+=back
+
+=head1 BUGS
+
+Replacing a directory with a symlink is racy unless we do a complex dance
+of bind mounts. We should decide if this is really needed.
+
+Conflicting relative symbolic links are not handled automatically.
+
+The libc6-i386 and libc6-x32 packages require to convert the /lib32 and
+/libx32 directories as well, otherwise they would only contain the
+symlink to the dynamic linker specified by the architecture ABI.
+
+=head1 AUTHOR
+
+The program and this man page have been written by Marco d'Itri
+and are licensed under the terms of the GNU General Public License,
+version 2 or higher.
+