summaryrefslogtreecommitdiffstats
path: root/support
diff options
context:
space:
mode:
Diffstat (limited to 'support')
-rw-r--r--support/Makefile6
-rwxr-xr-xsupport/atomic-rsync138
-rwxr-xr-xsupport/cvs2includes42
-rwxr-xr-xsupport/deny-rsync36
-rwxr-xr-xsupport/file-attr-restore173
-rwxr-xr-xsupport/files-to-excludes27
-rwxr-xr-xsupport/git-set-file-times94
-rwxr-xr-xsupport/instant-rsyncd126
-rwxr-xr-xsupport/json-rsync-version93
-rwxr-xr-xsupport/logfilter34
-rwxr-xr-xsupport/lsh108
-rwxr-xr-xsupport/lsh.sh37
-rwxr-xr-xsupport/mapfrom15
-rwxr-xr-xsupport/mapto15
-rwxr-xr-xsupport/mnt-excl49
-rwxr-xr-xsupport/munge-symlinks71
-rwxr-xr-xsupport/nameconvert50
-rwxr-xr-xsupport/rrsync379
-rw-r--r--support/rrsync.1.md166
-rwxr-xr-xsupport/rsync-no-vanished24
-rwxr-xr-xsupport/rsync-slash-strip24
-rwxr-xr-xsupport/rsyncstats312
-rw-r--r--support/savetransfer.c175
23 files changed, 2194 insertions, 0 deletions
diff --git a/support/Makefile b/support/Makefile
new file mode 100644
index 0000000..c6a1e30
--- /dev/null
+++ b/support/Makefile
@@ -0,0 +1,6 @@
+all: savetransfer
+
+savetransfer: savetransfer.o
+
+clean:
+ rm -f *.o savetransfer
diff --git a/support/atomic-rsync b/support/atomic-rsync
new file mode 100755
index 0000000..1964090
--- /dev/null
+++ b/support/atomic-rsync
@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+# This script lets you update a hierarchy of files in an atomic way by
+# first creating a new hierarchy using rsync's --link-dest option, and
+# then swapping the hierarchy into place. **See the usage message for
+# more details and some important caveats!**
+
+import os, sys, re, subprocess, shutil
+
+ALT_DEST_ARG_RE = re.compile('^--[a-z][^ =]+-dest(=|$)')
+
+RSYNC_PROG = '/usr/bin/rsync'
+
+def main():
+ cmd_args = sys.argv[1:]
+ if '--help' in cmd_args:
+ usage_and_exit()
+
+ if len(cmd_args) < 2:
+ usage_and_exit(True)
+
+ dest_dir = cmd_args[-1].rstrip('/')
+ if dest_dir == '' or dest_dir.startswith('-'):
+ usage_and_exit(True)
+
+ if not os.path.isdir(dest_dir):
+ die(dest_dir, "is not a directory or a symlink to a dir.\nUse --help for help.")
+
+ bad_args = [ arg for arg in cmd_args if ALT_DEST_ARG_RE.match(arg) ]
+ if bad_args:
+ die("You cannot use the", ' or '.join(bad_args), "option with atomic-rsync.\nUse --help for help.")
+
+ # We ignore exit-code 24 (file vanished) by default.
+ allowed_exit_codes = '0 ' + os.environ.get('ATOMIC_RSYNC_OK_CODES', '24')
+ try:
+ allowed_exit_codes = set(int(num) for num in re.split(r'[, ]+', allowed_exit_codes) if num != '')
+ except ValueError:
+ die('Invalid integer in ATOMIC_RSYNC_OK_CODES:', allowed_exit_codes[2:])
+
+ symlink_content = os.readlink(dest_dir) if os.path.islink(dest_dir) else None
+
+ dest_arg = dest_dir
+ dest_dir = os.path.realpath(dest_dir) # The real destination dir with all symlinks dereferenced
+ if dest_dir == '/':
+ die('You must not use "/" as the destination directory.\nUse --help for help.')
+
+ old_dir = new_dir = None
+ if symlink_content is not None and dest_dir.endswith(('-1','-2')):
+ if not symlink_content.endswith(dest_dir[-2:]):
+ die("Symlink suffix out of sync with dest_dir name:", symlink_content, 'vs', dest_dir)
+ num = 3 - int(dest_dir[-1]);
+ old_dir = None
+ new_dir = dest_dir[:-1] + str(num)
+ symlink_content = symlink_content[:-1] + str(num)
+ else:
+ old_dir = dest_dir + '~old~'
+ new_dir = dest_dir + '~new~'
+
+ cmd_args[-1] = new_dir + '/'
+
+ if old_dir is not None and os.path.isdir(old_dir):
+ shutil.rmtree(old_dir)
+ if os.path.isdir(new_dir):
+ shutil.rmtree(new_dir)
+
+ child = subprocess.run([RSYNC_PROG, '--link-dest=' + dest_dir, *cmd_args])
+ if child.returncode not in allowed_exit_codes:
+ die('The rsync copy failed with code', child.returncode, exitcode=child.returncode)
+
+ if not os.path.isdir(new_dir):
+ die('The rsync copy failed to create:', new_dir)
+
+ if old_dir is None:
+ atomic_symlink(symlink_content, dest_arg)
+ else:
+ os.rename(dest_dir, old_dir)
+ os.rename(new_dir, dest_dir)
+
+
+def atomic_symlink(target, link):
+ newlink = link + "~new~"
+ try:
+ os.unlink(newlink); # Just in case
+ except OSError:
+ pass
+ os.symlink(target, newlink)
+ os.rename(newlink, link)
+
+
+def usage_and_exit(use_stderr=False):
+ usage_msg = """\
+Usage: atomic-rsync [RSYNC-OPTIONS] [HOST:]/SOURCE/DIR/ /DEST/DIR/
+ atomic-rsync [RSYNC-OPTIONS] HOST::MOD/DIR/ /DEST/DIR/
+
+This script lets you update a hierarchy of files in an atomic way by first
+creating a new hierarchy (using hard-links to leverage the existing files),
+and then swapping the new hierarchy into place. You must be pulling files
+to a local directory, and that directory must already exist. For example:
+
+ mkdir /local/files-1
+ ln -s files-1 /local/files
+ atomic-rsync -aiv host:/remote/files/ /local/files/
+
+If /local/files is a symlink to a directory that ends in -1 or -2, the copy
+will go to the alternate suffix and the symlink will be changed to point to
+the new dir. This is a fully atomic update. If the destination is not a
+symlink (or not a symlink to a *-1 or a *-2 directory), this will instead
+create a directory with "~new~" suffixed, move the current directory to a
+name with "~old~" suffixed, and then move the ~new~ directory to the original
+destination name (this double rename is not fully atomic, but is rapid). In
+both cases, the prior destintaion directory will be preserved until the next
+update, at which point it will be deleted.
+
+By default, rsync exit-code 24 (file vanished) is allowed without halting the
+atomic update. If you want to change that, specify the environment variable
+ATOMIC_RSYNC_OK_CODES with numeric values separated by spaces and/or commas.
+Specify an empty string to only allow a successful copy. An override example:
+
+ ATOMIC_RSYNC_OK_CODES='23 24' atomic-rsync -aiv host:src/ dest/
+
+See the errcode.h file for a list of all the exit codes.
+
+See the "rsync" command for its list of options. You may not use the
+--link-dest, --compare-dest, or --copy-dest options (since this script
+uses --link-dest to make the transfer efficient).
+"""
+ print(usage_msg, file=sys.stderr if use_stderr else sys.stdout)
+ sys.exit(1 if use_stderr else 0)
+
+
+def die(*args, exitcode=1):
+ print(*args, file=sys.stderr)
+ sys.exit(exitcode)
+
+
+if __name__ == '__main__':
+ main()
+
+# vim: sw=4 et
diff --git a/support/cvs2includes b/support/cvs2includes
new file mode 100755
index 0000000..fc7f78f
--- /dev/null
+++ b/support/cvs2includes
@@ -0,0 +1,42 @@
+#!/usr/bin/env perl
+#
+# This script finds all CVS/Entries files in the current directory and below
+# and creates a local .cvsinclude file with non-inherited rules including each
+# checked-in file. Then, use this option whenever using --cvs-exclude (-C):
+#
+# -f ': .cvsinclude'
+#
+# That ensures that all checked-in files/dirs are included in the transfer.
+# (You could alternately put ": .cvsinclude" into an .rsync-filter file and
+# use the -F option, which is easier to type.)
+#
+# The downside is that you need to remember to re-run cvs2includes whenever
+# you add a new file to the project.
+use strict;
+
+open(FIND, 'find . -name CVS -type d |') or die $!;
+while (<FIND>) {
+ chomp;
+ s#^\./##;
+
+ my $entries = "$_/Entries";
+ s/CVS$/.cvsinclude/;
+ my $filter = $_;
+
+ open(ENTRIES, $entries) or die "Unable to open $entries: $!\n";
+ my @includes;
+ while (<ENTRIES>) {
+ push(@includes, $1) if m#/(.+?)/#;
+ }
+ close ENTRIES;
+ if (@includes) {
+ open(FILTER, ">$filter") or die "Unable to write $filter: $!\n";
+ print FILTER map "+ /$_\n", @includes;
+ close FILTER;
+ print "Updated $filter\n";
+ } elsif (-f $filter) {
+ unlink($filter);
+ print "Removed $filter\n";
+ }
+}
+close FIND;
diff --git a/support/deny-rsync b/support/deny-rsync
new file mode 100755
index 0000000..bd4da9e
--- /dev/null
+++ b/support/deny-rsync
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Send an error message via the rsync-protocol to a non-daemon client rsync.
+#
+# Usage: deny-rsync "message"
+
+protocol_version=29
+exit_code=4 # same as a daemon that refuses an option
+
+# e.g. byte_escape 29 => \035
+function byte_escape {
+ echo -ne "\\0$(printf "%o" $1)"
+}
+
+msg="$1"
+if [ "${#msg}" -gt 254 ]; then
+ # truncate a message that is too long for this naive script to handle
+ msg="${msg:0:251}..."
+fi
+msglen=$(( ${#msg} + 1 )) # add 1 for the newline we append below
+
+# Send protocol version. All numbers are LSB-first 4-byte ints.
+echo -ne "$(byte_escape $protocol_version)\\000\\000\\000"
+
+# Send a zero checksum seed.
+echo -ne "\\000\\000\\000\\000"
+
+# The following is equivalent to rprintf(FERROR_XFER, "%s\n", $msg).
+# 1. Message header: ((MPLEX_BASE + FERROR_XFER) << 24) + $msglen.
+echo -ne "$(byte_escape $msglen)\\000\\000\\010"
+# 2. The actual data.
+echo -E "$msg"
+
+# Make sure the client gets our message, not a write failure.
+sleep 1
+
+exit $exit_code
diff --git a/support/file-attr-restore b/support/file-attr-restore
new file mode 100755
index 0000000..2e4a21b
--- /dev/null
+++ b/support/file-attr-restore
@@ -0,0 +1,173 @@
+#!/usr/bin/env perl
+# This script will parse the output of "find ARG [ARG...] -ls" and
+# apply (at your discretion) the permissions, owner, and group info
+# it reads onto any existing files and dirs (it doesn't try to affect
+# symlinks). Run this with --help (-h) for a usage summary.
+
+use strict;
+use Getopt::Long;
+
+our($p_opt, $o_opt, $g_opt, $map_file, $dry_run, $verbosity, $help_opt);
+
+&Getopt::Long::Configure('bundling');
+&usage if !&GetOptions(
+ 'all|a' => sub { $p_opt = $o_opt = $g_opt = 1 },
+ 'perms|p' => \$p_opt,
+ 'owner|o' => \$o_opt,
+ 'groups|g' => \$g_opt,
+ 'map|m=s' => \$map_file,
+ 'dry-run|n' => \$dry_run,
+ 'help|h' => \$help_opt,
+ 'verbose|v+' => \$verbosity,
+) || $help_opt;
+
+our(%uid_hash, %gid_hash);
+
+$" = ', '; # How to join arrays referenced in double-quotes.
+
+&parse_map_file($map_file) if defined $map_file;
+
+my $detail_line = qr{
+ ^ \s* \d+ \s+ # ignore inode
+ \d+ \s+ # ignore size
+ ([-bcdlps]) # 1. File type
+ ( [-r][-w][-xsS] # 2. user-permissions
+ [-r][-w][-xsS] # group-permissions
+ [-r][-w][-xtT] ) \s+ # other-permissions
+ \d+ \s+ # ignore number of links
+ (\S+) \s+ # 3. owner
+ (\S+) \s+ # 4. group
+ (?: \d+ \s+ )? # ignore size (when present)
+ \w+ \s+ \d+ \s+ # ignore month and date
+ \d+ (?: : \d+ )? \s+ # ignore time or year
+ ([^\r\n]+) $ # 5. name
+}x;
+
+while (<>) {
+ my($type, $perms, $owner, $group, $name) = /$detail_line/;
+ die "Invalid input line $.:\n$_" unless defined $name;
+ die "A filename is not properly escaped:\n$_" unless $name =~ /^[^"\\]*(\\(\d\d\d|\D)[^"\\]*)*$/;
+ my $fn = $name;
+ $fn =~ s/\\(\d+|.)/ eval "\"\\$1\"" /eg;
+ if ($type eq '-') {
+ undef $type unless -f $fn;
+ } elsif ($type eq 'd') {
+ undef $type unless -d $fn;
+ } elsif ($type eq 'b') {
+ undef $type unless -b $fn;
+ } elsif ($type eq 'c') {
+ undef $type unless -c $fn;
+ } elsif ($type eq 'p') {
+ undef $type unless -p $fn;
+ } elsif ($type eq 's') {
+ undef $type unless -S $fn;
+ } else {
+ if ($verbosity) {
+ if ($type eq 'l') {
+ $name =~ s/ -> .*//;
+ $type = 'symlink';
+ } else {
+ $type = "type '$type'";
+ }
+ print "Skipping $name ($type ignored)\n";
+ }
+ next;
+ }
+ if (!defined $type) {
+ my $reason = -e _ ? "types don't match" : 'missing';
+ print "Skipping $name ($reason)\n";
+ next;
+ }
+ my($cur_mode, $cur_uid, $cur_gid) = (stat(_))[2,4,5];
+ $cur_mode &= 07777;
+ my $highs = join('', $perms =~ /..(.)..(.)..(.)/);
+ $highs =~ tr/-rwxSTst/00001111/;
+ $perms =~ tr/-STrwxst/00011111/;
+ my $mode = $p_opt ? oct('0b' . $highs . $perms) : $cur_mode;
+ my $uid = $o_opt ? $uid_hash{$owner} : $cur_uid;
+ if (!defined $uid) {
+ if ($owner =~ /^\d+$/) {
+ $uid = $owner;
+ } else {
+ $uid = getpwnam($owner);
+ }
+ $uid_hash{$owner} = $uid;
+ }
+ my $gid = $g_opt ? $gid_hash{$group} : $cur_gid;
+ if (!defined $gid) {
+ if ($group =~ /^\d+$/) {
+ $gid = $group;
+ } else {
+ $gid = getgrnam($group);
+ }
+ $gid_hash{$group} = $gid;
+ }
+
+ my @changes;
+ if ($mode != $cur_mode) {
+ push(@changes, 'permissions');
+ if (!$dry_run && !chmod($mode, $fn)) {
+ warn "chmod($mode, \"$name\") failed: $!\n";
+ }
+ }
+ if ($uid != $cur_uid || $gid != $cur_gid) {
+ push(@changes, 'owner') if $uid != $cur_uid;
+ push(@changes, 'group') if $gid != $cur_gid;
+ if (!$dry_run) {
+ if (!chown($uid, $gid, $fn)) {
+ warn "chown($uid, $gid, \"$name\") failed: $!\n";
+ }
+ if (($mode & 06000) && !chmod($mode, $fn)) {
+ warn "post-chown chmod($mode, \"$name\") failed: $!\n";
+ }
+ }
+ }
+ if (@changes) {
+ print "$name: changed @changes\n";
+ } elsif ($verbosity) {
+ print "$name: OK\n";
+ }
+}
+exit;
+
+sub parse_map_file
+{
+ my($fn) = @_;
+ open(IN, $fn) or die "Unable to open $fn: $!\n";
+ while (<IN>) {
+ if (/^user\s+(\S+)\s+(\S+)/) {
+ $uid_hash{$1} = $2;
+ } elsif (/^group\s+(\S+)\s+(\S+)/) {
+ $gid_hash{$1} = $2;
+ } else {
+ die "Invalid line #$. in mapfile `$fn':\n$_";
+ }
+ }
+ close IN;
+}
+
+sub usage
+{
+ die <<EOT;
+Usage: file-attr-restore [OPTIONS] FILE [FILE...]
+ -a, --all Restore all the attributes (-pog)
+ -p, --perms Restore the permissions
+ -o, --owner Restore the ownership
+ -g, --groups Restore the group
+ -m, --map=FILE Read user/group mappings from FILE
+ -n, --dry-run Don't actually make the changes
+ -v, --verbose Increase verbosity
+ -h, --help Show this help text
+
+The FILE arg(s) should have been created by running the "find"
+program with "-ls" as the output specifier.
+
+The input file for the --map option must be in this format:
+
+ user FROM TO
+ group FROM TO
+
+The "FROM" should be an user/group mentioned in the input, and the TO
+should be either a uid/gid number, or a local user/group name.
+EOT
+}
diff --git a/support/files-to-excludes b/support/files-to-excludes
new file mode 100755
index 0000000..a28955c
--- /dev/null
+++ b/support/files-to-excludes
@@ -0,0 +1,27 @@
+#!/usr/bin/env perl
+# This script takes an input of filenames and outputs a set of
+# include/exclude directives that can be used by rsync to copy
+# just the indicated files using an --exclude-from=FILE option.
+use strict;
+
+my %hash;
+
+while (<>) {
+ chomp;
+ s#^/+##;
+ my $path = '/';
+ while (m#([^/]+/)/*#g) {
+ $path .= $1;
+ print "+ $path\n" unless $hash{$path}++;
+ }
+ if (m#([^/]+)$#) {
+ print "+ $path$1\n";
+ } else {
+ delete $hash{$path};
+ }
+}
+
+foreach (sort keys %hash) {
+ print "- $_*\n";
+}
+print "- /*\n";
diff --git a/support/git-set-file-times b/support/git-set-file-times
new file mode 100755
index 0000000..e06f073
--- /dev/null
+++ b/support/git-set-file-times
@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+
+import os, re, argparse, subprocess
+from datetime import datetime
+
+NULL_COMMIT_RE = re.compile(r'\0\0commit [a-f0-9]{40}$|\0$')
+
+def main():
+ if not args.git_dir:
+ cmd = 'git rev-parse --show-toplevel 2>/dev/null || echo .'
+ top_dir = subprocess.check_output(cmd, shell=True, encoding='utf-8').strip()
+ args.git_dir = os.path.join(top_dir, '.git')
+ if not args.prefix:
+ os.chdir(top_dir)
+
+ git = [ 'git', '--git-dir=' + args.git_dir ]
+
+ if args.tree:
+ cmd = git + 'ls-tree -z -r --name-only'.split() + [ args.tree ]
+ else:
+ cmd = git + 'ls-files -z'.split()
+
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, encoding='utf-8')
+ out = proc.communicate()[0]
+ ls = set(out.split('\0'))
+ ls.discard('')
+
+ if not args.tree:
+ # All modified files keep their current mtime.
+ proc = subprocess.Popen(git + 'status -z --no-renames'.split(), stdout=subprocess.PIPE, encoding='utf-8')
+ out = proc.communicate()[0]
+ for fn in out.split('\0'):
+ if fn == '' or (fn[0] != 'M' and fn[1] != 'M'):
+ continue
+ fn = fn[3:]
+ if args.list:
+ mtime = os.lstat(fn).st_mtime
+ print_line(fn, mtime, mtime)
+ ls.discard(fn)
+
+ cmd = git + 'log -r --name-only --format=%x00commit%x20%H%n%x00commit_time%x20%ct%n --no-renames -z'.split()
+ if args.tree:
+ cmd.append(args.tree)
+ cmd += ['--'] + args.files
+
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, encoding='utf-8')
+ for line in proc.stdout:
+ line = line.strip()
+ m = re.match(r'^\0commit_time (\d+)$', line)
+ if m:
+ commit_time = int(m[1])
+ elif NULL_COMMIT_RE.search(line):
+ line = NULL_COMMIT_RE.sub('', line)
+ files = set(fn for fn in line.split('\0') if fn in ls)
+ if not files:
+ continue
+ for fn in files:
+ if args.prefix:
+ fn = args.prefix + fn
+ mtime = os.lstat(fn).st_mtime
+ if args.list:
+ print_line(fn, mtime, commit_time)
+ elif mtime != commit_time:
+ if not args.quiet:
+ print(f"Setting {fn}")
+ os.utime(fn, (commit_time, commit_time), follow_symlinks = False)
+ ls -= files
+ if not ls:
+ break
+ proc.communicate()
+
+
+def print_line(fn, mtime, commit_time):
+ if args.list > 1:
+ ts = str(commit_time).rjust(10)
+ else:
+ ts = datetime.utcfromtimestamp(commit_time).strftime("%Y-%m-%d %H:%M:%S")
+ chg = '.' if mtime == commit_time else '*'
+ print(chg, ts, fn)
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(description="Set the times of the files in the current git checkout to their last-changed time.", add_help=False)
+ parser.add_argument('--git-dir', metavar='GIT_DIR', help="The git dir to query (defaults to affecting the current git checkout).")
+ parser.add_argument('--tree', metavar='TREE-ISH', help="The tree-ish to query (defaults to the current branch).")
+ parser.add_argument('--prefix', metavar='PREFIX_STR', help="Prepend the PREFIX_STR to each filename we tweak (defaults to the top of current checkout).")
+ parser.add_argument('--quiet', '-q', action='store_true', help="Don't output the changed-file information.")
+ parser.add_argument('--list', '-l', action='count', help="List files & times instead of changing them. Repeat for Unix timestamp instead of human readable.")
+ parser.add_argument('files', metavar='FILE', nargs='*', help="Specify a subset of checked-out files to tweak.")
+ parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.")
+ args = parser.parse_args()
+ main()
+
+# vim: sw=4 et
diff --git a/support/instant-rsyncd b/support/instant-rsyncd
new file mode 100755
index 0000000..8bcfd00
--- /dev/null
+++ b/support/instant-rsyncd
@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+
+# instant-rsyncd lets you quickly set up and start a simple, unprivileged rsync
+# daemon with a single module in the current directory. I've found it
+# invaluable for quick testing, and I use it when writing a list of commands
+# that people can paste into a terminal to reproduce a daemon-related bug.
+# Sysadmins deploying an rsync daemon for the first time may find it helpful as
+# a starting point.
+#
+# Usage: instant-rsyncd MODULE PORT RSYNCD-USERNAME [RSYNC-PATH]
+# The script asks for the rsyncd user's password twice on stdin, once to set it
+# and once to log in to test the daemon.
+# -- Matt McCutchen <matt@mattmccutchen.net>
+
+set -e
+
+dir="$(pwd)"
+
+echo
+echo "This will setup an rsync daemon in $dir"
+
+if [ $# = 0 ]; then
+ IFS='' read -p 'Module name to create (or return to exit): ' module
+ [ ! "$module" ] && exit
+else
+ module="$1"
+ shift
+fi
+
+if [ $# = 0 ]; then
+ IFS='' read -p 'Port number the daemon should listen on [873]: ' port
+else
+ port="$1"
+ shift
+fi
+[ "$port" ] || port=873
+
+if [ $# = 0 ]; then
+ IFS='' read -p 'User name for authentication (empty for none): ' user
+else
+ user="$1"
+ shift
+fi
+
+if [ "$user" ]; then
+ IFS='' read -s -p 'Desired password: ' password
+ echo
+fi
+
+rsync="$1"
+[ "$rsync" ] || rsync=rsync
+
+moduledir="${dir%/}/$module"
+
+mkdir "$module"
+
+cat >rsyncd.conf <<EOF
+log file = rsyncd.log
+pid file = rsyncd.pid
+port = $port
+use chroot = no
+
+[$module]
+ path = $module
+ read only = false
+EOF
+
+if [ "$user" ]; then
+ cat >>rsyncd.conf <<-EOF
+ auth users = $user
+ secrets file = $module.secrets
+ EOF
+ touch "$module".secrets
+ chmod go-rwx "$module".secrets
+ echo "$user:$password" >"$module".secrets
+ user="$user@"
+fi
+
+cat >start <<EOF
+#!/bin/bash
+set -e
+cd \`dirname \$0\`
+! [ -e rsyncd.pid ] || {
+ echo "Is the daemon already running? If not, delete rsyncd.pid."
+ exit 1
+}
+$rsync --daemon --config=rsyncd.conf
+EOF
+chmod +x start
+
+cat >stop <<"EOF"
+#!/bin/bash
+set -e
+cd `dirname $0`
+! [ -e rsyncd.pid ] || kill -s SIGTERM $(< rsyncd.pid)
+EOF
+chmod +x stop
+
+path="rsync://$user$(hostname):$port/$module/"
+
+if ./start; then
+ sleep .2
+ echo
+ echo "I ran the start command for the daemon. The log file rsyncd.log says:"
+ echo
+ cat rsyncd.log
+ echo
+ echo "You can start and stop it with ./start and ./stop respectively."
+ echo "You can customize the configuration file rsyncd.conf."
+ echo
+ echo "Give rsync the following path to access the module:"
+ echo " $path"
+ echo
+ if [ "$user" ]; then
+ echo "Let's test the daemon now. Enter the password you chose at the prompt."
+ else
+ echo "Let's test the daemon now."
+ fi
+ echo
+ echo '$' $rsync --list-only "$path"
+ $rsync --list-only "$path"
+ echo
+ echo "You should see an empty folder; it's $moduledir."
+else
+ echo "Something went wrong. Do you see an error message?"
+fi
diff --git a/support/json-rsync-version b/support/json-rsync-version
new file mode 100755
index 0000000..31fed7f
--- /dev/null
+++ b/support/json-rsync-version
@@ -0,0 +1,93 @@
+#!/usr/bin/python3
+
+import sys, argparse, subprocess, json
+
+TWEAK_NAME = {
+ 'asm': 'asm_roll',
+ 'ASM': 'asm_roll',
+ 'hardlink_special': 'hardlink_specials',
+ 'protect_args': 'secluded_args',
+ 'protected_args': 'secluded_args',
+ 'SIMD': 'SIMD_roll',
+ }
+
+MOVE_OPTIM = set('asm_roll SIMD_roll'.split())
+
+def main():
+ if not args.rsync or args.rsync == '-':
+ ver_out = sys.stdin.read().strip()
+ else:
+ ver_out = subprocess.check_output([args.rsync, '--version', '--version'], encoding='utf-8').strip()
+ if ver_out.startswith('{'):
+ print(ver_out)
+ return
+ info = { }
+ misplaced_optims = { }
+ for line in ver_out.splitlines():
+ if line.startswith('rsync '):
+ prog, vstr, ver, pstr, vstr2, proto = line.split()
+ info['program'] = prog
+ if ver.startswith('v'):
+ ver = ver[1:]
+ info[vstr] = ver
+ if '.' not in proto:
+ proto += '.0'
+ else:
+ proto = proto.replace('.PR', '.')
+ info[pstr] = proto
+ elif line.startswith('Copyright '):
+ info['copyright'] = line[10:]
+ elif line.startswith('Web site: '):
+ info['url'] = line[10:]
+ elif line.startswith(' '):
+ if not saw_comma and ',' in line:
+ saw_comma = True
+ info[sect_name] = { }
+ if saw_comma:
+ for x in line.strip(' ,').split(', '):
+ if ' ' in x:
+ val, var = x.split(' ', 1)
+ if val == 'no':
+ val = False
+ elif val.endswith('-bit'):
+ var = var[:-1] + '_bits'
+ val = int(val.split('-')[0])
+ else:
+ var = x
+ val = True
+ var = var.replace(' ', '_').replace('-', '_')
+ if var in TWEAK_NAME:
+ var = TWEAK_NAME[var]
+ if sect_name[0] != 'o' and var in MOVE_OPTIM:
+ misplaced_optims[var] = val
+ else:
+ info[sect_name][var] = val
+ else:
+ info[sect_name] += [ x for x in line.split() if not x.startswith('(') ]
+ elif line == '':
+ break
+ else:
+ sect_name = line.strip(' :').replace(' ', '_').lower()
+ info[sect_name] = [ ]
+ saw_comma = False
+ for chk in 'capabilities optimizations'.split():
+ if chk not in info:
+ info[chk] = { }
+ if misplaced_optims:
+ info['optimizations'].update(misplaced_optims)
+ for chk in 'checksum_list compress_list daemon_auth_list'.split():
+ if chk not in info:
+ info[chk] = [ ]
+ info['license'] = 'GPLv3' if ver[0] == '3' else 'GPLv2'
+ info['caveat'] = 'rsync comes with ABSOLUTELY NO WARRANTY'
+ print(json.dumps(info))
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(description="Output rsync's version data in JSON format, even if the rsync doesn't support a native json-output method.", add_help=False)
+ parser.add_argument('rsync', nargs='?', help="Specify an rsync command to run. Otherwise stdin is consumed.")
+ parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.")
+ args = parser.parse_args()
+ main()
+
+# vim: sw=4 et
diff --git a/support/logfilter b/support/logfilter
new file mode 100755
index 0000000..29cfe69
--- /dev/null
+++ b/support/logfilter
@@ -0,0 +1,34 @@
+#!/usr/bin/env perl
+# Filter the rsync daemon log messages by module name. The log file can be
+# in either syslog format or rsync's own log-file format. Note that the
+# MODULE_NAME parameter is used in a regular-expression match in order to
+# allow regex wildcards to be used. You can also limit the output by
+# directory hierarchy in a module. Examples:
+#
+# logfilter foo /var/log/rsyncd.log # output lines for module foo
+# logfilter foo/dir /var/log/syslog # limit lines to those in dir of foo
+
+use strict;
+
+my $match = shift;
+die "Usage: logfilter MODULE_NAME [LOGFILE ...]\n" unless defined $match;
+
+my $syslog_prefix = '\w\w\w +\d+ \d\d:\d\d:\d\d \S+ rsyncd';
+my $rsyncd_prefix = '\d\d\d\d/\d\d/\d\d \d\d:\d\d:\d\d ';
+
+my %pids;
+
+while (<>) {
+ my($pid,$msg) = /^(?:$syslog_prefix|$rsyncd_prefix)\[(\d+)\]:? (.*)/o;
+ next unless defined $pid;
+ my($mod_spec) = $msg =~ /^rsync (?:on|to) (\S+) from /;
+ if (defined $mod_spec) {
+ if ($mod_spec =~ /^$match(\/\S*)?$/o) {
+ $pids{$pid} = 1;
+ } else {
+ delete $pids{$pid};
+ }
+ }
+ next unless $pids{$pid};
+ print $_;
+}
diff --git a/support/lsh b/support/lsh
new file mode 100755
index 0000000..7b3c065
--- /dev/null
+++ b/support/lsh
@@ -0,0 +1,108 @@
+#!/usr/bin/env perl
+# This is a "local shell" command that works like a remote shell but only for
+# the local host. See the usage message for more details.
+
+use strict;
+use warnings;
+use Getopt::Long;
+use English '-no_match_vars';
+
+&Getopt::Long::Configure('bundling');
+&Getopt::Long::Configure('require_order');
+GetOptions(
+ 'l=s' => \( my $login_name ),
+ '1|2|4|6|A|a|C|f|g|k|M|N|n|q|s|T|t|V|v|X|x|Y' => sub { }, # Ignore
+ 'b|c|D|e|F|i|L|m|O|o|p|R|S|w=s' => sub { }, # Ignore
+ 'no-cd' => \( my $no_chdir ),
+ 'sudo' => \( my $use_sudo ),
+ 'rrsync=s' => \( my $rrsync_dir ),
+ 'rropts=s' => \( my $rrsync_opts ),
+) or &usage;
+&usage unless @ARGV > 1;
+
+my $host = shift;
+if ($host =~ s/^([^@]+)\@//) {
+ $login_name = $1;
+}
+if ($host eq 'lh') {
+ $no_chdir = 1;
+} elsif ($host ne 'localhost') {
+ die "lsh: unable to connect to host $host\n";
+}
+
+my ($home_dir, @cmd);
+if ($login_name) {
+ my ($uid, $gid);
+ if ($login_name =~ /\D/) {
+ $uid = getpwnam($login_name);
+ die "Unknown user: $login_name\n" unless defined $uid;
+ } else {
+ $uid = $login_name;
+ }
+ ($login_name, $gid, $home_dir) = (getpwuid($uid))[0,3,7];
+ if ($use_sudo) {
+ unshift @ARGV, "cd '$home_dir' &&" unless $no_chdir;
+ unshift @cmd, qw( sudo -H -u ), $login_name;
+ $no_chdir = 1;
+ } else {
+ my $groups = "$gid $gid";
+ while (my ($grgid, $grmembers) = (getgrent)[2,3]) {
+ if ($grgid != $gid && $grmembers =~ /(^|\s)\Q$login_name\E(\s|$)/o) {
+ $groups .= " $grgid";
+ }
+ }
+
+ my ($ruid, $euid) = ($UID, $EUID);
+ $GID = $EGID = $groups;
+ $UID = $EUID = $uid;
+ die "Cannot set ruid: $! (use --sudo?)\n" if $UID == $ruid && $ruid != $uid;
+ die "Cannot set euid: $! (use --sudo?)\n" if $EUID == $euid && $euid != $uid;
+
+ $ENV{USER} = $ENV{USERNAME} = $login_name;
+ $ENV{HOME} = $home_dir;
+ }
+} else {
+ $home_dir = (getpwuid($UID))[7];
+}
+
+unless ($no_chdir) {
+ chdir $home_dir or die "Unable to chdir to $home_dir: $!\n";
+}
+
+if ($rrsync_dir) {
+ $ENV{SSH_ORIGINAL_COMMAND} = join(' ', @ARGV);
+ push @cmd, 'rrsync';
+ if ($rrsync_opts) {
+ foreach my $opt (split(/[ ,]+/, $rrsync_opts)) {
+ $opt = "-$opt" unless $opt =~ /^-/;
+ push @cmd, $opt;
+ }
+ }
+ push @cmd, $rrsync_dir;
+} else {
+ push @cmd, '/bin/sh', '-c', "@ARGV";
+}
+exec @cmd;
+die "Failed to exec: $!\n";
+
+sub usage
+{
+ die <<EOT;
+Usage: lsh [OPTIONS] localhost|lh COMMAND [...]
+
+This is a "local shell" command that works like a remote shell but only for the
+local host. This is useful for rsync testing or for running a local copy where
+the sender and the receiver need to use different options (e.g. --fake-super).
+
+Options:
+
+-l USER Choose the USER that lsh tries to become.
+--no-cd Skip the chdir \$HOME (the default with hostname "lh")
+--sudo Use sudo -H -l USER to become root or the specified USER.
+--rrsync=DIR Test rrsync restricted copying without using ssh.
+--rropts=STR The string "munge,no-del,no-lock" would pass 3 options to
+ rrsync (must be combined with --rrsync=DIR).
+
+The script also ignores a bunch of single-letter ssh options.
+EOT
+}
diff --git a/support/lsh.sh b/support/lsh.sh
new file mode 100755
index 0000000..db03422
--- /dev/null
+++ b/support/lsh.sh
@@ -0,0 +1,37 @@
+#!/bin/sh
+# This script can be used as a "remote shell" command that is only
+# capable of pretending to connect to "localhost". This is useful
+# for testing or for running a local copy where the sender and the
+# receiver needs to use different options (e.g. --fake-super). If
+# we get a -l USER option, we try to use "sudo -u USER" to run the
+# command. Supports only the hostnames "localhost" and "lh", with
+# the latter implying the --no-cd option.
+
+user=''
+do_cd=y # Default path is user's home dir (just like ssh) unless host is "lh".
+
+while : ; do
+ case "$1" in
+ -l) user="$2"; shift; shift ;;
+ -l*) user=`echo "$1" | sed 's/^-l//'`; shift ;;
+ --no-cd) do_cd=n; shift ;;
+ -*) shift ;;
+ localhost) shift; break ;;
+ lh) do_cd=n; shift; break ;;
+ *) echo "lsh: unable to connect to host $1" 1>&2; exit 1 ;;
+ esac
+done
+
+if [ "$user" ]; then
+ prefix=''
+ if [ $do_cd = y ]; then
+ home=`perl -e "print((getpwnam('$user'))[7])"`
+ prefix="cd '$home' &&"
+ fi
+ sudo -H -u "$user" sh -c "$prefix $*"
+else
+ if [ $do_cd = y ]; then
+ cd || exit 1
+ fi
+ eval "${@}"
+fi
diff --git a/support/mapfrom b/support/mapfrom
new file mode 100755
index 0000000..88946bc
--- /dev/null
+++ b/support/mapfrom
@@ -0,0 +1,15 @@
+#!/usr/bin/env perl
+# This helper script makes it easy to use a passwd or group file to map
+# values in a LOCAL transfer. For instance, if you mount a backup that
+# does not have the same passwd setup as the local machine, you can do
+# a copy FROM the backup area as follows and get the differing ID values
+# mapped just like a remote transfer FROM the backed-up machine would do:
+#
+# rsync -av --usermap=`mapfrom /mnt/backup/etc/passwd` \
+# --groupmap=`mapfrom /mnt/backup/etc/group` \
+# /mnt/backup/some/src/ /some/dest/
+
+while (<>) {
+ push @_, "$2:$1" if /^(\w+):[^:]+:(\d+)/;
+}
+print join(',', @_), "\n";
diff --git a/support/mapto b/support/mapto
new file mode 100755
index 0000000..9588752
--- /dev/null
+++ b/support/mapto
@@ -0,0 +1,15 @@
+#!/usr/bin/env perl
+# This helper script makes it easy to use a passwd or group file to map
+# values in a LOCAL transfer. For instance, if you mount a backup that
+# does not have the same passwd setup as the local machine, you can do
+# a copy TO the backup area as follows and get the differing ID values
+# mapped just like a remote transfer TO the backed-up machine would do:
+#
+# rsync -av --usermap=`mapto /mnt/backup/etc/passwd` \
+# --groupmap=`mapto /mnt/backup/etc/group` \
+# /some/src/ /mnt/backup/some/dest/
+
+while (<>) {
+ push @_, "$1:$2" if /^(\w+):[^:]+:(\d+)/;
+}
+print join(',', @_), "\n";
diff --git a/support/mnt-excl b/support/mnt-excl
new file mode 100755
index 0000000..ed7b49b
--- /dev/null
+++ b/support/mnt-excl
@@ -0,0 +1,49 @@
+#!/usr/bin/env perl
+# This script takes a command-line arg of a source directory
+# that will be passed to rsync, and generates a set of excludes
+# that will exclude all mount points from the list. This is
+# useful if you have "bind" mounts since the --one-file-system
+# option won't notice the transition to a different spot on
+# the same disk. For example:
+#
+# mnt-excl /dir | rsync --exclude-from=- ... /dir /dest/
+# mnt-excl /dir/ | rsync --exclude-from=- ... /dir/ /dest/
+# ssh host mnt-excl /dir | rsync --exclude-from=- ... host:/dir /dest/
+#
+# Imagine that /dir/foo is a mount point: the first invocation of
+# mnt-excl would have output /dir/foo, while the second would have
+# output /foo (which are the properly anchored excludes).
+#
+# NOTE: This script expects /proc/mounts to exist, but could be
+# easily adapted to read /etc/mtab or similar.
+#
+# ADDENDUM: The addition of the --filter option (which has support for
+# absolute-anchored excludes) can make this script unneeded in some
+# scenarios. If you don't need delete protection on the receiving side
+# (or if the destination path is identical to the source path), then you
+# can exclude some absolute paths from the transfer based on the mount
+# dirs. For instance:
+#
+# awk '{print $2}' /proc/mounts | grep -v '^/$' | \
+# rsync -avf 'merge,/- -' /dir host:/dest/
+
+use strict;
+use warnings;
+use Cwd 'abs_path';
+
+my $file = '/proc/mounts';
+my $dir = shift || '/';
+my $trailing_slash = $dir =~ m{./$} ? '/' : '';
+$dir = abs_path($dir) . $trailing_slash;
+$dir =~ s{([^/]*)$}{};
+my $trailing = $1;
+$trailing = '' if $trailing eq '.' || !-d "$dir$trailing";
+$trailing .= '/' if $trailing ne '';
+
+open(IN, $file) or die "Unable to open $file: $!\n";
+while (<IN>) {
+ $_ = (split)[1];
+ next unless s{^\Q$dir$trailing\E}{}o && $_ ne '';
+ print "- /$trailing$_\n";
+}
+close IN;
diff --git a/support/munge-symlinks b/support/munge-symlinks
new file mode 100755
index 0000000..e7a5474
--- /dev/null
+++ b/support/munge-symlinks
@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+# This script will either prefix all symlink values with the string
+# "/rsyncd-munged/" or remove that prefix.
+
+import os, sys, argparse
+
+SYMLINK_PREFIX = '/rsyncd-munged/'
+PREFIX_LEN = len(SYMLINK_PREFIX)
+
+def main():
+ for arg in args.names:
+ if os.path.islink(arg):
+ process_one_arg(arg)
+ elif os.path.isdir(arg):
+ for fn in find_symlinks(arg):
+ process_one_arg(fn)
+ else:
+ print("Arg is not a symlink or a dir:", arg, file=sys.stderr)
+
+
+def find_symlinks(path):
+ for entry in os.scandir(path):
+ if entry.is_symlink():
+ yield entry.path
+ elif entry.is_dir(follow_symlinks=False):
+ yield from find_symlinks(entry.path)
+
+
+def process_one_arg(fn):
+ lnk = os.readlink(fn)
+ if args.unmunge:
+ if not lnk.startswith(SYMLINK_PREFIX):
+ return
+ lnk = lnk[PREFIX_LEN:]
+ while args.all and lnk.startswith(SYMLINK_PREFIX):
+ lnk = lnk[PREFIX_LEN:]
+ else:
+ if not args.all and lnk.startswith(SYMLINK_PREFIX):
+ return
+ lnk = SYMLINK_PREFIX + lnk
+
+ try:
+ os.unlink(fn)
+ except OSError as e:
+ print("Unable to unlink symlink:", str(e), file=sys.stderr)
+ return
+ try:
+ os.symlink(lnk, fn)
+ except OSError as e:
+ print("Unable to recreate symlink", fn, '->', lnk + ':', str(e), file=sys.stderr)
+ return
+ print(fn, '->', lnk)
+
+
+if __name__ == '__main__':
+ our_desc = """\
+Adds or removes the %s prefix to/from the start of each symlink's value.
+When given the name of a directory, affects all the symlinks in that directory hierarchy.
+""" % SYMLINK_PREFIX
+ epilog = 'See the "munge symlinks" option in the rsyncd.conf manpage for more details.'
+ parser = argparse.ArgumentParser(description=our_desc, epilog=epilog, add_help=False)
+ uniq_group = parser.add_mutually_exclusive_group()
+ uniq_group.add_argument('--munge', action='store_true', help="Add the prefix to symlinks (the default).")
+ uniq_group.add_argument('--unmunge', action='store_true', help="Remove the prefix from symlinks.")
+ parser.add_argument('--all', action='store_true', help="Always adds the prefix when munging (even if already munged) or removes multiple instances of the prefix when unmunging.")
+ parser.add_argument('--help', '-h', action='help', help="Output this help message and exit.")
+ parser.add_argument('names', metavar='NAME', nargs='+', help="One or more directories and/or symlinks to process.")
+ args = parser.parse_args()
+ main()
+
+# vim: sw=4 et
diff --git a/support/nameconvert b/support/nameconvert
new file mode 100755
index 0000000..ecfe28d
--- /dev/null
+++ b/support/nameconvert
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+# This implements a simple protocol to do user & group conversions between
+# names & ids. All input and output consists of simple strings with a
+# terminating newline.
+#
+# The requests can be:
+#
+# uid ID_NUM\n -> NAME\n
+# gid ID_NUM\n -> NAME\n
+# usr NAME\n -> ID_NUM\n
+# grp NAME\n -> ID_NUM\n
+#
+# An unknown ID_NUM or NAME results in an empty return value.
+#
+# This is used by an rsync daemon when configured with the "name converter" and
+# (often) "use chroot = true". While this converter uses real user & group
+# lookups you could change it to use any mapping idiom you'd like.
+
+import sys, argparse, pwd, grp
+
+def main():
+ for line in sys.stdin:
+ try:
+ req, arg = line.rstrip().split(' ', 1)
+ except:
+ req = None
+ try:
+ if req == 'uid':
+ ans = pwd.getpwuid(int(arg)).pw_name
+ elif req == 'gid':
+ ans = grp.getgrgid(int(arg)).gr_name
+ elif req == 'usr':
+ ans = pwd.getpwnam(arg).pw_uid
+ elif req == 'grp':
+ ans = grp.getgrnam(arg).gr_gid
+ else:
+ print("Invalid request", file=sys.stderr)
+ sys.exit(1)
+ except KeyError:
+ ans = ''
+ print(ans, flush=True)
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(description="Convert users & groups between names & numbers for an rsync daemon.")
+ args = parser.parse_args()
+ main()
+
+# vim: sw=4 et
diff --git a/support/rrsync b/support/rrsync
new file mode 100755
index 0000000..94c85f5
--- /dev/null
+++ b/support/rrsync
@@ -0,0 +1,379 @@
+#!/usr/bin/env python3
+
+# Restricts rsync to subdirectory declared in .ssh/authorized_keys. See
+# the rrsync man page for details of how to make use of this script.
+
+# NOTE: install python3 braceexpand to support brace expansion in the args!
+
+# Originally a perl script by: Joe Smith <js-cgi@inwap.com> 30-Sep-2004
+# Python version by: Wayne Davison <wayne@opencoder.net>
+
+# You may configure these 2 values to your liking. See also the section of
+# short & long options if you want to disable any options that rsync accepts.
+RSYNC = '/usr/bin/rsync'
+LOGFILE = 'rrsync.log' # NOTE: the file must exist for a line to be appended!
+
+# The following options are mainly the options that a client rsync can send
+# to the server, and usually just in the one option format that the stock
+# rsync produces. However, there are some additional convenience options
+# added as well, and thus a few options are present in both the short and
+# long lists (such as --group, --owner, and --perms).
+
+# NOTE when disabling: check for both a short & long version of the option!
+
+### START of options data produced by the cull-options script. ###
+
+# To disable a short-named option, add its letter to this string:
+short_disabled = 's'
+
+# These are also disabled when the restricted dir is not "/":
+short_disabled_subdir = 'KLk'
+
+# These are all possible short options that we will accept (when not disabled above):
+short_no_arg = 'ACDEHIJKLNORSUWXbcdgklmnopqrstuvxyz' # DO NOT REMOVE ANY
+short_with_num = '@B' # DO NOT REMOVE ANY
+
+# To disable a long-named option, change its value to a -1. The values mean:
+# 0 = the option has no arg; 1 = the arg doesn't need any checking; 2 = only
+# check the arg when receiving; and 3 = always check the arg.
+long_opts = {
+ 'append': 0,
+ 'backup-dir': 2,
+ 'block-size': 1,
+ 'bwlimit': 1,
+ 'checksum-choice': 1,
+ 'checksum-seed': 1,
+ 'compare-dest': 2,
+ 'compress-choice': 1,
+ 'compress-level': 1,
+ 'copy-dest': 2,
+ 'copy-devices': -1,
+ 'copy-unsafe-links': 0,
+ 'daemon': -1,
+ 'debug': 1,
+ 'delay-updates': 0,
+ 'delete': 0,
+ 'delete-after': 0,
+ 'delete-before': 0,
+ 'delete-delay': 0,
+ 'delete-during': 0,
+ 'delete-excluded': 0,
+ 'delete-missing-args': 0,
+ 'existing': 0,
+ 'fake-super': 0,
+ 'files-from': 3,
+ 'force': 0,
+ 'from0': 0,
+ 'fsync': 0,
+ 'fuzzy': 0,
+ 'group': 0,
+ 'groupmap': 1,
+ 'hard-links': 0,
+ 'iconv': 1,
+ 'ignore-errors': 0,
+ 'ignore-existing': 0,
+ 'ignore-missing-args': 0,
+ 'ignore-times': 0,
+ 'info': 1,
+ 'inplace': 0,
+ 'link-dest': 2,
+ 'links': 0,
+ 'list-only': 0,
+ 'log-file': 3,
+ 'log-format': 1,
+ 'max-alloc': 1,
+ 'max-delete': 1,
+ 'max-size': 1,
+ 'min-size': 1,
+ 'mkpath': 0,
+ 'modify-window': 1,
+ 'msgs2stderr': 0,
+ 'munge-links': 0,
+ 'new-compress': 0,
+ 'no-W': 0,
+ 'no-implied-dirs': 0,
+ 'no-msgs2stderr': 0,
+ 'no-munge-links': -1,
+ 'no-r': 0,
+ 'no-relative': 0,
+ 'no-specials': 0,
+ 'numeric-ids': 0,
+ 'old-compress': 0,
+ 'one-file-system': 0,
+ 'only-write-batch': 1,
+ 'open-noatime': 0,
+ 'owner': 0,
+ 'partial': 0,
+ 'partial-dir': 2,
+ 'perms': 0,
+ 'preallocate': 0,
+ 'recursive': 0,
+ 'remove-sent-files': 0,
+ 'remove-source-files': 0,
+ 'safe-links': 0,
+ 'sender': 0,
+ 'server': 0,
+ 'size-only': 0,
+ 'skip-compress': 1,
+ 'specials': 0,
+ 'stats': 0,
+ 'stderr': 1,
+ 'suffix': 1,
+ 'super': 0,
+ 'temp-dir': 2,
+ 'timeout': 1,
+ 'times': 0,
+ 'use-qsort': 0,
+ 'usermap': 1,
+ 'write-devices': -1,
+}
+
+### END of options data produced by the cull-options script. ###
+
+import os, sys, re, argparse, glob, socket, time, subprocess
+from argparse import RawTextHelpFormatter
+
+try:
+ from braceexpand import braceexpand
+except:
+ braceexpand = lambda x: [ DE_BACKSLASH_RE.sub(r'\1', x) ]
+
+HAS_DOT_DOT_RE = re.compile(r'(^|/)\.\.(/|$)')
+LONG_OPT_RE = re.compile(r'^--([^=]+)(?:=(.*))?$')
+DE_BACKSLASH_RE = re.compile(r'\\(.)')
+
+def main():
+ if not os.path.isdir(args.dir):
+ die("Restricted directory does not exist!")
+
+ # The format of the environment variables set by sshd:
+ # SSH_ORIGINAL_COMMAND:
+ # rsync --server -vlogDtpre.iLsfxCIvu --etc . ARG # push
+ # rsync --server --sender -vlogDtpre.iLsfxCIvu --etc . ARGS # pull
+ # SSH_CONNECTION (client_ip client_port server_ip server_port):
+ # 192.168.1.100 64106 192.168.1.2 22
+
+ command = os.environ.get('SSH_ORIGINAL_COMMAND', None)
+ if not command:
+ die("Not invoked via sshd")
+ command = command.split(' ', 2)
+ if command[0:1] != ['rsync']:
+ die("SSH_ORIGINAL_COMMAND does not run rsync")
+ if command[1:2] != ['--server']:
+ die("--server option is not the first arg")
+ command = '' if len(command) < 3 else command[2]
+
+ global am_sender
+ am_sender = command.startswith("--sender ") # Restrictive on purpose!
+ if args.ro and not am_sender:
+ die("sending to read-only server is not allowed")
+ if args.wo and am_sender:
+ die("reading from write-only server is not allowed")
+
+ if args.wo or not am_sender:
+ long_opts['sender'] = -1
+ if args.no_del:
+ for opt in long_opts:
+ if opt.startswith(('remove', 'delete')):
+ long_opts[opt] = -1
+ if args.ro:
+ long_opts['log-file'] = -1
+
+ if args.dir != '/':
+ global short_disabled
+ short_disabled += short_disabled_subdir
+
+ short_no_arg_re = short_no_arg
+ short_with_num_re = short_with_num
+ if short_disabled:
+ for ltr in short_disabled:
+ short_no_arg_re = short_no_arg_re.replace(ltr, '')
+ short_with_num_re = short_with_num_re.replace(ltr, '')
+ short_disabled_re = re.compile(r'^-[%s]*([%s])' % (short_no_arg_re, short_disabled))
+ short_no_arg_re = re.compile(r'^-(?=.)[%s]*(e\d*\.\w*)?$' % short_no_arg_re)
+ short_with_num_re = re.compile(r'^-[%s]\d+$' % short_with_num_re)
+
+ log_fh = open(LOGFILE, 'a') if os.path.isfile(LOGFILE) else None
+
+ try:
+ os.chdir(args.dir)
+ except OSError as e:
+ die('unable to chdir to restricted dir:', str(e))
+
+ rsync_opts = [ '--server' ]
+ rsync_args = [ ]
+ saw_the_dot_arg = False
+ last_opt = check_type = None
+
+ for arg in re.findall(r'(?:[^\s\\]+|\\.[^\s\\]*)+', command):
+ if check_type:
+ rsync_opts.append(validated_arg(last_opt, arg, check_type))
+ check_type = None
+ elif saw_the_dot_arg:
+ # NOTE: an arg that starts with a '-' is safe due to our use of "--" in the cmd tuple.
+ try:
+ b_e = braceexpand(arg) # Also removes backslashes
+ except: # Handle errors such as unbalanced braces by just de-backslashing the arg:
+ b_e = [ DE_BACKSLASH_RE.sub(r'\1', arg) ]
+ for xarg in b_e:
+ rsync_args += validated_arg('arg', xarg, wild=True)
+ else: # parsing the option args
+ if arg == '.':
+ saw_the_dot_arg = True
+ continue
+ rsync_opts.append(arg)
+ if short_no_arg_re.match(arg) or short_with_num_re.match(arg):
+ continue
+ disabled = False
+ m = LONG_OPT_RE.match(arg)
+ if m:
+ opt = m.group(1)
+ opt_arg = m.group(2)
+ ct = long_opts.get(opt, None)
+ if ct is None:
+ break # Generate generic failure due to unfinished arg parsing
+ if ct == 0:
+ continue
+ opt = '--' + opt
+ if ct > 0:
+ if opt_arg is not None:
+ rsync_opts[-1] = opt + '=' + validated_arg(opt, opt_arg, ct)
+ else:
+ check_type = ct
+ last_opt = opt
+ continue
+ disabled = True
+ elif short_disabled:
+ m = short_disabled_re.match(arg)
+ if m:
+ disabled = True
+ opt = '-' + m.group(1)
+
+ if disabled:
+ die("option", opt, "has been disabled on this server.")
+ break # Generate a generic failure
+
+ if not saw_the_dot_arg:
+ die("invalid rsync-command syntax or options")
+
+ if args.munge:
+ rsync_opts.append('--munge-links')
+
+ if not rsync_args:
+ rsync_args = [ '.' ]
+
+ cmd = (RSYNC, *rsync_opts, '--', '.', *rsync_args)
+
+ if log_fh:
+ now = time.localtime()
+ host = os.environ.get('SSH_CONNECTION', 'unknown').split()[0] # Drop everything after the IP addr
+ if host.startswith('::ffff:'):
+ host = host[7:]
+ try:
+ host = socket.gethostbyaddr(socket.inet_aton(host))
+ except:
+ pass
+ log_fh.write("%02d:%02d:%02d %-16s %s\n" % (now.tm_hour, now.tm_min, now.tm_sec, host, str(cmd)))
+ log_fh.close()
+
+ # NOTE: This assumes that the rsync protocol will not be maliciously hijacked.
+ if args.no_lock:
+ os.execlp(RSYNC, *cmd)
+ die("execlp(", RSYNC, *cmd, ') failed')
+ child = subprocess.run(cmd)
+ if child.returncode != 0:
+ sys.exit(child.returncode)
+
+
+def validated_arg(opt, arg, typ=3, wild=False):
+ if opt != 'arg': # arg values already have their backslashes removed.
+ arg = DE_BACKSLASH_RE.sub(r'\1', arg)
+
+ orig_arg = arg
+ if arg.startswith('./'):
+ arg = arg[1:]
+ arg = arg.replace('//', '/')
+ if args.dir != '/':
+ if HAS_DOT_DOT_RE.search(arg):
+ die("do not use .. in", opt, "(anchor the path at the root of your restricted dir)")
+ if arg.startswith('/'):
+ arg = args.dir + arg
+
+ if wild:
+ got = glob.glob(arg)
+ if not got:
+ got = [ arg ]
+ else:
+ got = [ arg ]
+
+ ret = [ ]
+ for arg in got:
+ if args.dir != '/' and arg != '.' and (typ == 3 or (typ == 2 and not am_sender)):
+ arg_has_trailing_slash = arg.endswith('/')
+ if arg_has_trailing_slash:
+ arg = arg[:-1]
+ else:
+ arg_has_trailing_slash_dot = arg.endswith('/.')
+ if arg_has_trailing_slash_dot:
+ arg = arg[:-2]
+ real_arg = os.path.realpath(arg)
+ if arg != real_arg and not real_arg.startswith(args.dir_slash):
+ die('unsafe arg:', orig_arg, [arg, real_arg])
+ if arg_has_trailing_slash:
+ arg += '/'
+ elif arg_has_trailing_slash_dot:
+ arg += '/.'
+ if opt == 'arg' and arg.startswith(args.dir_slash):
+ arg = arg[args.dir_slash_len:]
+ if arg == '':
+ arg = '.'
+ ret.append(arg)
+
+ return ret if wild else ret[0]
+
+
+def lock_or_die(dirname):
+ import fcntl
+ global lock_handle
+ lock_handle = os.open(dirname, os.O_RDONLY)
+ try:
+ fcntl.flock(lock_handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except:
+ die('Another instance of rrsync is already accessing this directory.')
+
+
+def die(*msg):
+ print(sys.argv[0], 'error:', *msg, file=sys.stderr)
+ if sys.stdin.isatty():
+ arg_parser.print_help(sys.stderr)
+ sys.exit(1)
+
+
+# This class displays the --help to the user on argparse error IFF they're running it interactively.
+class OurArgParser(argparse.ArgumentParser):
+ def error(self, msg):
+ die(msg)
+
+
+if __name__ == '__main__':
+ our_desc = """Use "man rrsync" to learn how to restrict ssh users to using a restricted rsync command."""
+ arg_parser = OurArgParser(description=our_desc, add_help=False)
+ only_group = arg_parser.add_mutually_exclusive_group()
+ only_group.add_argument('-ro', action='store_true', help="Allow only reading from the DIR. Implies -no-del and -no-lock.")
+ only_group.add_argument('-wo', action='store_true', help="Allow only writing to the DIR.")
+ arg_parser.add_argument('-munge', action='store_true', help="Enable rsync's --munge-links on the server side.")
+ arg_parser.add_argument('-no-del', action='store_true', help="Disable rsync's --delete* and --remove* options.")
+ arg_parser.add_argument('-no-lock', action='store_true', help="Avoid the single-run (per-user) lock check.")
+ arg_parser.add_argument('-help', '-h', action='help', help="Output this help message and exit.")
+ arg_parser.add_argument('dir', metavar='DIR', help="The restricted directory to use.")
+ args = arg_parser.parse_args()
+ args.dir = os.path.realpath(args.dir)
+ args.dir_slash = args.dir + '/'
+ args.dir_slash_len = len(args.dir_slash)
+ if args.ro:
+ args.no_del = True
+ elif not args.no_lock:
+ lock_or_die(args.dir)
+ main()
+
+# vim: sw=4 et
diff --git a/support/rrsync.1.md b/support/rrsync.1.md
new file mode 100644
index 0000000..98f2cab
--- /dev/null
+++ b/support/rrsync.1.md
@@ -0,0 +1,166 @@
+## NAME
+
+rrsync - a script to setup restricted rsync users via ssh logins
+
+## SYNOPSIS
+
+```
+rrsync [-ro|-rw] [-munge] [-no-del] [-no-lock] DIR
+```
+
+The single non-option argument specifies the restricted _DIR_ to use. It can be
+relative to the user's home directory or an absolute path.
+
+The online version of this manpage (that includes cross-linking of topics)
+is available at <https://download.samba.org/pub/rsync/rrsync.1>.
+
+## DESCRIPTION
+
+A user's ssh login can be restricted to only allow the running of an rsync
+transfer in one of two easy ways:
+
+* forcing the running of the rrsync script
+* forcing the running of an rsync daemon-over-ssh command.
+
+Both of these setups use a feature of ssh that allows a command to be forced to
+run instead of an interactive shell. However, if the user's home shell is bash,
+please see [BASH SECURITY ISSUE](#) for a potential issue.
+
+To use the rrsync script, edit the user's `~/.ssh/authorized_keys` file and add
+a prefix like one of the following (followed by a space) in front of each
+ssh-key line that should be restricted:
+
+> ```
+> command="rrsync DIR"
+> command="rrsync -ro DIR"
+> command="rrsync -munge -no-del DIR"
+> ```
+
+Then, ensure that the rrsync script has your desired option restrictions. You
+may want to copy the script to a local bin dir with a unique name if you want
+to have multiple configurations. One or more rrsync options can be specified
+prior to the _DIR_ if you want to further restrict the transfer.
+
+To use an rsync daemon setup, edit the user's `~/.ssh/authorized_keys` file and
+add a prefix like one of the following (followed by a space) in front of each
+ssh-key line that should be restricted:
+
+> ```
+> command="rsync --server --daemon ."
+> command="rsync --server --daemon --config=/PATH/TO/rsyncd.conf ."
+> ```
+
+Then, ensure that the rsyncd.conf file is created with one or more module names
+with the appropriate path and option restrictions. If rsync's
+[`--config`](rsync.1#dopt) option is omitted, it defaults to `~/rsyncd.conf`.
+See the [**rsyncd.conf**(5)](rsyncd.conf.5) manpage for details of how to
+configure an rsync daemon.
+
+When using rrsync, there can be just one restricted dir per authorized key. A
+daemon setup, on the other hand, allows multiple module names inside the config
+file, each one with its own path setting.
+
+The remainder of this manpage is dedicated to using the rrsync script.
+
+## OPTIONS
+
+0. `-ro`
+
+ Allow only reading from the DIR. Implies [`-no-del`](#opt) and
+ [`-no-lock`](#opt).
+
+0. `-wo`
+
+ Allow only writing to the DIR.
+
+0. `-munge`
+
+ Enable rsync's [`--munge-links`](rsync.1#opt) on the server side.
+
+0. `-no-del`
+
+ Disable rsync's `--delete*` and `--remove*` options.
+
+0. `-no-lock`
+
+ Avoid the single-run (per-user) lock check. Useful with [`-munge`](#opt).
+
+0. `-help`, `-h`
+
+ Output this help message and exit.
+
+## SECURITY RESTRICTIONS
+
+The rrsync script validates the path arguments it is sent to try to restrict
+them to staying within the specified DIR.
+
+The rrsync script rejects rsync's [`--copy-links`](rsync.1#opt) option (by
+default) so that a copy cannot dereference a symlink within the DIR to get to a
+file outside the DIR.
+
+The rrsync script rejects rsync's [`--protect-args`](rsync.1#opt) (`-s`) option
+because it would allow options to be sent to the server-side that the script
+cannot check. If you want to support `--protect-args`, use a daemon-over-ssh
+setup.
+
+The rrsync script accepts just a subset of rsync's options that the real rsync
+uses when running the server command. A few extra convenience options are also
+included to help it to interact with BackupPC and accept some convenient user
+overrides.
+
+The script (or a copy of it) can be manually edited if you want it to customize
+the option handling.
+
+## BASH SECURITY ISSUE
+
+If your users have bash set as their home shell, bash may try to be overly
+helpful and ensure that the user's login bashrc files are run prior to
+executing the forced command. This can be a problem if the user can somehow
+update their home bashrc files, perhaps via the restricted copy, a shared home
+directory, or something similar.
+
+One simple way to avoid the issue is to switch the user to a simpler shell,
+such as dash. When choosing the new home shell, make sure that you're not
+choosing bash in disguise, as it is unclear if it avoids the security issue.
+
+Another potential fix is to ensure that the user's home directory is not a
+shared mount and that they have no means of copying files outside of their
+restricted directories. This may require you to force the enabling of symlink
+munging on the server side.
+
+A future version of openssh may have a change to the handling of forced
+commands that allows it to avoid using the user's home shell.
+
+## EXAMPLES
+
+The `~/.ssh/authorized_keys` file might have lines in it like this:
+
+> ```
+> command="rrsync client/logs" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAzG...
+> command="rrsync -ro results" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAmk...
+> ```
+
+## FILES
+
+~/.ssh/authorized_keys
+
+## SEE ALSO
+
+[**rsync**(1)](rsync.1), [**rsyncd.conf**(5)](rsyncd.conf.5)
+
+## VERSION
+
+This manpage is current for version @VERSION@ of rsync.
+
+## CREDITS
+
+rsync is distributed under the GNU General Public License. See the file
+[COPYING](COPYING) for details.
+
+An rsync web site is available at <https://rsync.samba.org/> and its github
+project is <https://github.com/WayneD/rsync>.
+
+## AUTHOR
+
+The original rrsync perl script was written by Joe Smith. Many people have
+later contributed to it. The python version was created by Wayne Davison.
diff --git a/support/rsync-no-vanished b/support/rsync-no-vanished
new file mode 100755
index 0000000..b31a5d2
--- /dev/null
+++ b/support/rsync-no-vanished
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+REAL_RSYNC=/usr/bin/rsync
+IGNOREEXIT=24
+IGNOREOUT='^(file has vanished: |rsync warning: some files vanished before they could be transferred)'
+
+# If someone installs this as "rsync", make sure we don't affect a server run.
+for arg in "${@}"; do
+ if [[ "$arg" == --server ]]; then
+ exec $REAL_RSYNC "${@}"
+ exit $? # Not reached
+ fi
+done
+
+set -o pipefail
+
+# This filters stderr without merging it with stdout:
+{ $REAL_RSYNC "${@}" 2>&1 1>&3 3>&- | grep -E -v "$IGNOREOUT"; ret=${PIPESTATUS[0]}; } 3>&1 1>&2
+
+if [[ $ret == $IGNOREEXIT ]]; then
+ ret=0
+fi
+
+exit $ret
diff --git a/support/rsync-slash-strip b/support/rsync-slash-strip
new file mode 100755
index 0000000..b57e61c
--- /dev/null
+++ b/support/rsync-slash-strip
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# This script can be used as an rsync command-line filter that strips a single
+# trailing slash from each arg. That treats "src/" the same as "src", thus
+# you need to use "src/." or "src//" for just the contents of the "src" dir.
+# (Note that command-line dir-excludes would need to use "excl//" too.)
+#
+# To use this, name it something like "rs", put it somewhere in your path, and
+# then use "rs" in place of "rsync" when you are typing your copy commands.
+
+REAL_RSYNC=/usr/bin/rsync
+
+args=()
+for arg in "${@}"; do
+ if [[ "$arg" == --server ]]; then
+ exec $REAL_RSYNC "${@}"
+ exit $? # Not reached
+ fi
+ if [[ "$arg" == / ]]; then
+ args=("${args[@]}" /)
+ else
+ args=("${args[@]}" "${arg%/}")
+ fi
+done
+exec $REAL_RSYNC "${args[@]}"
diff --git a/support/rsyncstats b/support/rsyncstats
new file mode 100755
index 0000000..99fd545
--- /dev/null
+++ b/support/rsyncstats
@@ -0,0 +1,312 @@
+#!/usr/bin/env perl
+#
+# This script parses the default logfile format produced by rsync when running
+# as a daemon with transfer logging enabled. It also parses a slightly tweaked
+# version of the default format where %o has been replaced with %i.
+#
+# This script is derived from the xferstats script that comes with wuftpd. See
+# the usage message at the bottom for the options it takes.
+#
+# Andrew Tridgell, October 1998
+
+use Getopt::Long;
+
+# You may wish to edit the next line to customize for your default log file.
+$usage_file = "/var/log/rsyncd.log";
+
+# Edit the following lines for default report settings.
+# Entries defined here will be over-ridden by the command line.
+
+$hourly_report = 0;
+$domain_report = 0;
+$total_report = 0;
+$depth_limit = 9999;
+$only_section = '';
+
+&Getopt::Long::Configure('bundling');
+&usage if !&GetOptions(
+ 'hourly-report|h' => \$hourly_report,
+ 'domain-report|d' => \$domain_report,
+ 'domain|D:s' => \$only_domain,
+ 'total-report|t' => \$total_report,
+ 'depth-limit|l:i' => \$depth_limit,
+ 'real|r' => \$real,
+ 'anon|a' => \$anon,
+ 'section|s:s' => \$only_section,
+ 'file|f:s' => \$usage_file,
+);
+
+$anon = 1 if !$real && !$anon;
+
+open(LOG, $usage_file) || die "Error opening usage log file: $usage_file\n";
+
+if ($only_domain) {
+ print "Transfer Totals include the '$only_domain' domain only.\n";
+ print "All other domains are filtered out for this report.\n\n";
+}
+
+if ($only_section) {
+ print "Transfer Totals include the '$only_section' section only.\n";
+ print "All other sections are filtered out for this report.\n\n";
+}
+
+line: while (<LOG>) {
+
+my $syslog_prefix = '\w\w\w +\d+ \d\d:\d\d:\d\d \S+ rsyncd';
+my $rsyncd_prefix = '\d\d\d\d/\d\d/\d\d \d\d:\d\d:\d\d ';
+
+ next unless ($day,$time,$op,$host,$module,$file,$bytes)
+ = m{^
+ ( \w\w\w\s+\d+ | \d+/\d\d/\d\d ) \s+ # day
+ (\d\d:\d\d:\d\d) \s+ # time
+ [^[]* \[\d+\]:? \s+ # pid (ignored)
+ (send|recv|[<>]f\S+) \s+ # op (%o or %i)
+ (\S+) \s+ # host
+ \[\d+\.\d+\.\d+\.\d+\] \s+ # IP (ignored)
+ (\S+) \s+ # module
+ \(\S*\) \s+ # user (ignored)
+ (.*) \s+ # file name
+ (\d+) # file length in bytes
+ $ }x;
+
+ # TODO actually divide the data by into send/recv categories
+ if ($op =~ /^>/) {
+ $op = 'send';
+ } elsif ($op =~ /^</) {
+ $op = 'recv';
+ }
+
+ $daytime = $day;
+ $hour = substr($time,0,2);
+
+ $file = $module . "/" . $file;
+
+ $file =~ s|//|/|mg;
+
+ @path = split(/\//, $file);
+
+ $pathkey = "";
+ for ($i=0; $i <= $#path && $i <= $depth_limit; $i++) {
+ $pathkey = $pathkey . "/" . $path[$i];
+ }
+
+ if ($only_section ne '') {
+ next unless (substr($pathkey,0,length($only_section)) eq $only_section);
+ }
+
+ $host =~ tr/A-Z/a-z/;
+
+ @address = split(/\./, $host);
+
+ $domain = $address[$#address];
+ if ( int($address[0]) > 0 || $#address < 2 )
+ { $domain = "unresolved"; }
+
+ if ($only_domain ne '') {
+ next unless (substr($domain,0,length($only_domain)) eq $only_domain);
+ }
+
+
+# printf("c=%d day=%s bytes=%d file=%s path=%s\n",
+# $#line, $daytime, $bytes, $file, $pathkey);
+
+ $xferfiles++; # total files sent
+ $xfertfiles++; # total files sent
+ $xferfiles{$daytime}++; # files per day
+ $groupfiles{$pathkey}++; # per-group accesses
+ $domainfiles{$domain}++;
+
+ $xferbytes{$daytime} += $bytes; # bytes per day
+ $domainbytes{$domain} += $bytes; # xmit bytes to domain
+ $xferbytes += $bytes; # total bytes sent
+ $groupbytes{$pathkey} += $bytes; # per-group bytes sent
+
+ $xfertfiles{$hour}++; # files per hour
+ $xfertbytes{$hour} += $bytes; # bytes per hour
+ $xfertbytes += $bytes; # total bytes sent
+}
+close LOG;
+
+#@syslist = keys %systemfiles;
+@dates = sort datecompare keys %xferbytes;
+
+if ($xferfiles == 0) {die "There was no data to process.\n";}
+
+
+print "TOTALS FOR SUMMARY PERIOD ", $dates[0], " TO ", $dates[$#dates], "\n\n";
+printf("Files Transmitted During Summary Period %12.0f\n", $xferfiles);
+printf("Bytes Transmitted During Summary Period %12.0f\n", $xferbytes);
+#printf("Systems Using Archives %12.0f\n\n", $#syslist+1);
+
+printf("Average Files Transmitted Daily %12.0f\n",
+ $xferfiles / ($#dates + 1));
+printf("Average Bytes Transmitted Daily %12.0f\n",
+ $xferbytes / ($#dates + 1));
+
+format top1 =
+
+Daily Transmission Statistics
+
+ Number Of Number of Percent Of Percent Of
+ Date Files Sent MB Sent Files Sent Bytes Sent
+--------------- ---------- ----------- ---------- ----------
+.
+
+format line1 =
+@<<<<<<<<<<<<<< @>>>>>>>>> @>>>>>>>>>> @>>>>>>> @>>>>>>>
+$date, $nfiles, $nbytes/(1024*1024), $pctfiles, $pctbytes
+.
+
+$^ = top1;
+$~ = line1;
+
+foreach $date (sort datecompare keys %xferbytes) {
+
+ $nfiles = $xferfiles{$date};
+ $nbytes = $xferbytes{$date};
+ $pctfiles = sprintf("%8.2f", 100*$xferfiles{$date} / $xferfiles);
+ $pctbytes = sprintf("%8.2f", 100*$xferbytes{$date} / $xferbytes);
+ write;
+}
+
+if ($total_report) {
+format top2 =
+
+Total Transfers from each Archive Section (By bytes)
+
+ - Percent -
+ Archive Section NFiles MB Files Bytes
+------------------------------------- ------- ----------- ----- -------
+.
+
+format line2 =
+@<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< @>>>>>> @>>>>>>>>>> @>>>> @>>>>
+$section, $files, $bytes/(1024*1024), $pctfiles, $pctbytes
+.
+
+$| = 1;
+$- = 0;
+$^ = top2;
+$~ = line2;
+
+foreach $section (sort bytecompare keys %groupfiles) {
+
+ $files = $groupfiles{$section};
+ $bytes = $groupbytes{$section};
+ $pctbytes = sprintf("%8.2f", 100 * $groupbytes{$section} / $xferbytes);
+ $pctfiles = sprintf("%8.2f", 100 * $groupfiles{$section} / $xferfiles);
+ write;
+
+}
+
+if ( $xferfiles < 1 ) { $xferfiles = 1; }
+if ( $xferbytes < 1 ) { $xferbytes = 1; }
+}
+
+if ($domain_report) {
+format top3 =
+
+Total Transfer Amount By Domain
+
+ Number Of Number of Percent Of Percent Of
+Domain Name Files Sent MB Sent Files Sent Bytes Sent
+----------- ---------- ------------ ---------- ----------
+.
+
+format line3 =
+@<<<<<<<<<< @>>>>>>>>> @>>>>>>>>>>> @>>>>>>> @>>>>>>>
+$domain, $files, $bytes/(1024*1024), $pctfiles, $pctbytes
+.
+
+$- = 0;
+$^ = top3;
+$~ = line3;
+
+foreach $domain (sort domnamcompare keys %domainfiles) {
+
+ if ( $domainsecs{$domain} < 1 ) { $domainsecs{$domain} = 1; }
+
+ $files = $domainfiles{$domain};
+ $bytes = $domainbytes{$domain};
+ $pctfiles = sprintf("%8.2f", 100 * $domainfiles{$domain} / $xferfiles);
+ $pctbytes = sprintf("%8.2f", 100 * $domainbytes{$domain} / $xferbytes);
+ write;
+
+}
+
+}
+
+if ($hourly_report) {
+
+format top8 =
+
+Hourly Transmission Statistics
+
+ Number Of Number of Percent Of Percent Of
+ Time Files Sent MB Sent Files Sent Bytes Sent
+--------------- ---------- ----------- ---------- ----------
+.
+
+format line8 =
+@<<<<<<<<<<<<<< @>>>>>>>>> @>>>>>>>>>> @>>>>>>> @>>>>>>>
+$hour, $nfiles, $nbytes/(1024*1024), $pctfiles, $pctbytes
+.
+
+
+$| = 1;
+$- = 0;
+$^ = top8;
+$~ = line8;
+
+foreach $hour (sort keys %xfertbytes) {
+
+ $nfiles = $xfertfiles{$hour};
+ $nbytes = $xfertbytes{$hour};
+ $pctfiles = sprintf("%8.2f", 100*$xfertfiles{$hour} / $xferfiles);
+ $pctbytes = sprintf("%8.2f", 100*$xfertbytes{$hour} / $xferbytes);
+ write;
+}
+}
+exit(0);
+
+sub datecompare {
+ $a cmp $b;
+}
+
+sub domnamcompare {
+
+ $sdiff = length($a) - length($b);
+ ($sdiff < 0) ? -1 : ($sdiff > 0) ? 1 : $a cmp $b;
+
+}
+
+sub bytecompare {
+
+ $bdiff = $groupbytes{$b} - $groupbytes{$a};
+ ($bdiff < 0) ? -1 : ($bdiff > 0) ? 1 : $a cmp $b;
+
+}
+
+sub faccompare {
+
+ $fdiff = $fac{$b} - $fac{$a};
+ ($fdiff < 0) ? -1 : ($fdiff > 0) ? 1 : $a cmp $b;
+
+}
+
+sub usage
+{
+ die <<EOT;
+USAGE: rsyncstats [options]
+
+OPTIONS:
+ -f FILENAME Use FILENAME for the log file.
+ -h Include report on hourly traffic.
+ -d Include report on domain traffic.
+ -t Report on total traffic by section.
+ -D DOMAIN Report only on traffic from DOMAIN.
+ -l DEPTH Set DEPTH of path detail for sections.
+ -s SECTION Set SECTION to report on. For example, "-s /pub"
+ will report only on paths under "/pub".
+EOT
+}
diff --git a/support/savetransfer.c b/support/savetransfer.c
new file mode 100644
index 0000000..808a6f2
--- /dev/null
+++ b/support/savetransfer.c
@@ -0,0 +1,175 @@
+/* This program can record the stream of data flowing to or from a program.
+ * This allows it to be used to check that rsync's data that is flowing
+ * through a remote shell is not being corrupted (for example).
+ *
+ * Usage: savetransfer [-i|-o] OUTPUT_FILE PROGRAM [ARGS...]
+ * -i Save the input going to PROGRAM to the OUTPUT_FILE
+ * -o Save the output coming from PROGRAM to the OUTPUT_FILE
+ *
+ * If you want to capture the flow of data for an rsync command, use one of
+ * the following commands (the resulting files should be identical):
+ *
+ * rsync -av --rsh="savetransfer -i /tmp/to.server ssh"
+ * --rsync-path="savetransfer -i /tmp/from.client rsync" SOURCE DEST
+ *
+ * rsync -av --rsh="savetransfer -o /tmp/from.server ssh"
+ * --rsync-path="savetransfer -o /tmp/to.client rsync" SOURCE DEST
+ *
+ * Note that this program aborts after 30 seconds of inactivity, so you'll need
+ * to change it if that is not enough dead time for your transfer. Also, some
+ * of the above commands will not notice that the transfer is done (if we're
+ * saving the input to a PROGRAM and the PROGRAM goes away: we won't notice
+ * that it's gone unless more data comes in) -- when this happens it will delay
+ * at the end of the transfer until the timeout period expires.
+ */
+
+#include "../rsync.h"
+
+#define TIMEOUT_SECONDS 30
+
+#ifdef HAVE_SIGACTION
+static struct sigaction sigact;
+#endif
+
+void run_program(char **command);
+
+char buf[4096];
+int save_data_from_program = 0;
+
+int
+main(int argc, char *argv[])
+{
+ int fd_file, len;
+ struct timeval tv;
+ fd_set fds;
+
+ argv++;
+ if (--argc && argv[0][0] == '-') {
+ if (argv[0][1] == 'o')
+ save_data_from_program = 1;
+ else if (argv[0][1] == 'i')
+ save_data_from_program = 0;
+ else {
+ fprintf(stderr, "Unknown option: %s\n", argv[0]);
+ exit(1);
+ }
+ argv++;
+ argc--;
+ }
+ if (argc < 2) {
+ fprintf(stderr, "Usage: savetransfer [-i|-o] OUTPUT_FILE PROGRAM [ARGS...]\n");
+ fprintf(stderr, "-i Save the input going to PROGRAM to the OUTPUT_FILE\n");
+ fprintf(stderr, "-o Save the output coming from PROGRAM to the OUTPUT_FILE\n");
+ exit(1);
+ }
+ if ((fd_file = open(*argv, O_WRONLY|O_TRUNC|O_CREAT|O_BINARY, 0644)) < 0) {
+ fprintf(stderr, "Unable to write to `%s': %s\n", *argv, strerror(errno));
+ exit(1);
+ }
+ set_blocking(fd_file);
+
+ SIGACTION(SIGPIPE, SIG_IGN);
+
+ run_program(argv + 1);
+
+#if defined HAVE_SETMODE && O_BINARY
+ setmode(STDIN_FILENO, O_BINARY);
+ setmode(STDOUT_FILENO, O_BINARY);
+#endif
+ set_nonblocking(STDIN_FILENO);
+ set_blocking(STDOUT_FILENO);
+
+ while (1) {
+ FD_ZERO(&fds);
+ FD_SET(STDIN_FILENO, &fds);
+ tv.tv_sec = TIMEOUT_SECONDS;
+ tv.tv_usec = 0;
+ if (!select(STDIN_FILENO+1, &fds, NULL, NULL, &tv))
+ break;
+ if (!FD_ISSET(STDIN_FILENO, &fds))
+ break;
+ if ((len = read(STDIN_FILENO, buf, sizeof buf)) <= 0)
+ break;
+ if (write(STDOUT_FILENO, buf, len) != len) {
+ fprintf(stderr, "Failed to write data to stdout: %s\n", strerror(errno));
+ exit(1);
+ }
+ if (write(fd_file, buf, len) != len) {
+ fprintf(stderr, "Failed to write data to fd_file: %s\n", strerror(errno));
+ exit(1);
+ }
+ }
+ return 0;
+}
+
+void
+run_program(char **command)
+{
+ int pipe_fds[2], ret;
+ pid_t pid;
+
+ if (pipe(pipe_fds) < 0) {
+ fprintf(stderr, "pipe failed: %s\n", strerror(errno));
+ exit(1);
+ }
+
+ if ((pid = fork()) < 0) {
+ fprintf(stderr, "fork failed: %s\n", strerror(errno));
+ exit(1);
+ }
+
+ if (pid == 0) {
+ if (save_data_from_program)
+ ret = dup2(pipe_fds[1], STDOUT_FILENO);
+ else
+ ret = dup2(pipe_fds[0], STDIN_FILENO);
+ if (ret < 0) {
+ fprintf(stderr, "Failed to dup (in child): %s\n", strerror(errno));
+ exit(1);
+ }
+ close(pipe_fds[0]);
+ close(pipe_fds[1]);
+ set_blocking(STDIN_FILENO);
+ set_blocking(STDOUT_FILENO);
+ execvp(command[0], command);
+ fprintf(stderr, "Failed to exec %s: %s\n", command[0], strerror(errno));
+ exit(1);
+ }
+
+ if (save_data_from_program)
+ ret = dup2(pipe_fds[0], STDIN_FILENO);
+ else
+ ret = dup2(pipe_fds[1], STDOUT_FILENO);
+ if (ret < 0) {
+ fprintf(stderr, "Failed to dup (in parent): %s\n", strerror(errno));
+ exit(1);
+ }
+ close(pipe_fds[0]);
+ close(pipe_fds[1]);
+}
+
+void
+set_nonblocking(int fd)
+{
+ int val;
+
+ if ((val = fcntl(fd, F_GETFL, 0)) == -1)
+ return;
+ if (!(val & NONBLOCK_FLAG)) {
+ val |= NONBLOCK_FLAG;
+ fcntl(fd, F_SETFL, val);
+ }
+}
+
+void
+set_blocking(int fd)
+{
+ int val;
+
+ if ((val = fcntl(fd, F_GETFL, 0)) < 0)
+ return;
+ if (val & NONBLOCK_FLAG) {
+ val &= ~NONBLOCK_FLAG;
+ fcntl(fd, F_SETFL, val);
+ }
+}