diff options
Diffstat (limited to 'support')
| -rw-r--r-- | support/Makefile | 6 | ||||
| -rwxr-xr-x | support/atomic-rsync | 138 | ||||
| -rwxr-xr-x | support/cvs2includes | 42 | ||||
| -rwxr-xr-x | support/deny-rsync | 36 | ||||
| -rwxr-xr-x | support/file-attr-restore | 173 | ||||
| -rwxr-xr-x | support/files-to-excludes | 27 | ||||
| -rwxr-xr-x | support/git-set-file-times | 94 | ||||
| -rwxr-xr-x | support/instant-rsyncd | 126 | ||||
| -rwxr-xr-x | support/json-rsync-version | 93 | ||||
| -rwxr-xr-x | support/logfilter | 34 | ||||
| -rwxr-xr-x | support/lsh | 108 | ||||
| -rwxr-xr-x | support/lsh.sh | 37 | ||||
| -rwxr-xr-x | support/mapfrom | 15 | ||||
| -rwxr-xr-x | support/mapto | 15 | ||||
| -rwxr-xr-x | support/mnt-excl | 49 | ||||
| -rwxr-xr-x | support/munge-symlinks | 71 | ||||
| -rwxr-xr-x | support/nameconvert | 50 | ||||
| -rwxr-xr-x | support/rrsync | 379 | ||||
| -rw-r--r-- | support/rrsync.1.md | 166 | ||||
| -rwxr-xr-x | support/rsync-no-vanished | 24 | ||||
| -rwxr-xr-x | support/rsync-slash-strip | 24 | ||||
| -rwxr-xr-x | support/rsyncstats | 312 | ||||
| -rw-r--r-- | support/savetransfer.c | 175 |
23 files changed, 2194 insertions, 0 deletions
diff --git a/support/Makefile b/support/Makefile new file mode 100644 index 0000000..c6a1e30 --- /dev/null +++ b/support/Makefile @@ -0,0 +1,6 @@ +all: savetransfer + +savetransfer: savetransfer.o + +clean: + rm -f *.o savetransfer diff --git a/support/atomic-rsync b/support/atomic-rsync new file mode 100755 index 0000000..1964090 --- /dev/null +++ b/support/atomic-rsync @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# This script lets you update a hierarchy of files in an atomic way by +# first creating a new hierarchy using rsync's --link-dest option, and +# then swapping the hierarchy into place. **See the usage message for +# more details and some important caveats!** + +import os, sys, re, subprocess, shutil + +ALT_DEST_ARG_RE = re.compile('^--[a-z][^ =]+-dest(=|$)') + +RSYNC_PROG = '/usr/bin/rsync' + +def main(): + cmd_args = sys.argv[1:] + if '--help' in cmd_args: + usage_and_exit() + + if len(cmd_args) < 2: + usage_and_exit(True) + + dest_dir = cmd_args[-1].rstrip('/') + if dest_dir == '' or dest_dir.startswith('-'): + usage_and_exit(True) + + if not os.path.isdir(dest_dir): + die(dest_dir, "is not a directory or a symlink to a dir.\nUse --help for help.") + + bad_args = [ arg for arg in cmd_args if ALT_DEST_ARG_RE.match(arg) ] + if bad_args: + die("You cannot use the", ' or '.join(bad_args), "option with atomic-rsync.\nUse --help for help.") + + # We ignore exit-code 24 (file vanished) by default. + allowed_exit_codes = '0 ' + os.environ.get('ATOMIC_RSYNC_OK_CODES', '24') + try: + allowed_exit_codes = set(int(num) for num in re.split(r'[, ]+', allowed_exit_codes) if num != '') + except ValueError: + die('Invalid integer in ATOMIC_RSYNC_OK_CODES:', allowed_exit_codes[2:]) + + symlink_content = os.readlink(dest_dir) if os.path.islink(dest_dir) else None + + dest_arg = dest_dir + dest_dir = os.path.realpath(dest_dir) # The real destination dir with all symlinks dereferenced + if dest_dir == '/': + die('You must not use "/" as the destination directory.\nUse --help for help.') + + old_dir = new_dir = None + if symlink_content is not None and dest_dir.endswith(('-1','-2')): + if not symlink_content.endswith(dest_dir[-2:]): + die("Symlink suffix out of sync with dest_dir name:", symlink_content, 'vs', dest_dir) + num = 3 - int(dest_dir[-1]); + old_dir = None + new_dir = dest_dir[:-1] + str(num) + symlink_content = symlink_content[:-1] + str(num) + else: + old_dir = dest_dir + '~old~' + new_dir = dest_dir + '~new~' + + cmd_args[-1] = new_dir + '/' + + if old_dir is not None and os.path.isdir(old_dir): + shutil.rmtree(old_dir) + if os.path.isdir(new_dir): + shutil.rmtree(new_dir) + + child = subprocess.run([RSYNC_PROG, '--link-dest=' + dest_dir, *cmd_args]) + if child.returncode not in allowed_exit_codes: + die('The rsync copy failed with code', child.returncode, exitcode=child.returncode) + + if not os.path.isdir(new_dir): + die('The rsync copy failed to create:', new_dir) + + if old_dir is None: + atomic_symlink(symlink_content, dest_arg) + else: + os.rename(dest_dir, old_dir) + os.rename(new_dir, dest_dir) + + +def atomic_symlink(target, link): + newlink = link + "~new~" + try: + os.unlink(newlink); # Just in case + except OSError: + pass + os.symlink(target, newlink) + os.rename(newlink, link) + + +def usage_and_exit(use_stderr=False): + usage_msg = """\ +Usage: atomic-rsync [RSYNC-OPTIONS] [HOST:]/SOURCE/DIR/ /DEST/DIR/ + atomic-rsync [RSYNC-OPTIONS] HOST::MOD/DIR/ /DEST/DIR/ + +This script lets you update a hierarchy of files in an atomic way by first +creating a new hierarchy (using hard-links to leverage the existing files), +and then swapping the new hierarchy into place. You must be pulling files +to a local directory, and that directory must already exist. For example: + + mkdir /local/files-1 + ln -s files-1 /local/files + atomic-rsync -aiv host:/remote/files/ /local/files/ + +If /local/files is a symlink to a directory that ends in -1 or -2, the copy +will go to the alternate suffix and the symlink will be changed to point to +the new dir. This is a fully atomic update. If the destination is not a +symlink (or not a symlink to a *-1 or a *-2 directory), this will instead +create a directory with "~new~" suffixed, move the current directory to a +name with "~old~" suffixed, and then move the ~new~ directory to the original +destination name (this double rename is not fully atomic, but is rapid). In +both cases, the prior destintaion directory will be preserved until the next +update, at which point it will be deleted. + +By default, rsync exit-code 24 (file vanished) is allowed without halting the +atomic update. If you want to change that, specify the environment variable +ATOMIC_RSYNC_OK_CODES with numeric values separated by spaces and/or commas. +Specify an empty string to only allow a successful copy. An override example: + + ATOMIC_RSYNC_OK_CODES='23 24' atomic-rsync -aiv host:src/ dest/ + +See the errcode.h file for a list of all the exit codes. + +See the "rsync" command for its list of options. You may not use the +--link-dest, --compare-dest, or --copy-dest options (since this script +uses --link-dest to make the transfer efficient). +""" + print(usage_msg, file=sys.stderr if use_stderr else sys.stdout) + sys.exit(1 if use_stderr else 0) + + +def die(*args, exitcode=1): + print(*args, file=sys.stderr) + sys.exit(exitcode) + + +if __name__ == '__main__': + main() + +# vim: sw=4 et diff --git a/support/cvs2includes b/support/cvs2includes new file mode 100755 index 0000000..fc7f78f --- /dev/null +++ b/support/cvs2includes @@ -0,0 +1,42 @@ +#!/usr/bin/env perl +# +# This script finds all CVS/Entries files in the current directory and below +# and creates a local .cvsinclude file with non-inherited rules including each +# checked-in file. Then, use this option whenever using --cvs-exclude (-C): +# +# -f ': .cvsinclude' +# +# That ensures that all checked-in files/dirs are included in the transfer. +# (You could alternately put ": .cvsinclude" into an .rsync-filter file and +# use the -F option, which is easier to type.) +# +# The downside is that you need to remember to re-run cvs2includes whenever +# you add a new file to the project. +use strict; + +open(FIND, 'find . -name CVS -type d |') or die $!; +while (<FIND>) { + chomp; + s#^\./##; + + my $entries = "$_/Entries"; + s/CVS$/.cvsinclude/; + my $filter = $_; + + open(ENTRIES, $entries) or die "Unable to open $entries: $!\n"; + my @includes; + while (<ENTRIES>) { + push(@includes, $1) if m#/(.+?)/#; + } + close ENTRIES; + if (@includes) { + open(FILTER, ">$filter") or die "Unable to write $filter: $!\n"; + print FILTER map "+ /$_\n", @includes; + close FILTER; + print "Updated $filter\n"; + } elsif (-f $filter) { + unlink($filter); + print "Removed $filter\n"; + } +} +close FIND; diff --git a/support/deny-rsync b/support/deny-rsync new file mode 100755 index 0000000..bd4da9e --- /dev/null +++ b/support/deny-rsync @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Send an error message via the rsync-protocol to a non-daemon client rsync. +# +# Usage: deny-rsync "message" + +protocol_version=29 +exit_code=4 # same as a daemon that refuses an option + +# e.g. byte_escape 29 => \035 +function byte_escape { + echo -ne "\\0$(printf "%o" $1)" +} + +msg="$1" +if [ "${#msg}" -gt 254 ]; then + # truncate a message that is too long for this naive script to handle + msg="${msg:0:251}..." +fi +msglen=$(( ${#msg} + 1 )) # add 1 for the newline we append below + +# Send protocol version. All numbers are LSB-first 4-byte ints. +echo -ne "$(byte_escape $protocol_version)\\000\\000\\000" + +# Send a zero checksum seed. +echo -ne "\\000\\000\\000\\000" + +# The following is equivalent to rprintf(FERROR_XFER, "%s\n", $msg). +# 1. Message header: ((MPLEX_BASE + FERROR_XFER) << 24) + $msglen. +echo -ne "$(byte_escape $msglen)\\000\\000\\010" +# 2. The actual data. +echo -E "$msg" + +# Make sure the client gets our message, not a write failure. +sleep 1 + +exit $exit_code diff --git a/support/file-attr-restore b/support/file-attr-restore new file mode 100755 index 0000000..2e4a21b --- /dev/null +++ b/support/file-attr-restore @@ -0,0 +1,173 @@ +#!/usr/bin/env perl +# This script will parse the output of "find ARG [ARG...] -ls" and +# apply (at your discretion) the permissions, owner, and group info +# it reads onto any existing files and dirs (it doesn't try to affect +# symlinks). Run this with --help (-h) for a usage summary. + +use strict; +use Getopt::Long; + +our($p_opt, $o_opt, $g_opt, $map_file, $dry_run, $verbosity, $help_opt); + +&Getopt::Long::Configure('bundling'); +&usage if !&GetOptions( + 'all|a' => sub { $p_opt = $o_opt = $g_opt = 1 }, + 'perms|p' => \$p_opt, + 'owner|o' => \$o_opt, + 'groups|g' => \$g_opt, + 'map|m=s' => \$map_file, + 'dry-run|n' => \$dry_run, + 'help|h' => \$help_opt, + 'verbose|v+' => \$verbosity, +) || $help_opt; + +our(%uid_hash, %gid_hash); + +$" = ', '; # How to join arrays referenced in double-quotes. + +&parse_map_file($map_file) if defined $map_file; + +my $detail_line = qr{ + ^ \s* \d+ \s+ # ignore inode + \d+ \s+ # ignore size + ([-bcdlps]) # 1. File type + ( [-r][-w][-xsS] # 2. user-permissions + [-r][-w][-xsS] # group-permissions + [-r][-w][-xtT] ) \s+ # other-permissions + \d+ \s+ # ignore number of links + (\S+) \s+ # 3. owner + (\S+) \s+ # 4. group + (?: \d+ \s+ )? # ignore size (when present) + \w+ \s+ \d+ \s+ # ignore month and date + \d+ (?: : \d+ )? \s+ # ignore time or year + ([^\r\n]+) $ # 5. name +}x; + +while (<>) { + my($type, $perms, $owner, $group, $name) = /$detail_line/; + die "Invalid input line $.:\n$_" unless defined $name; + die "A filename is not properly escaped:\n$_" unless $name =~ /^[^"\\]*(\\(\d\d\d|\D)[^"\\]*)*$/; + my $fn = $name; + $fn =~ s/\\(\d+|.)/ eval "\"\\$1\"" /eg; + if ($type eq '-') { + undef $type unless -f $fn; + } elsif ($type eq 'd') { + undef $type unless -d $fn; + } elsif ($type eq 'b') { + undef $type unless -b $fn; + } elsif ($type eq 'c') { + undef $type unless -c $fn; + } elsif ($type eq 'p') { + undef $type unless -p $fn; + } elsif ($type eq 's') { + undef $type unless -S $fn; + } else { + if ($verbosity) { + if ($type eq 'l') { + $name =~ s/ -> .*//; + $type = 'symlink'; + } else { + $type = "type '$type'"; + } + print "Skipping $name ($type ignored)\n"; + } + next; + } + if (!defined $type) { + my $reason = -e _ ? "types don't match" : 'missing'; + print "Skipping $name ($reason)\n"; + next; + } + my($cur_mode, $cur_uid, $cur_gid) = (stat(_))[2,4,5]; + $cur_mode &= 07777; + my $highs = join('', $perms =~ /..(.)..(.)..(.)/); + $highs =~ tr/-rwxSTst/00001111/; + $perms =~ tr/-STrwxst/00011111/; + my $mode = $p_opt ? oct('0b' . $highs . $perms) : $cur_mode; + my $uid = $o_opt ? $uid_hash{$owner} : $cur_uid; + if (!defined $uid) { + if ($owner =~ /^\d+$/) { + $uid = $owner; + } else { + $uid = getpwnam($owner); + } + $uid_hash{$owner} = $uid; + } + my $gid = $g_opt ? $gid_hash{$group} : $cur_gid; + if (!defined $gid) { + if ($group =~ /^\d+$/) { + $gid = $group; + } else { + $gid = getgrnam($group); + } + $gid_hash{$group} = $gid; + } + + my @changes; + if ($mode != $cur_mode) { + push(@changes, 'permissions'); + if (!$dry_run && !chmod($mode, $fn)) { + warn "chmod($mode, \"$name\") failed: $!\n"; + } + } + if ($uid != $cur_uid || $gid != $cur_gid) { + push(@changes, 'owner') if $uid != $cur_uid; + push(@changes, 'group') if $gid != $cur_gid; + if (!$dry_run) { + if (!chown($uid, $gid, $fn)) { + warn "chown($uid, $gid, \"$name\") failed: $!\n"; + } + if (($mode & 06000) && !chmod($mode, $fn)) { + warn "post-chown chmod($mode, \"$name\") failed: $!\n"; + } + } + } + if (@changes) { + print "$name: changed @changes\n"; + } elsif ($verbosity) { + print "$name: OK\n"; + } +} +exit; + +sub parse_map_file +{ + my($fn) = @_; + open(IN, $fn) or die "Unable to open $fn: $!\n"; + while (<IN>) { + if (/^user\s+(\S+)\s+(\S+)/) { + $uid_hash{$1} = $2; + } elsif (/^group\s+(\S+)\s+(\S+)/) { + $gid_hash{$1} = $2; + } else { + die "Invalid line #$. in mapfile `$fn':\n$_"; + } + } + close IN; +} + +sub usage +{ + die <<EOT; +Usage: file-attr-restore [OPTIONS] FILE [FILE...] + -a, --all Restore all the attributes (-pog) + -p, --perms Restore the permissions + -o, --owner Restore the ownership + -g, --groups Restore the group + -m, --map=FILE Read user/group mappings from FILE + -n, --dry-run Don't actually make the changes + -v, --verbose Increase verbosity + -h, --help Show this help text + +The FILE arg(s) should have been created by running the "find" +program with "-ls" as the output specifier. + +The input file for the --map option must be in this format: + + user FROM TO + group FROM TO + +The "FROM" should be an user/group mentioned in the input, and the TO +should be either a uid/gid number, or a local user/group name. +EOT +} diff --git a/support/files-to-excludes b/support/files-to-excludes new file mode 100755 index 0000000..a28955c --- /dev/null +++ b/support/files-to-excludes @@ -0,0 +1,27 @@ +#!/usr/bin/env perl +# This script takes an input of filenames and outputs a set of +# include/exclude directives that can be used by rsync to copy +# just the indicated files using an --exclude-from=FILE option. +use strict; + +my %hash; + +while (<>) { + chomp; + s#^/+##; + my $path = '/'; + while (m#([^/]+/)/*#g) { + $path .= $1; + print "+ $path\n" unless $hash{$path}++; + } + if (m#([^/]+)$#) { + print "+ $path$1\n"; + } else { + delete $hash{$path}; + } +} + +foreach (sort keys %hash) { + print "- $_*\n"; +} +print "- /*\n"; diff --git a/support/git-set-file-times b/support/git-set-file-times new file mode 100755 index 0000000..e06f073 --- /dev/null +++ b/support/git-set-file-times @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 + +import os, re, argparse, subprocess +from datetime import datetime + +NULL_COMMIT_RE = re.compile(r'\0\0commit [a-f0-9]{40}$|\0$') + +def main(): + if not args.git_dir: + cmd = 'git rev-parse --show-toplevel 2>/dev/null || echo .' + top_dir = subprocess.check_output(cmd, shell=True, encoding='utf-8').strip() + args.git_dir = os.path.join(top_dir, '.git') + if not args.prefix: + os.chdir(top_dir) + + git = [ 'git', '--git-dir=' + args.git_dir ] + + if args.tree: + cmd = git + 'ls-tree -z -r --name-only'.split() + [ args.tree ] + else: + cmd = git + 'ls-files -z'.split() + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, encoding='utf-8') + out = proc.communicate()[0] + ls = set(out.split('\0')) + ls.discard('') + + if not args.tree: + # All modified files keep their current mtime. + proc = subprocess.Popen(git + 'status -z --no-renames'.split(), stdout=subprocess.PIPE, encoding='utf-8') + out = proc.communicate()[0] + for fn in out.split('\0'): + if fn == '' or (fn[0] != 'M' and fn[1] != 'M'): + continue + fn = fn[3:] + if args.list: + mtime = os.lstat(fn).st_mtime + print_line(fn, mtime, mtime) + ls.discard(fn) + + cmd = git + 'log -r --name-only --format=%x00commit%x20%H%n%x00commit_time%x20%ct%n --no-renames -z'.split() + if args.tree: + cmd.append(args.tree) + cmd += ['--'] + args.files + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, encoding='utf-8') + for line in proc.stdout: + line = line.strip() + m = re.match(r'^\0commit_time (\d+)$', line) + if m: + commit_time = int(m[1]) + elif NULL_COMMIT_RE.search(line): + line = NULL_COMMIT_RE.sub('', line) + files = set(fn for fn in line.split('\0') if fn in ls) + if not files: + continue + for fn in files: + if args.prefix: + fn = args.prefix + fn + mtime = os.lstat(fn).st_mtime + if args.list: + print_line(fn, mtime, commit_time) + elif mtime != commit_time: + if not args.quiet: + print(f"Setting {fn}") + os.utime(fn, (commit_time, commit_time), follow_symlinks = False) + ls -= files + if not ls: + break + proc.communicate() + + +def print_line(fn, mtime, commit_time): + if args.list > 1: + ts = str(commit_time).rjust(10) + else: + ts = datetime.utcfromtimestamp(commit_time).strftime("%Y-%m-%d %H:%M:%S") + chg = '.' if mtime == commit_time else '*' + print(chg, ts, fn) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Set the times of the files in the current git checkout to their last-changed time.", add_help=False) + parser.add_argument('--git-dir', metavar='GIT_DIR', help="The git dir to query (defaults to affecting the current git checkout).") + parser.add_argument('--tree', metavar='TREE-ISH', help="The tree-ish to query (defaults to the current branch).") + parser.add_argument('--prefix', metavar='PREFIX_STR', help="Prepend the PREFIX_STR to each filename we tweak (defaults to the top of current checkout).") + parser.add_argument('--quiet', '-q', action='store_true', help="Don't output the changed-file information.") + parser.add_argument('--list', '-l', action='count', help="List files & times instead of changing them. Repeat for Unix timestamp instead of human readable.") + parser.add_argument('files', metavar='FILE', nargs='*', help="Specify a subset of checked-out files to tweak.") + parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.") + args = parser.parse_args() + main() + +# vim: sw=4 et diff --git a/support/instant-rsyncd b/support/instant-rsyncd new file mode 100755 index 0000000..8bcfd00 --- /dev/null +++ b/support/instant-rsyncd @@ -0,0 +1,126 @@ +#!/usr/bin/env bash + +# instant-rsyncd lets you quickly set up and start a simple, unprivileged rsync +# daemon with a single module in the current directory. I've found it +# invaluable for quick testing, and I use it when writing a list of commands +# that people can paste into a terminal to reproduce a daemon-related bug. +# Sysadmins deploying an rsync daemon for the first time may find it helpful as +# a starting point. +# +# Usage: instant-rsyncd MODULE PORT RSYNCD-USERNAME [RSYNC-PATH] +# The script asks for the rsyncd user's password twice on stdin, once to set it +# and once to log in to test the daemon. +# -- Matt McCutchen <matt@mattmccutchen.net> + +set -e + +dir="$(pwd)" + +echo +echo "This will setup an rsync daemon in $dir" + +if [ $# = 0 ]; then + IFS='' read -p 'Module name to create (or return to exit): ' module + [ ! "$module" ] && exit +else + module="$1" + shift +fi + +if [ $# = 0 ]; then + IFS='' read -p 'Port number the daemon should listen on [873]: ' port +else + port="$1" + shift +fi +[ "$port" ] || port=873 + +if [ $# = 0 ]; then + IFS='' read -p 'User name for authentication (empty for none): ' user +else + user="$1" + shift +fi + +if [ "$user" ]; then + IFS='' read -s -p 'Desired password: ' password + echo +fi + +rsync="$1" +[ "$rsync" ] || rsync=rsync + +moduledir="${dir%/}/$module" + +mkdir "$module" + +cat >rsyncd.conf <<EOF +log file = rsyncd.log +pid file = rsyncd.pid +port = $port +use chroot = no + +[$module] + path = $module + read only = false +EOF + +if [ "$user" ]; then + cat >>rsyncd.conf <<-EOF + auth users = $user + secrets file = $module.secrets + EOF + touch "$module".secrets + chmod go-rwx "$module".secrets + echo "$user:$password" >"$module".secrets + user="$user@" +fi + +cat >start <<EOF +#!/bin/bash +set -e +cd \`dirname \$0\` +! [ -e rsyncd.pid ] || { + echo "Is the daemon already running? If not, delete rsyncd.pid." + exit 1 +} +$rsync --daemon --config=rsyncd.conf +EOF +chmod +x start + +cat >stop <<"EOF" +#!/bin/bash +set -e +cd `dirname $0` +! [ -e rsyncd.pid ] || kill -s SIGTERM $(< rsyncd.pid) +EOF +chmod +x stop + +path="rsync://$user$(hostname):$port/$module/" + +if ./start; then + sleep .2 + echo + echo "I ran the start command for the daemon. The log file rsyncd.log says:" + echo + cat rsyncd.log + echo + echo "You can start and stop it with ./start and ./stop respectively." + echo "You can customize the configuration file rsyncd.conf." + echo + echo "Give rsync the following path to access the module:" + echo " $path" + echo + if [ "$user" ]; then + echo "Let's test the daemon now. Enter the password you chose at the prompt." + else + echo "Let's test the daemon now." + fi + echo + echo '$' $rsync --list-only "$path" + $rsync --list-only "$path" + echo + echo "You should see an empty folder; it's $moduledir." +else + echo "Something went wrong. Do you see an error message?" +fi diff --git a/support/json-rsync-version b/support/json-rsync-version new file mode 100755 index 0000000..31fed7f --- /dev/null +++ b/support/json-rsync-version @@ -0,0 +1,93 @@ +#!/usr/bin/python3 + +import sys, argparse, subprocess, json + +TWEAK_NAME = { + 'asm': 'asm_roll', + 'ASM': 'asm_roll', + 'hardlink_special': 'hardlink_specials', + 'protect_args': 'secluded_args', + 'protected_args': 'secluded_args', + 'SIMD': 'SIMD_roll', + } + +MOVE_OPTIM = set('asm_roll SIMD_roll'.split()) + +def main(): + if not args.rsync or args.rsync == '-': + ver_out = sys.stdin.read().strip() + else: + ver_out = subprocess.check_output([args.rsync, '--version', '--version'], encoding='utf-8').strip() + if ver_out.startswith('{'): + print(ver_out) + return + info = { } + misplaced_optims = { } + for line in ver_out.splitlines(): + if line.startswith('rsync '): + prog, vstr, ver, pstr, vstr2, proto = line.split() + info['program'] = prog + if ver.startswith('v'): + ver = ver[1:] + info[vstr] = ver + if '.' not in proto: + proto += '.0' + else: + proto = proto.replace('.PR', '.') + info[pstr] = proto + elif line.startswith('Copyright '): + info['copyright'] = line[10:] + elif line.startswith('Web site: '): + info['url'] = line[10:] + elif line.startswith(' '): + if not saw_comma and ',' in line: + saw_comma = True + info[sect_name] = { } + if saw_comma: + for x in line.strip(' ,').split(', '): + if ' ' in x: + val, var = x.split(' ', 1) + if val == 'no': + val = False + elif val.endswith('-bit'): + var = var[:-1] + '_bits' + val = int(val.split('-')[0]) + else: + var = x + val = True + var = var.replace(' ', '_').replace('-', '_') + if var in TWEAK_NAME: + var = TWEAK_NAME[var] + if sect_name[0] != 'o' and var in MOVE_OPTIM: + misplaced_optims[var] = val + else: + info[sect_name][var] = val + else: + info[sect_name] += [ x for x in line.split() if not x.startswith('(') ] + elif line == '': + break + else: + sect_name = line.strip(' :').replace(' ', '_').lower() + info[sect_name] = [ ] + saw_comma = False + for chk in 'capabilities optimizations'.split(): + if chk not in info: + info[chk] = { } + if misplaced_optims: + info['optimizations'].update(misplaced_optims) + for chk in 'checksum_list compress_list daemon_auth_list'.split(): + if chk not in info: + info[chk] = [ ] + info['license'] = 'GPLv3' if ver[0] == '3' else 'GPLv2' + info['caveat'] = 'rsync comes with ABSOLUTELY NO WARRANTY' + print(json.dumps(info)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Output rsync's version data in JSON format, even if the rsync doesn't support a native json-output method.", add_help=False) + parser.add_argument('rsync', nargs='?', help="Specify an rsync command to run. Otherwise stdin is consumed.") + parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.") + args = parser.parse_args() + main() + +# vim: sw=4 et diff --git a/support/logfilter b/support/logfilter new file mode 100755 index 0000000..29cfe69 --- /dev/null +++ b/support/logfilter @@ -0,0 +1,34 @@ +#!/usr/bin/env perl +# Filter the rsync daemon log messages by module name. The log file can be +# in either syslog format or rsync's own log-file format. Note that the +# MODULE_NAME parameter is used in a regular-expression match in order to +# allow regex wildcards to be used. You can also limit the output by +# directory hierarchy in a module. Examples: +# +# logfilter foo /var/log/rsyncd.log # output lines for module foo +# logfilter foo/dir /var/log/syslog # limit lines to those in dir of foo + +use strict; + +my $match = shift; +die "Usage: logfilter MODULE_NAME [LOGFILE ...]\n" unless defined $match; + +my $syslog_prefix = '\w\w\w +\d+ \d\d:\d\d:\d\d \S+ rsyncd'; +my $rsyncd_prefix = '\d\d\d\d/\d\d/\d\d \d\d:\d\d:\d\d '; + +my %pids; + +while (<>) { + my($pid,$msg) = /^(?:$syslog_prefix|$rsyncd_prefix)\[(\d+)\]:? (.*)/o; + next unless defined $pid; + my($mod_spec) = $msg =~ /^rsync (?:on|to) (\S+) from /; + if (defined $mod_spec) { + if ($mod_spec =~ /^$match(\/\S*)?$/o) { + $pids{$pid} = 1; + } else { + delete $pids{$pid}; + } + } + next unless $pids{$pid}; + print $_; +} diff --git a/support/lsh b/support/lsh new file mode 100755 index 0000000..7b3c065 --- /dev/null +++ b/support/lsh @@ -0,0 +1,108 @@ +#!/usr/bin/env perl +# This is a "local shell" command that works like a remote shell but only for +# the local host. See the usage message for more details. + +use strict; +use warnings; +use Getopt::Long; +use English '-no_match_vars'; + +&Getopt::Long::Configure('bundling'); +&Getopt::Long::Configure('require_order'); +GetOptions( + 'l=s' => \( my $login_name ), + '1|2|4|6|A|a|C|f|g|k|M|N|n|q|s|T|t|V|v|X|x|Y' => sub { }, # Ignore + 'b|c|D|e|F|i|L|m|O|o|p|R|S|w=s' => sub { }, # Ignore + 'no-cd' => \( my $no_chdir ), + 'sudo' => \( my $use_sudo ), + 'rrsync=s' => \( my $rrsync_dir ), + 'rropts=s' => \( my $rrsync_opts ), +) or &usage; +&usage unless @ARGV > 1; + +my $host = shift; +if ($host =~ s/^([^@]+)\@//) { + $login_name = $1; +} +if ($host eq 'lh') { + $no_chdir = 1; +} elsif ($host ne 'localhost') { + die "lsh: unable to connect to host $host\n"; +} + +my ($home_dir, @cmd); +if ($login_name) { + my ($uid, $gid); + if ($login_name =~ /\D/) { + $uid = getpwnam($login_name); + die "Unknown user: $login_name\n" unless defined $uid; + } else { + $uid = $login_name; + } + ($login_name, $gid, $home_dir) = (getpwuid($uid))[0,3,7]; + if ($use_sudo) { + unshift @ARGV, "cd '$home_dir' &&" unless $no_chdir; + unshift @cmd, qw( sudo -H -u ), $login_name; + $no_chdir = 1; + } else { + my $groups = "$gid $gid"; + while (my ($grgid, $grmembers) = (getgrent)[2,3]) { + if ($grgid != $gid && $grmembers =~ /(^|\s)\Q$login_name\E(\s|$)/o) { + $groups .= " $grgid"; + } + } + + my ($ruid, $euid) = ($UID, $EUID); + $GID = $EGID = $groups; + $UID = $EUID = $uid; + die "Cannot set ruid: $! (use --sudo?)\n" if $UID == $ruid && $ruid != $uid; + die "Cannot set euid: $! (use --sudo?)\n" if $EUID == $euid && $euid != $uid; + + $ENV{USER} = $ENV{USERNAME} = $login_name; + $ENV{HOME} = $home_dir; + } +} else { + $home_dir = (getpwuid($UID))[7]; +} + +unless ($no_chdir) { + chdir $home_dir or die "Unable to chdir to $home_dir: $!\n"; +} + +if ($rrsync_dir) { + $ENV{SSH_ORIGINAL_COMMAND} = join(' ', @ARGV); + push @cmd, 'rrsync'; + if ($rrsync_opts) { + foreach my $opt (split(/[ ,]+/, $rrsync_opts)) { + $opt = "-$opt" unless $opt =~ /^-/; + push @cmd, $opt; + } + } + push @cmd, $rrsync_dir; +} else { + push @cmd, '/bin/sh', '-c', "@ARGV"; +} +exec @cmd; +die "Failed to exec: $!\n"; + +sub usage +{ + die <<EOT; +Usage: lsh [OPTIONS] localhost|lh COMMAND [...] + +This is a "local shell" command that works like a remote shell but only for the +local host. This is useful for rsync testing or for running a local copy where +the sender and the receiver need to use different options (e.g. --fake-super). + +Options: + +-l USER Choose the USER that lsh tries to become. +--no-cd Skip the chdir \$HOME (the default with hostname "lh") +--sudo Use sudo -H -l USER to become root or the specified USER. +--rrsync=DIR Test rrsync restricted copying without using ssh. +--rropts=STR The string "munge,no-del,no-lock" would pass 3 options to + rrsync (must be combined with --rrsync=DIR). + +The script also ignores a bunch of single-letter ssh options. +EOT +} diff --git a/support/lsh.sh b/support/lsh.sh new file mode 100755 index 0000000..db03422 --- /dev/null +++ b/support/lsh.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# This script can be used as a "remote shell" command that is only +# capable of pretending to connect to "localhost". This is useful +# for testing or for running a local copy where the sender and the +# receiver needs to use different options (e.g. --fake-super). If +# we get a -l USER option, we try to use "sudo -u USER" to run the +# command. Supports only the hostnames "localhost" and "lh", with +# the latter implying the --no-cd option. + +user='' +do_cd=y # Default path is user's home dir (just like ssh) unless host is "lh". + +while : ; do + case "$1" in + -l) user="$2"; shift; shift ;; + -l*) user=`echo "$1" | sed 's/^-l//'`; shift ;; + --no-cd) do_cd=n; shift ;; + -*) shift ;; + localhost) shift; break ;; + lh) do_cd=n; shift; break ;; + *) echo "lsh: unable to connect to host $1" 1>&2; exit 1 ;; + esac +done + +if [ "$user" ]; then + prefix='' + if [ $do_cd = y ]; then + home=`perl -e "print((getpwnam('$user'))[7])"` + prefix="cd '$home' &&" + fi + sudo -H -u "$user" sh -c "$prefix $*" +else + if [ $do_cd = y ]; then + cd || exit 1 + fi + eval "${@}" +fi diff --git a/support/mapfrom b/support/mapfrom new file mode 100755 index 0000000..88946bc --- /dev/null +++ b/support/mapfrom @@ -0,0 +1,15 @@ +#!/usr/bin/env perl +# This helper script makes it easy to use a passwd or group file to map +# values in a LOCAL transfer. For instance, if you mount a backup that +# does not have the same passwd setup as the local machine, you can do +# a copy FROM the backup area as follows and get the differing ID values +# mapped just like a remote transfer FROM the backed-up machine would do: +# +# rsync -av --usermap=`mapfrom /mnt/backup/etc/passwd` \ +# --groupmap=`mapfrom /mnt/backup/etc/group` \ +# /mnt/backup/some/src/ /some/dest/ + +while (<>) { + push @_, "$2:$1" if /^(\w+):[^:]+:(\d+)/; +} +print join(',', @_), "\n"; diff --git a/support/mapto b/support/mapto new file mode 100755 index 0000000..9588752 --- /dev/null +++ b/support/mapto @@ -0,0 +1,15 @@ +#!/usr/bin/env perl +# This helper script makes it easy to use a passwd or group file to map +# values in a LOCAL transfer. For instance, if you mount a backup that +# does not have the same passwd setup as the local machine, you can do +# a copy TO the backup area as follows and get the differing ID values +# mapped just like a remote transfer TO the backed-up machine would do: +# +# rsync -av --usermap=`mapto /mnt/backup/etc/passwd` \ +# --groupmap=`mapto /mnt/backup/etc/group` \ +# /some/src/ /mnt/backup/some/dest/ + +while (<>) { + push @_, "$1:$2" if /^(\w+):[^:]+:(\d+)/; +} +print join(',', @_), "\n"; diff --git a/support/mnt-excl b/support/mnt-excl new file mode 100755 index 0000000..ed7b49b --- /dev/null +++ b/support/mnt-excl @@ -0,0 +1,49 @@ +#!/usr/bin/env perl +# This script takes a command-line arg of a source directory +# that will be passed to rsync, and generates a set of excludes +# that will exclude all mount points from the list. This is +# useful if you have "bind" mounts since the --one-file-system +# option won't notice the transition to a different spot on +# the same disk. For example: +# +# mnt-excl /dir | rsync --exclude-from=- ... /dir /dest/ +# mnt-excl /dir/ | rsync --exclude-from=- ... /dir/ /dest/ +# ssh host mnt-excl /dir | rsync --exclude-from=- ... host:/dir /dest/ +# +# Imagine that /dir/foo is a mount point: the first invocation of +# mnt-excl would have output /dir/foo, while the second would have +# output /foo (which are the properly anchored excludes). +# +# NOTE: This script expects /proc/mounts to exist, but could be +# easily adapted to read /etc/mtab or similar. +# +# ADDENDUM: The addition of the --filter option (which has support for +# absolute-anchored excludes) can make this script unneeded in some +# scenarios. If you don't need delete protection on the receiving side +# (or if the destination path is identical to the source path), then you +# can exclude some absolute paths from the transfer based on the mount +# dirs. For instance: +# +# awk '{print $2}' /proc/mounts | grep -v '^/$' | \ +# rsync -avf 'merge,/- -' /dir host:/dest/ + +use strict; +use warnings; +use Cwd 'abs_path'; + +my $file = '/proc/mounts'; +my $dir = shift || '/'; +my $trailing_slash = $dir =~ m{./$} ? '/' : ''; +$dir = abs_path($dir) . $trailing_slash; +$dir =~ s{([^/]*)$}{}; +my $trailing = $1; +$trailing = '' if $trailing eq '.' || !-d "$dir$trailing"; +$trailing .= '/' if $trailing ne ''; + +open(IN, $file) or die "Unable to open $file: $!\n"; +while (<IN>) { + $_ = (split)[1]; + next unless s{^\Q$dir$trailing\E}{}o && $_ ne ''; + print "- /$trailing$_\n"; +} +close IN; diff --git a/support/munge-symlinks b/support/munge-symlinks new file mode 100755 index 0000000..e7a5474 --- /dev/null +++ b/support/munge-symlinks @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# This script will either prefix all symlink values with the string +# "/rsyncd-munged/" or remove that prefix. + +import os, sys, argparse + +SYMLINK_PREFIX = '/rsyncd-munged/' +PREFIX_LEN = len(SYMLINK_PREFIX) + +def main(): + for arg in args.names: + if os.path.islink(arg): + process_one_arg(arg) + elif os.path.isdir(arg): + for fn in find_symlinks(arg): + process_one_arg(fn) + else: + print("Arg is not a symlink or a dir:", arg, file=sys.stderr) + + +def find_symlinks(path): + for entry in os.scandir(path): + if entry.is_symlink(): + yield entry.path + elif entry.is_dir(follow_symlinks=False): + yield from find_symlinks(entry.path) + + +def process_one_arg(fn): + lnk = os.readlink(fn) + if args.unmunge: + if not lnk.startswith(SYMLINK_PREFIX): + return + lnk = lnk[PREFIX_LEN:] + while args.all and lnk.startswith(SYMLINK_PREFIX): + lnk = lnk[PREFIX_LEN:] + else: + if not args.all and lnk.startswith(SYMLINK_PREFIX): + return + lnk = SYMLINK_PREFIX + lnk + + try: + os.unlink(fn) + except OSError as e: + print("Unable to unlink symlink:", str(e), file=sys.stderr) + return + try: + os.symlink(lnk, fn) + except OSError as e: + print("Unable to recreate symlink", fn, '->', lnk + ':', str(e), file=sys.stderr) + return + print(fn, '->', lnk) + + +if __name__ == '__main__': + our_desc = """\ +Adds or removes the %s prefix to/from the start of each symlink's value. +When given the name of a directory, affects all the symlinks in that directory hierarchy. +""" % SYMLINK_PREFIX + epilog = 'See the "munge symlinks" option in the rsyncd.conf manpage for more details.' + parser = argparse.ArgumentParser(description=our_desc, epilog=epilog, add_help=False) + uniq_group = parser.add_mutually_exclusive_group() + uniq_group.add_argument('--munge', action='store_true', help="Add the prefix to symlinks (the default).") + uniq_group.add_argument('--unmunge', action='store_true', help="Remove the prefix from symlinks.") + parser.add_argument('--all', action='store_true', help="Always adds the prefix when munging (even if already munged) or removes multiple instances of the prefix when unmunging.") + parser.add_argument('--help', '-h', action='help', help="Output this help message and exit.") + parser.add_argument('names', metavar='NAME', nargs='+', help="One or more directories and/or symlinks to process.") + args = parser.parse_args() + main() + +# vim: sw=4 et diff --git a/support/nameconvert b/support/nameconvert new file mode 100755 index 0000000..ecfe28d --- /dev/null +++ b/support/nameconvert @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +# This implements a simple protocol to do user & group conversions between +# names & ids. All input and output consists of simple strings with a +# terminating newline. +# +# The requests can be: +# +# uid ID_NUM\n -> NAME\n +# gid ID_NUM\n -> NAME\n +# usr NAME\n -> ID_NUM\n +# grp NAME\n -> ID_NUM\n +# +# An unknown ID_NUM or NAME results in an empty return value. +# +# This is used by an rsync daemon when configured with the "name converter" and +# (often) "use chroot = true". While this converter uses real user & group +# lookups you could change it to use any mapping idiom you'd like. + +import sys, argparse, pwd, grp + +def main(): + for line in sys.stdin: + try: + req, arg = line.rstrip().split(' ', 1) + except: + req = None + try: + if req == 'uid': + ans = pwd.getpwuid(int(arg)).pw_name + elif req == 'gid': + ans = grp.getgrgid(int(arg)).gr_name + elif req == 'usr': + ans = pwd.getpwnam(arg).pw_uid + elif req == 'grp': + ans = grp.getgrnam(arg).gr_gid + else: + print("Invalid request", file=sys.stderr) + sys.exit(1) + except KeyError: + ans = '' + print(ans, flush=True) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Convert users & groups between names & numbers for an rsync daemon.") + args = parser.parse_args() + main() + +# vim: sw=4 et diff --git a/support/rrsync b/support/rrsync new file mode 100755 index 0000000..94c85f5 --- /dev/null +++ b/support/rrsync @@ -0,0 +1,379 @@ +#!/usr/bin/env python3 + +# Restricts rsync to subdirectory declared in .ssh/authorized_keys. See +# the rrsync man page for details of how to make use of this script. + +# NOTE: install python3 braceexpand to support brace expansion in the args! + +# Originally a perl script by: Joe Smith <js-cgi@inwap.com> 30-Sep-2004 +# Python version by: Wayne Davison <wayne@opencoder.net> + +# You may configure these 2 values to your liking. See also the section of +# short & long options if you want to disable any options that rsync accepts. +RSYNC = '/usr/bin/rsync' +LOGFILE = 'rrsync.log' # NOTE: the file must exist for a line to be appended! + +# The following options are mainly the options that a client rsync can send +# to the server, and usually just in the one option format that the stock +# rsync produces. However, there are some additional convenience options +# added as well, and thus a few options are present in both the short and +# long lists (such as --group, --owner, and --perms). + +# NOTE when disabling: check for both a short & long version of the option! + +### START of options data produced by the cull-options script. ### + +# To disable a short-named option, add its letter to this string: +short_disabled = 's' + +# These are also disabled when the restricted dir is not "/": +short_disabled_subdir = 'KLk' + +# These are all possible short options that we will accept (when not disabled above): +short_no_arg = 'ACDEHIJKLNORSUWXbcdgklmnopqrstuvxyz' # DO NOT REMOVE ANY +short_with_num = '@B' # DO NOT REMOVE ANY + +# To disable a long-named option, change its value to a -1. The values mean: +# 0 = the option has no arg; 1 = the arg doesn't need any checking; 2 = only +# check the arg when receiving; and 3 = always check the arg. +long_opts = { + 'append': 0, + 'backup-dir': 2, + 'block-size': 1, + 'bwlimit': 1, + 'checksum-choice': 1, + 'checksum-seed': 1, + 'compare-dest': 2, + 'compress-choice': 1, + 'compress-level': 1, + 'copy-dest': 2, + 'copy-devices': -1, + 'copy-unsafe-links': 0, + 'daemon': -1, + 'debug': 1, + 'delay-updates': 0, + 'delete': 0, + 'delete-after': 0, + 'delete-before': 0, + 'delete-delay': 0, + 'delete-during': 0, + 'delete-excluded': 0, + 'delete-missing-args': 0, + 'existing': 0, + 'fake-super': 0, + 'files-from': 3, + 'force': 0, + 'from0': 0, + 'fsync': 0, + 'fuzzy': 0, + 'group': 0, + 'groupmap': 1, + 'hard-links': 0, + 'iconv': 1, + 'ignore-errors': 0, + 'ignore-existing': 0, + 'ignore-missing-args': 0, + 'ignore-times': 0, + 'info': 1, + 'inplace': 0, + 'link-dest': 2, + 'links': 0, + 'list-only': 0, + 'log-file': 3, + 'log-format': 1, + 'max-alloc': 1, + 'max-delete': 1, + 'max-size': 1, + 'min-size': 1, + 'mkpath': 0, + 'modify-window': 1, + 'msgs2stderr': 0, + 'munge-links': 0, + 'new-compress': 0, + 'no-W': 0, + 'no-implied-dirs': 0, + 'no-msgs2stderr': 0, + 'no-munge-links': -1, + 'no-r': 0, + 'no-relative': 0, + 'no-specials': 0, + 'numeric-ids': 0, + 'old-compress': 0, + 'one-file-system': 0, + 'only-write-batch': 1, + 'open-noatime': 0, + 'owner': 0, + 'partial': 0, + 'partial-dir': 2, + 'perms': 0, + 'preallocate': 0, + 'recursive': 0, + 'remove-sent-files': 0, + 'remove-source-files': 0, + 'safe-links': 0, + 'sender': 0, + 'server': 0, + 'size-only': 0, + 'skip-compress': 1, + 'specials': 0, + 'stats': 0, + 'stderr': 1, + 'suffix': 1, + 'super': 0, + 'temp-dir': 2, + 'timeout': 1, + 'times': 0, + 'use-qsort': 0, + 'usermap': 1, + 'write-devices': -1, +} + +### END of options data produced by the cull-options script. ### + +import os, sys, re, argparse, glob, socket, time, subprocess +from argparse import RawTextHelpFormatter + +try: + from braceexpand import braceexpand +except: + braceexpand = lambda x: [ DE_BACKSLASH_RE.sub(r'\1', x) ] + +HAS_DOT_DOT_RE = re.compile(r'(^|/)\.\.(/|$)') +LONG_OPT_RE = re.compile(r'^--([^=]+)(?:=(.*))?$') +DE_BACKSLASH_RE = re.compile(r'\\(.)') + +def main(): + if not os.path.isdir(args.dir): + die("Restricted directory does not exist!") + + # The format of the environment variables set by sshd: + # SSH_ORIGINAL_COMMAND: + # rsync --server -vlogDtpre.iLsfxCIvu --etc . ARG # push + # rsync --server --sender -vlogDtpre.iLsfxCIvu --etc . ARGS # pull + # SSH_CONNECTION (client_ip client_port server_ip server_port): + # 192.168.1.100 64106 192.168.1.2 22 + + command = os.environ.get('SSH_ORIGINAL_COMMAND', None) + if not command: + die("Not invoked via sshd") + command = command.split(' ', 2) + if command[0:1] != ['rsync']: + die("SSH_ORIGINAL_COMMAND does not run rsync") + if command[1:2] != ['--server']: + die("--server option is not the first arg") + command = '' if len(command) < 3 else command[2] + + global am_sender + am_sender = command.startswith("--sender ") # Restrictive on purpose! + if args.ro and not am_sender: + die("sending to read-only server is not allowed") + if args.wo and am_sender: + die("reading from write-only server is not allowed") + + if args.wo or not am_sender: + long_opts['sender'] = -1 + if args.no_del: + for opt in long_opts: + if opt.startswith(('remove', 'delete')): + long_opts[opt] = -1 + if args.ro: + long_opts['log-file'] = -1 + + if args.dir != '/': + global short_disabled + short_disabled += short_disabled_subdir + + short_no_arg_re = short_no_arg + short_with_num_re = short_with_num + if short_disabled: + for ltr in short_disabled: + short_no_arg_re = short_no_arg_re.replace(ltr, '') + short_with_num_re = short_with_num_re.replace(ltr, '') + short_disabled_re = re.compile(r'^-[%s]*([%s])' % (short_no_arg_re, short_disabled)) + short_no_arg_re = re.compile(r'^-(?=.)[%s]*(e\d*\.\w*)?$' % short_no_arg_re) + short_with_num_re = re.compile(r'^-[%s]\d+$' % short_with_num_re) + + log_fh = open(LOGFILE, 'a') if os.path.isfile(LOGFILE) else None + + try: + os.chdir(args.dir) + except OSError as e: + die('unable to chdir to restricted dir:', str(e)) + + rsync_opts = [ '--server' ] + rsync_args = [ ] + saw_the_dot_arg = False + last_opt = check_type = None + + for arg in re.findall(r'(?:[^\s\\]+|\\.[^\s\\]*)+', command): + if check_type: + rsync_opts.append(validated_arg(last_opt, arg, check_type)) + check_type = None + elif saw_the_dot_arg: + # NOTE: an arg that starts with a '-' is safe due to our use of "--" in the cmd tuple. + try: + b_e = braceexpand(arg) # Also removes backslashes + except: # Handle errors such as unbalanced braces by just de-backslashing the arg: + b_e = [ DE_BACKSLASH_RE.sub(r'\1', arg) ] + for xarg in b_e: + rsync_args += validated_arg('arg', xarg, wild=True) + else: # parsing the option args + if arg == '.': + saw_the_dot_arg = True + continue + rsync_opts.append(arg) + if short_no_arg_re.match(arg) or short_with_num_re.match(arg): + continue + disabled = False + m = LONG_OPT_RE.match(arg) + if m: + opt = m.group(1) + opt_arg = m.group(2) + ct = long_opts.get(opt, None) + if ct is None: + break # Generate generic failure due to unfinished arg parsing + if ct == 0: + continue + opt = '--' + opt + if ct > 0: + if opt_arg is not None: + rsync_opts[-1] = opt + '=' + validated_arg(opt, opt_arg, ct) + else: + check_type = ct + last_opt = opt + continue + disabled = True + elif short_disabled: + m = short_disabled_re.match(arg) + if m: + disabled = True + opt = '-' + m.group(1) + + if disabled: + die("option", opt, "has been disabled on this server.") + break # Generate a generic failure + + if not saw_the_dot_arg: + die("invalid rsync-command syntax or options") + + if args.munge: + rsync_opts.append('--munge-links') + + if not rsync_args: + rsync_args = [ '.' ] + + cmd = (RSYNC, *rsync_opts, '--', '.', *rsync_args) + + if log_fh: + now = time.localtime() + host = os.environ.get('SSH_CONNECTION', 'unknown').split()[0] # Drop everything after the IP addr + if host.startswith('::ffff:'): + host = host[7:] + try: + host = socket.gethostbyaddr(socket.inet_aton(host)) + except: + pass + log_fh.write("%02d:%02d:%02d %-16s %s\n" % (now.tm_hour, now.tm_min, now.tm_sec, host, str(cmd))) + log_fh.close() + + # NOTE: This assumes that the rsync protocol will not be maliciously hijacked. + if args.no_lock: + os.execlp(RSYNC, *cmd) + die("execlp(", RSYNC, *cmd, ') failed') + child = subprocess.run(cmd) + if child.returncode != 0: + sys.exit(child.returncode) + + +def validated_arg(opt, arg, typ=3, wild=False): + if opt != 'arg': # arg values already have their backslashes removed. + arg = DE_BACKSLASH_RE.sub(r'\1', arg) + + orig_arg = arg + if arg.startswith('./'): + arg = arg[1:] + arg = arg.replace('//', '/') + if args.dir != '/': + if HAS_DOT_DOT_RE.search(arg): + die("do not use .. in", opt, "(anchor the path at the root of your restricted dir)") + if arg.startswith('/'): + arg = args.dir + arg + + if wild: + got = glob.glob(arg) + if not got: + got = [ arg ] + else: + got = [ arg ] + + ret = [ ] + for arg in got: + if args.dir != '/' and arg != '.' and (typ == 3 or (typ == 2 and not am_sender)): + arg_has_trailing_slash = arg.endswith('/') + if arg_has_trailing_slash: + arg = arg[:-1] + else: + arg_has_trailing_slash_dot = arg.endswith('/.') + if arg_has_trailing_slash_dot: + arg = arg[:-2] + real_arg = os.path.realpath(arg) + if arg != real_arg and not real_arg.startswith(args.dir_slash): + die('unsafe arg:', orig_arg, [arg, real_arg]) + if arg_has_trailing_slash: + arg += '/' + elif arg_has_trailing_slash_dot: + arg += '/.' + if opt == 'arg' and arg.startswith(args.dir_slash): + arg = arg[args.dir_slash_len:] + if arg == '': + arg = '.' + ret.append(arg) + + return ret if wild else ret[0] + + +def lock_or_die(dirname): + import fcntl + global lock_handle + lock_handle = os.open(dirname, os.O_RDONLY) + try: + fcntl.flock(lock_handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + except: + die('Another instance of rrsync is already accessing this directory.') + + +def die(*msg): + print(sys.argv[0], 'error:', *msg, file=sys.stderr) + if sys.stdin.isatty(): + arg_parser.print_help(sys.stderr) + sys.exit(1) + + +# This class displays the --help to the user on argparse error IFF they're running it interactively. +class OurArgParser(argparse.ArgumentParser): + def error(self, msg): + die(msg) + + +if __name__ == '__main__': + our_desc = """Use "man rrsync" to learn how to restrict ssh users to using a restricted rsync command.""" + arg_parser = OurArgParser(description=our_desc, add_help=False) + only_group = arg_parser.add_mutually_exclusive_group() + only_group.add_argument('-ro', action='store_true', help="Allow only reading from the DIR. Implies -no-del and -no-lock.") + only_group.add_argument('-wo', action='store_true', help="Allow only writing to the DIR.") + arg_parser.add_argument('-munge', action='store_true', help="Enable rsync's --munge-links on the server side.") + arg_parser.add_argument('-no-del', action='store_true', help="Disable rsync's --delete* and --remove* options.") + arg_parser.add_argument('-no-lock', action='store_true', help="Avoid the single-run (per-user) lock check.") + arg_parser.add_argument('-help', '-h', action='help', help="Output this help message and exit.") + arg_parser.add_argument('dir', metavar='DIR', help="The restricted directory to use.") + args = arg_parser.parse_args() + args.dir = os.path.realpath(args.dir) + args.dir_slash = args.dir + '/' + args.dir_slash_len = len(args.dir_slash) + if args.ro: + args.no_del = True + elif not args.no_lock: + lock_or_die(args.dir) + main() + +# vim: sw=4 et diff --git a/support/rrsync.1.md b/support/rrsync.1.md new file mode 100644 index 0000000..98f2cab --- /dev/null +++ b/support/rrsync.1.md @@ -0,0 +1,166 @@ +## NAME + +rrsync - a script to setup restricted rsync users via ssh logins + +## SYNOPSIS + +``` +rrsync [-ro|-rw] [-munge] [-no-del] [-no-lock] DIR +``` + +The single non-option argument specifies the restricted _DIR_ to use. It can be +relative to the user's home directory or an absolute path. + +The online version of this manpage (that includes cross-linking of topics) +is available at <https://download.samba.org/pub/rsync/rrsync.1>. + +## DESCRIPTION + +A user's ssh login can be restricted to only allow the running of an rsync +transfer in one of two easy ways: + +* forcing the running of the rrsync script +* forcing the running of an rsync daemon-over-ssh command. + +Both of these setups use a feature of ssh that allows a command to be forced to +run instead of an interactive shell. However, if the user's home shell is bash, +please see [BASH SECURITY ISSUE](#) for a potential issue. + +To use the rrsync script, edit the user's `~/.ssh/authorized_keys` file and add +a prefix like one of the following (followed by a space) in front of each +ssh-key line that should be restricted: + +> ``` +> command="rrsync DIR" +> command="rrsync -ro DIR" +> command="rrsync -munge -no-del DIR" +> ``` + +Then, ensure that the rrsync script has your desired option restrictions. You +may want to copy the script to a local bin dir with a unique name if you want +to have multiple configurations. One or more rrsync options can be specified +prior to the _DIR_ if you want to further restrict the transfer. + +To use an rsync daemon setup, edit the user's `~/.ssh/authorized_keys` file and +add a prefix like one of the following (followed by a space) in front of each +ssh-key line that should be restricted: + +> ``` +> command="rsync --server --daemon ." +> command="rsync --server --daemon --config=/PATH/TO/rsyncd.conf ." +> ``` + +Then, ensure that the rsyncd.conf file is created with one or more module names +with the appropriate path and option restrictions. If rsync's +[`--config`](rsync.1#dopt) option is omitted, it defaults to `~/rsyncd.conf`. +See the [**rsyncd.conf**(5)](rsyncd.conf.5) manpage for details of how to +configure an rsync daemon. + +When using rrsync, there can be just one restricted dir per authorized key. A +daemon setup, on the other hand, allows multiple module names inside the config +file, each one with its own path setting. + +The remainder of this manpage is dedicated to using the rrsync script. + +## OPTIONS + +0. `-ro` + + Allow only reading from the DIR. Implies [`-no-del`](#opt) and + [`-no-lock`](#opt). + +0. `-wo` + + Allow only writing to the DIR. + +0. `-munge` + + Enable rsync's [`--munge-links`](rsync.1#opt) on the server side. + +0. `-no-del` + + Disable rsync's `--delete*` and `--remove*` options. + +0. `-no-lock` + + Avoid the single-run (per-user) lock check. Useful with [`-munge`](#opt). + +0. `-help`, `-h` + + Output this help message and exit. + +## SECURITY RESTRICTIONS + +The rrsync script validates the path arguments it is sent to try to restrict +them to staying within the specified DIR. + +The rrsync script rejects rsync's [`--copy-links`](rsync.1#opt) option (by +default) so that a copy cannot dereference a symlink within the DIR to get to a +file outside the DIR. + +The rrsync script rejects rsync's [`--protect-args`](rsync.1#opt) (`-s`) option +because it would allow options to be sent to the server-side that the script +cannot check. If you want to support `--protect-args`, use a daemon-over-ssh +setup. + +The rrsync script accepts just a subset of rsync's options that the real rsync +uses when running the server command. A few extra convenience options are also +included to help it to interact with BackupPC and accept some convenient user +overrides. + +The script (or a copy of it) can be manually edited if you want it to customize +the option handling. + +## BASH SECURITY ISSUE + +If your users have bash set as their home shell, bash may try to be overly +helpful and ensure that the user's login bashrc files are run prior to +executing the forced command. This can be a problem if the user can somehow +update their home bashrc files, perhaps via the restricted copy, a shared home +directory, or something similar. + +One simple way to avoid the issue is to switch the user to a simpler shell, +such as dash. When choosing the new home shell, make sure that you're not +choosing bash in disguise, as it is unclear if it avoids the security issue. + +Another potential fix is to ensure that the user's home directory is not a +shared mount and that they have no means of copying files outside of their +restricted directories. This may require you to force the enabling of symlink +munging on the server side. + +A future version of openssh may have a change to the handling of forced +commands that allows it to avoid using the user's home shell. + +## EXAMPLES + +The `~/.ssh/authorized_keys` file might have lines in it like this: + +> ``` +> command="rrsync client/logs" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAzG... +> command="rrsync -ro results" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAmk... +> ``` + +## FILES + +~/.ssh/authorized_keys + +## SEE ALSO + +[**rsync**(1)](rsync.1), [**rsyncd.conf**(5)](rsyncd.conf.5) + +## VERSION + +This manpage is current for version @VERSION@ of rsync. + +## CREDITS + +rsync is distributed under the GNU General Public License. See the file +[COPYING](COPYING) for details. + +An rsync web site is available at <https://rsync.samba.org/> and its github +project is <https://github.com/WayneD/rsync>. + +## AUTHOR + +The original rrsync perl script was written by Joe Smith. Many people have +later contributed to it. The python version was created by Wayne Davison. diff --git a/support/rsync-no-vanished b/support/rsync-no-vanished new file mode 100755 index 0000000..b31a5d2 --- /dev/null +++ b/support/rsync-no-vanished @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +REAL_RSYNC=/usr/bin/rsync +IGNOREEXIT=24 +IGNOREOUT='^(file has vanished: |rsync warning: some files vanished before they could be transferred)' + +# If someone installs this as "rsync", make sure we don't affect a server run. +for arg in "${@}"; do + if [[ "$arg" == --server ]]; then + exec $REAL_RSYNC "${@}" + exit $? # Not reached + fi +done + +set -o pipefail + +# This filters stderr without merging it with stdout: +{ $REAL_RSYNC "${@}" 2>&1 1>&3 3>&- | grep -E -v "$IGNOREOUT"; ret=${PIPESTATUS[0]}; } 3>&1 1>&2 + +if [[ $ret == $IGNOREEXIT ]]; then + ret=0 +fi + +exit $ret diff --git a/support/rsync-slash-strip b/support/rsync-slash-strip new file mode 100755 index 0000000..b57e61c --- /dev/null +++ b/support/rsync-slash-strip @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# This script can be used as an rsync command-line filter that strips a single +# trailing slash from each arg. That treats "src/" the same as "src", thus +# you need to use "src/." or "src//" for just the contents of the "src" dir. +# (Note that command-line dir-excludes would need to use "excl//" too.) +# +# To use this, name it something like "rs", put it somewhere in your path, and +# then use "rs" in place of "rsync" when you are typing your copy commands. + +REAL_RSYNC=/usr/bin/rsync + +args=() +for arg in "${@}"; do + if [[ "$arg" == --server ]]; then + exec $REAL_RSYNC "${@}" + exit $? # Not reached + fi + if [[ "$arg" == / ]]; then + args=("${args[@]}" /) + else + args=("${args[@]}" "${arg%/}") + fi +done +exec $REAL_RSYNC "${args[@]}" diff --git a/support/rsyncstats b/support/rsyncstats new file mode 100755 index 0000000..99fd545 --- /dev/null +++ b/support/rsyncstats @@ -0,0 +1,312 @@ +#!/usr/bin/env perl +# +# This script parses the default logfile format produced by rsync when running +# as a daemon with transfer logging enabled. It also parses a slightly tweaked +# version of the default format where %o has been replaced with %i. +# +# This script is derived from the xferstats script that comes with wuftpd. See +# the usage message at the bottom for the options it takes. +# +# Andrew Tridgell, October 1998 + +use Getopt::Long; + +# You may wish to edit the next line to customize for your default log file. +$usage_file = "/var/log/rsyncd.log"; + +# Edit the following lines for default report settings. +# Entries defined here will be over-ridden by the command line. + +$hourly_report = 0; +$domain_report = 0; +$total_report = 0; +$depth_limit = 9999; +$only_section = ''; + +&Getopt::Long::Configure('bundling'); +&usage if !&GetOptions( + 'hourly-report|h' => \$hourly_report, + 'domain-report|d' => \$domain_report, + 'domain|D:s' => \$only_domain, + 'total-report|t' => \$total_report, + 'depth-limit|l:i' => \$depth_limit, + 'real|r' => \$real, + 'anon|a' => \$anon, + 'section|s:s' => \$only_section, + 'file|f:s' => \$usage_file, +); + +$anon = 1 if !$real && !$anon; + +open(LOG, $usage_file) || die "Error opening usage log file: $usage_file\n"; + +if ($only_domain) { + print "Transfer Totals include the '$only_domain' domain only.\n"; + print "All other domains are filtered out for this report.\n\n"; +} + +if ($only_section) { + print "Transfer Totals include the '$only_section' section only.\n"; + print "All other sections are filtered out for this report.\n\n"; +} + +line: while (<LOG>) { + +my $syslog_prefix = '\w\w\w +\d+ \d\d:\d\d:\d\d \S+ rsyncd'; +my $rsyncd_prefix = '\d\d\d\d/\d\d/\d\d \d\d:\d\d:\d\d '; + + next unless ($day,$time,$op,$host,$module,$file,$bytes) + = m{^ + ( \w\w\w\s+\d+ | \d+/\d\d/\d\d ) \s+ # day + (\d\d:\d\d:\d\d) \s+ # time + [^[]* \[\d+\]:? \s+ # pid (ignored) + (send|recv|[<>]f\S+) \s+ # op (%o or %i) + (\S+) \s+ # host + \[\d+\.\d+\.\d+\.\d+\] \s+ # IP (ignored) + (\S+) \s+ # module + \(\S*\) \s+ # user (ignored) + (.*) \s+ # file name + (\d+) # file length in bytes + $ }x; + + # TODO actually divide the data by into send/recv categories + if ($op =~ /^>/) { + $op = 'send'; + } elsif ($op =~ /^</) { + $op = 'recv'; + } + + $daytime = $day; + $hour = substr($time,0,2); + + $file = $module . "/" . $file; + + $file =~ s|//|/|mg; + + @path = split(/\//, $file); + + $pathkey = ""; + for ($i=0; $i <= $#path && $i <= $depth_limit; $i++) { + $pathkey = $pathkey . "/" . $path[$i]; + } + + if ($only_section ne '') { + next unless (substr($pathkey,0,length($only_section)) eq $only_section); + } + + $host =~ tr/A-Z/a-z/; + + @address = split(/\./, $host); + + $domain = $address[$#address]; + if ( int($address[0]) > 0 || $#address < 2 ) + { $domain = "unresolved"; } + + if ($only_domain ne '') { + next unless (substr($domain,0,length($only_domain)) eq $only_domain); + } + + +# printf("c=%d day=%s bytes=%d file=%s path=%s\n", +# $#line, $daytime, $bytes, $file, $pathkey); + + $xferfiles++; # total files sent + $xfertfiles++; # total files sent + $xferfiles{$daytime}++; # files per day + $groupfiles{$pathkey}++; # per-group accesses + $domainfiles{$domain}++; + + $xferbytes{$daytime} += $bytes; # bytes per day + $domainbytes{$domain} += $bytes; # xmit bytes to domain + $xferbytes += $bytes; # total bytes sent + $groupbytes{$pathkey} += $bytes; # per-group bytes sent + + $xfertfiles{$hour}++; # files per hour + $xfertbytes{$hour} += $bytes; # bytes per hour + $xfertbytes += $bytes; # total bytes sent +} +close LOG; + +#@syslist = keys %systemfiles; +@dates = sort datecompare keys %xferbytes; + +if ($xferfiles == 0) {die "There was no data to process.\n";} + + +print "TOTALS FOR SUMMARY PERIOD ", $dates[0], " TO ", $dates[$#dates], "\n\n"; +printf("Files Transmitted During Summary Period %12.0f\n", $xferfiles); +printf("Bytes Transmitted During Summary Period %12.0f\n", $xferbytes); +#printf("Systems Using Archives %12.0f\n\n", $#syslist+1); + +printf("Average Files Transmitted Daily %12.0f\n", + $xferfiles / ($#dates + 1)); +printf("Average Bytes Transmitted Daily %12.0f\n", + $xferbytes / ($#dates + 1)); + +format top1 = + +Daily Transmission Statistics + + Number Of Number of Percent Of Percent Of + Date Files Sent MB Sent Files Sent Bytes Sent +--------------- ---------- ----------- ---------- ---------- +. + +format line1 = +@<<<<<<<<<<<<<< @>>>>>>>>> @>>>>>>>>>> @>>>>>>> @>>>>>>> +$date, $nfiles, $nbytes/(1024*1024), $pctfiles, $pctbytes +. + +$^ = top1; +$~ = line1; + +foreach $date (sort datecompare keys %xferbytes) { + + $nfiles = $xferfiles{$date}; + $nbytes = $xferbytes{$date}; + $pctfiles = sprintf("%8.2f", 100*$xferfiles{$date} / $xferfiles); + $pctbytes = sprintf("%8.2f", 100*$xferbytes{$date} / $xferbytes); + write; +} + +if ($total_report) { +format top2 = + +Total Transfers from each Archive Section (By bytes) + + - Percent - + Archive Section NFiles MB Files Bytes +------------------------------------- ------- ----------- ----- ------- +. + +format line2 = +@<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< @>>>>>> @>>>>>>>>>> @>>>> @>>>> +$section, $files, $bytes/(1024*1024), $pctfiles, $pctbytes +. + +$| = 1; +$- = 0; +$^ = top2; +$~ = line2; + +foreach $section (sort bytecompare keys %groupfiles) { + + $files = $groupfiles{$section}; + $bytes = $groupbytes{$section}; + $pctbytes = sprintf("%8.2f", 100 * $groupbytes{$section} / $xferbytes); + $pctfiles = sprintf("%8.2f", 100 * $groupfiles{$section} / $xferfiles); + write; + +} + +if ( $xferfiles < 1 ) { $xferfiles = 1; } +if ( $xferbytes < 1 ) { $xferbytes = 1; } +} + +if ($domain_report) { +format top3 = + +Total Transfer Amount By Domain + + Number Of Number of Percent Of Percent Of +Domain Name Files Sent MB Sent Files Sent Bytes Sent +----------- ---------- ------------ ---------- ---------- +. + +format line3 = +@<<<<<<<<<< @>>>>>>>>> @>>>>>>>>>>> @>>>>>>> @>>>>>>> +$domain, $files, $bytes/(1024*1024), $pctfiles, $pctbytes +. + +$- = 0; +$^ = top3; +$~ = line3; + +foreach $domain (sort domnamcompare keys %domainfiles) { + + if ( $domainsecs{$domain} < 1 ) { $domainsecs{$domain} = 1; } + + $files = $domainfiles{$domain}; + $bytes = $domainbytes{$domain}; + $pctfiles = sprintf("%8.2f", 100 * $domainfiles{$domain} / $xferfiles); + $pctbytes = sprintf("%8.2f", 100 * $domainbytes{$domain} / $xferbytes); + write; + +} + +} + +if ($hourly_report) { + +format top8 = + +Hourly Transmission Statistics + + Number Of Number of Percent Of Percent Of + Time Files Sent MB Sent Files Sent Bytes Sent +--------------- ---------- ----------- ---------- ---------- +. + +format line8 = +@<<<<<<<<<<<<<< @>>>>>>>>> @>>>>>>>>>> @>>>>>>> @>>>>>>> +$hour, $nfiles, $nbytes/(1024*1024), $pctfiles, $pctbytes +. + + +$| = 1; +$- = 0; +$^ = top8; +$~ = line8; + +foreach $hour (sort keys %xfertbytes) { + + $nfiles = $xfertfiles{$hour}; + $nbytes = $xfertbytes{$hour}; + $pctfiles = sprintf("%8.2f", 100*$xfertfiles{$hour} / $xferfiles); + $pctbytes = sprintf("%8.2f", 100*$xfertbytes{$hour} / $xferbytes); + write; +} +} +exit(0); + +sub datecompare { + $a cmp $b; +} + +sub domnamcompare { + + $sdiff = length($a) - length($b); + ($sdiff < 0) ? -1 : ($sdiff > 0) ? 1 : $a cmp $b; + +} + +sub bytecompare { + + $bdiff = $groupbytes{$b} - $groupbytes{$a}; + ($bdiff < 0) ? -1 : ($bdiff > 0) ? 1 : $a cmp $b; + +} + +sub faccompare { + + $fdiff = $fac{$b} - $fac{$a}; + ($fdiff < 0) ? -1 : ($fdiff > 0) ? 1 : $a cmp $b; + +} + +sub usage +{ + die <<EOT; +USAGE: rsyncstats [options] + +OPTIONS: + -f FILENAME Use FILENAME for the log file. + -h Include report on hourly traffic. + -d Include report on domain traffic. + -t Report on total traffic by section. + -D DOMAIN Report only on traffic from DOMAIN. + -l DEPTH Set DEPTH of path detail for sections. + -s SECTION Set SECTION to report on. For example, "-s /pub" + will report only on paths under "/pub". +EOT +} diff --git a/support/savetransfer.c b/support/savetransfer.c new file mode 100644 index 0000000..808a6f2 --- /dev/null +++ b/support/savetransfer.c @@ -0,0 +1,175 @@ +/* This program can record the stream of data flowing to or from a program. + * This allows it to be used to check that rsync's data that is flowing + * through a remote shell is not being corrupted (for example). + * + * Usage: savetransfer [-i|-o] OUTPUT_FILE PROGRAM [ARGS...] + * -i Save the input going to PROGRAM to the OUTPUT_FILE + * -o Save the output coming from PROGRAM to the OUTPUT_FILE + * + * If you want to capture the flow of data for an rsync command, use one of + * the following commands (the resulting files should be identical): + * + * rsync -av --rsh="savetransfer -i /tmp/to.server ssh" + * --rsync-path="savetransfer -i /tmp/from.client rsync" SOURCE DEST + * + * rsync -av --rsh="savetransfer -o /tmp/from.server ssh" + * --rsync-path="savetransfer -o /tmp/to.client rsync" SOURCE DEST + * + * Note that this program aborts after 30 seconds of inactivity, so you'll need + * to change it if that is not enough dead time for your transfer. Also, some + * of the above commands will not notice that the transfer is done (if we're + * saving the input to a PROGRAM and the PROGRAM goes away: we won't notice + * that it's gone unless more data comes in) -- when this happens it will delay + * at the end of the transfer until the timeout period expires. + */ + +#include "../rsync.h" + +#define TIMEOUT_SECONDS 30 + +#ifdef HAVE_SIGACTION +static struct sigaction sigact; +#endif + +void run_program(char **command); + +char buf[4096]; +int save_data_from_program = 0; + +int +main(int argc, char *argv[]) +{ + int fd_file, len; + struct timeval tv; + fd_set fds; + + argv++; + if (--argc && argv[0][0] == '-') { + if (argv[0][1] == 'o') + save_data_from_program = 1; + else if (argv[0][1] == 'i') + save_data_from_program = 0; + else { + fprintf(stderr, "Unknown option: %s\n", argv[0]); + exit(1); + } + argv++; + argc--; + } + if (argc < 2) { + fprintf(stderr, "Usage: savetransfer [-i|-o] OUTPUT_FILE PROGRAM [ARGS...]\n"); + fprintf(stderr, "-i Save the input going to PROGRAM to the OUTPUT_FILE\n"); + fprintf(stderr, "-o Save the output coming from PROGRAM to the OUTPUT_FILE\n"); + exit(1); + } + if ((fd_file = open(*argv, O_WRONLY|O_TRUNC|O_CREAT|O_BINARY, 0644)) < 0) { + fprintf(stderr, "Unable to write to `%s': %s\n", *argv, strerror(errno)); + exit(1); + } + set_blocking(fd_file); + + SIGACTION(SIGPIPE, SIG_IGN); + + run_program(argv + 1); + +#if defined HAVE_SETMODE && O_BINARY + setmode(STDIN_FILENO, O_BINARY); + setmode(STDOUT_FILENO, O_BINARY); +#endif + set_nonblocking(STDIN_FILENO); + set_blocking(STDOUT_FILENO); + + while (1) { + FD_ZERO(&fds); + FD_SET(STDIN_FILENO, &fds); + tv.tv_sec = TIMEOUT_SECONDS; + tv.tv_usec = 0; + if (!select(STDIN_FILENO+1, &fds, NULL, NULL, &tv)) + break; + if (!FD_ISSET(STDIN_FILENO, &fds)) + break; + if ((len = read(STDIN_FILENO, buf, sizeof buf)) <= 0) + break; + if (write(STDOUT_FILENO, buf, len) != len) { + fprintf(stderr, "Failed to write data to stdout: %s\n", strerror(errno)); + exit(1); + } + if (write(fd_file, buf, len) != len) { + fprintf(stderr, "Failed to write data to fd_file: %s\n", strerror(errno)); + exit(1); + } + } + return 0; +} + +void +run_program(char **command) +{ + int pipe_fds[2], ret; + pid_t pid; + + if (pipe(pipe_fds) < 0) { + fprintf(stderr, "pipe failed: %s\n", strerror(errno)); + exit(1); + } + + if ((pid = fork()) < 0) { + fprintf(stderr, "fork failed: %s\n", strerror(errno)); + exit(1); + } + + if (pid == 0) { + if (save_data_from_program) + ret = dup2(pipe_fds[1], STDOUT_FILENO); + else + ret = dup2(pipe_fds[0], STDIN_FILENO); + if (ret < 0) { + fprintf(stderr, "Failed to dup (in child): %s\n", strerror(errno)); + exit(1); + } + close(pipe_fds[0]); + close(pipe_fds[1]); + set_blocking(STDIN_FILENO); + set_blocking(STDOUT_FILENO); + execvp(command[0], command); + fprintf(stderr, "Failed to exec %s: %s\n", command[0], strerror(errno)); + exit(1); + } + + if (save_data_from_program) + ret = dup2(pipe_fds[0], STDIN_FILENO); + else + ret = dup2(pipe_fds[1], STDOUT_FILENO); + if (ret < 0) { + fprintf(stderr, "Failed to dup (in parent): %s\n", strerror(errno)); + exit(1); + } + close(pipe_fds[0]); + close(pipe_fds[1]); +} + +void +set_nonblocking(int fd) +{ + int val; + + if ((val = fcntl(fd, F_GETFL, 0)) == -1) + return; + if (!(val & NONBLOCK_FLAG)) { + val |= NONBLOCK_FLAG; + fcntl(fd, F_SETFL, val); + } +} + +void +set_blocking(int fd) +{ + int val; + + if ((val = fcntl(fd, F_GETFL, 0)) < 0) + return; + if (val & NONBLOCK_FLAG) { + val &= ~NONBLOCK_FLAG; + fcntl(fd, F_SETFL, val); + } +} |