diff options
Diffstat (limited to 'debian/scripts')
| -rwxr-xr-x | debian/scripts/audit-vendor-source | 40 | ||||
| -rwxr-xr-x | debian/scripts/debian-cargo-vendor | 165 | ||||
| -rwxr-xr-x | debian/scripts/guess-crate-copyright | 45 | ||||
| -rwxr-xr-x | debian/scripts/prune-checksums | 47 |
4 files changed, 297 insertions, 0 deletions
diff --git a/debian/scripts/audit-vendor-source b/debian/scripts/audit-vendor-source new file mode 100755 index 000000000..08a46d804 --- /dev/null +++ b/debian/scripts/audit-vendor-source @@ -0,0 +1,40 @@ +#!/bin/sh +# Audit Rust crate source for suspicious files in the current directory, that +# shouldn't or can't be part of a Debian source package. +# +# NOTE: this overwrites & deletes files in the current directory!!! Make a +# backup before running this script. +# +# Usage: $0 <whitelist> <filter_description> [<extra args to suspicious-source>] + +set -e + +whitelist="$1" +filter_description="$2" +shift 2 # everything else is args to suspicious-source + +# Remove tiny files 4 bytes or less +find . -size -4c -type f -delete +# Remove non-suspicious files, warning on patterns that match nothing +echo "Excluding (i.e. removing) whitelisted files..." +grep -v '^#' "$whitelist" | xargs -I% sh -c 'rm -r ./% || true' +echo "Checking for suspicious files..." +# Remove cargo metadata files +find . '(' -name '.cargo-checksum.json' -or -name '.cargo_vcs_info.json' ')' -delete +# Strip comments & blank lines before testing rust source code - +# some authors like to write really long comments +find . -name '*.rs' -execdir sed -i -e '\,^\s*//,d' -e '/^\s*$/d' '{}' \; + +# TODO: merge the -m stuff into suspicious-source(1). +suspicious-source -v "$@" +# The following shell snippet is a bit more strict than suspicious-source(1) +find . -type f -exec file '{}' \; | \ + sed -e 's/\btext\b\(.*\), with very long lines/verylongtext\1/g' | \ + grep -v '\b\(text\|empty\)\b' || true + +# Most C and JS code should be in their own package +find . -name '*.c' -o -name '*.js' + +echo "The above files (if any) seem suspicious, please audit them." +echo "If good, add them to $whitelist." +echo "If bad, add them to $filter_description." diff --git a/debian/scripts/debian-cargo-vendor b/debian/scripts/debian-cargo-vendor new file mode 100755 index 000000000..6566b316d --- /dev/null +++ b/debian/scripts/debian-cargo-vendor @@ -0,0 +1,165 @@ +#!/bin/bash +# To run this, you need to first install cargo-lock. +# +# TODO: this script has a known bug in: if the Debian patches being applied, +# changes the set of dependencies, then "cargo vendor" is not re-run in order +# to pick up this new set of dependencies. This is manifested by an error +# message like: "perhaps a crate was updated and forgotten to be re-vendored?" +# +set -e + +SCRIPTDIR="$(dirname "$(readlink -f "$0")")" + +not_needed() { + diff -ur packages-before packages-after | grep "^-- " | cut -d' ' -f2-3 +} + +ghetto_parse_cargo() { + cat "$1" \ + | tr '\n' '\t' \ + | sed -e 's/\t\[/\n[/g' \ + | perl -ne 'print if s/^\[(?:package|project)\].*\tname\s*=\s*"(.*?)".*\tversion\s*=\s*"(.*?)".*/\1 \2/g' +} + +pruned_paths() { + for i in vendor/*/Cargo.toml; do + pkgnamever= + pkgnamever=$(ghetto_parse_cargo "$i") + if [ -z "$pkgnamever" ]; then + echo >&2 "failed to parse: $i" + exit 1 + fi + echo "$pkgnamever $i" + done | grep -F -f <(not_needed) | cut '-d ' -f3 | while read x; do + echo " $(dirname $x)" + done +} + +crate_to_debcargo_conf() { + echo "$1" | sed -e 's/_/-/g' +} + +rm -rf vendor/ +if [ -e "$CARGO_PRE_VENDOR" ]; then + "$CARGO_PRE_VENDOR" +fi +cargo vendor --verbose vendor/ +mkdir -p .cargo +cat >.cargo/config <<eof +[source.crates-io] +replace-with = "vendored-sources" + +[source.vendored-sources] +directory = "$PWD/vendor" +eof +cargo lock list > packages-before +cp Cargo.lock Cargo.lock.orig + +if [ -d debcargo-conf ]; then ( cd debcargo-conf && git pull ); +else git clone "${DEBCARGO_CONF:-https://salsa.debian.org/rust-team/debcargo-conf}"; fi + +# keep applying patches, and drop to a subshell for manual fixing, until it succeeds +while ! ( cd vendor +x=true +for i in *; do + debname=$(crate_to_debcargo_conf "$i") + cd $i + # if there is a d/rules then don't mess with it, it's too custom for this + # script to deal with - just use the upstream version. example: backtrace-sys + # TODO: deal with those better, especially backtrace-sys + if [ -e ../../debcargo-conf/src/$debname/debian/rules ]; then + echo >&2 "$0: the debcargo-conf for crate $i has a custom rules file, but applying patches anyway" + echo >&2 "$0: you may want to examine this situation more closely" + fi + if [ -d ../../debcargo-conf/src/$debname/debian/patches ]; then + echo >&2 "$0: patching $i" + mkdir -p debian + if [ ! -d debian/patches ]; then + cp -a -n "../../debcargo-conf/src/$debname/debian/patches" debian/ + fi + # first unapply any patches applied in the previous iteration + QUILT_PATCHES=debian/patches quilt pop -af + QUILT_PATCHES=debian/patches quilt push -a + case $? in + 0|2) true;; + *) echo >&2 "$0: patching $i failed <<<<<<<<<<<<<<<<<<<<<<<<" + QUILT_PATCHES=debian/patches quilt pop -af + x=false;; + esac + fi + if [ -f ../../debcargo-conf/src/$debname/debian/build.rs ]; then + echo >&2 "$0: overwriting build.rs with our custom one" + if [ ! -f build.rs.orig ]; then + cp -f build.rs build.rs.orig + fi + cp -f ../../debcargo-conf/src/$i/debian/build.rs build.rs + fi + cd .. +done; $x ); do + echo >&2 "================================================================================" + echo >&2 "$0: You are now in a sub-shell!" + echo >&2 "$0: Fix the failed patches in debcargo-conf/, then exit the sub-shell by pressing ctrl-D ONCE." + echo >&2 "$0: If you need to abort this process, press ctrl-D then quickly ctrl-C." + if [ -f "${SRCDIR:-$PWD}/debian/debcargo-conf.patch" ]; then + echo >&2 "$0: Previous patch changes exist, to apply them run:" + echo >&2 " $ patch -d vendor -p2 < '${SRCDIR:-$PWD}/debian/debcargo-conf.patch'" + fi + echo >&2 "================================================================================" + bash || true + echo >&2 "$0: trying patches again..." +done +rm -rf vendor/*/.pc +find vendor/*/debian/patches -name '*~' -delete || true +cargo update +cargo lock list > packages-after +pruned_paths | while read x; do echo >&2 "$0: removing, because debcargo-conf patches makes it obsolete: $x"; rm -rf "$x"; done + +# remove excluded files +( cd vendor +for i in *; do ( + debname=$(crate_to_debcargo_conf "$i") + shopt -s globstar # needed for double-glob to work in excludes + cd $i + if [ -e ../../debcargo-conf/src/$debname/debian/rules ]; then + echo >&2 "$0: the debcargo-conf for crate $i has a custom rules file, but applying excludes anyway" + echo >&2 "$0: you may want to examine this situation more closely" + fi + if grep -q excludes ../../debcargo-conf/src/$debname/debian/debcargo.toml 2>/dev/null; then + sed -nre 's/.*excludes\s*=\s*(\[[^]]*\]).*/\1/p' \ + ../../debcargo-conf/src/$i/debian/debcargo.toml \ + | python3 -c "import ast, sys; x=ast.literal_eval(sys.stdin.read()); print('\n'.join((i[:-3] if i.endswith('/**') else i) for i in x));" \ + | while read x; do echo >&2 "$0: removing, since it's excluded by debcargo-conf: vendor/$i/$x"; rm -rf $x; done + fi +); done; ) + +# TODO: rm special logic from debcargo and put into debcargo-conf instead +echo >&2 "$0: removing winapi archives" +rm -rf vendor/winapi-*-pc-windows-gnu/lib/*.a + +echo >&2 "$0: pruning all checksums.." +for i in vendor/*; do ${SCRIPTDIR}/prune-checksums "$i"; done + +( cd vendor +for i in *; do ( + cd $i + debname=$(crate_to_debcargo_conf "$i") + if [ -d debian/patches ]; then + rm -rf "../../debcargo-conf/src/$debname/debian/patches" + cp -a debian/patches "../../debcargo-conf/src/$debname/debian/" + fi +); done; ) +( cd debcargo-conf +git add . +if ! git diff --cached --quiet; then + git commit -m "Manual changes from debian-cargo-vendor" + git diff @~ > ../../debcargo-conf.patch || true + (cd ../.. ; echo >&2 "$0: backed up patch changes to $PWD/debcargo-conf.patch") + echo >&2 "$0: you should backport/merge them back into debcargo-conf.git" +fi +) + +echo >&2 "$0: cleaning up..." +rm -rf .cargo Cargo.lock debcargo-conf packages-before packages-after + +echo >&2 "$0: restoring original Cargo.lock" +mv Cargo.lock.orig Cargo.lock diff --git a/debian/scripts/guess-crate-copyright b/debian/scripts/guess-crate-copyright new file mode 100755 index 000000000..15f35f634 --- /dev/null +++ b/debian/scripts/guess-crate-copyright @@ -0,0 +1,45 @@ +#!/usr/bin/python3 +# Copyright: 2015-2017 The Debian Project +# License: MIT or Apache-2.0 +# +# Guess the copyright of a cargo crate by looking at its git history. + +import datetime +import toml +import os +import subprocess +import sys + +this_year = datetime.datetime.now().year +crates = sys.argv[1:] +get_initial_commit = len(crates) == 1 + +for crate in crates: + with open(os.path.join(crate, "Cargo.toml")) as fp: + data = toml.load(fp) + repo = data["package"].get("repository", None) + if get_initial_commit and repo: + output = subprocess.check_output( + """git clone -q --bare "%s" tmp.crate-copyright >&2 && +cd tmp.crate-copyright && +git log --format=%%cI --reverse | head -n1 | cut -b1-4 && +git log --format=%%cI | head -n1 | cut -b1-4 && +cd .. && +rm -rf tmp.crate-copyright""" % repo, shell=True).decode("utf-8") + first_year, last_year = output.strip().split(maxsplit=2) + else: + first_year = "20XX" + last_year = this_year + + authors = data["package"].get("authors", ["UNKNOWN AUTHORS"]) + + print("""Files: {0} +Copyright: {1} +License: {2} +Comment: see {3} +""".format( + os.path.join(crate, "*"), + "\n ".join("%s-%s %s" % (first_year, last_year, a.replace(" <>", "")) for a in authors), + data["package"].get("license", "???").replace("/", " or "), + repo or "???" + )) diff --git a/debian/scripts/prune-checksums b/debian/scripts/prune-checksums new file mode 100755 index 000000000..0c895cff4 --- /dev/null +++ b/debian/scripts/prune-checksums @@ -0,0 +1,47 @@ +#!/usr/bin/python3 +# Copyright: 2015-2017 The Debian Project +# License: MIT or Apache-2.0 +# +# Helper to remove removed-files from .cargo-checksum +# TODO: rewrite to perl and add to dh-cargo, maybe? + +from collections import OrderedDict +import argparse +import json +import os +import sys + +def prune_keep(cfile): + with open(cfile) as fp: + sums = json.load(fp, object_pairs_hook=OrderedDict) + + oldfiles = sums["files"] + newfiles = OrderedDict([entry for entry in oldfiles.items() if os.path.exists(entry[0])]) + sums["files"] = newfiles + + if len(oldfiles) == len(newfiles): + return + + with open(cfile, "w") as fp: + json.dump(sums, fp, separators=(',', ':')) + +def prune(cfile): + with open(cfile, "r+") as fp: + sums = json.load(fp, object_pairs_hook=OrderedDict) + sums["files"] = {} + fp.seek(0) + json.dump(sums, fp, separators=(',', ':')) + fp.truncate() + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("-k", "--keep", action="store_true", help="keep " + "checksums of files that still exist, and assume they haven't changed.") + parser.add_argument('crates', nargs=argparse.REMAINDER, + help="crates whose checksums to prune. (default: ./)") + args = parser.parse_args(sys.argv[1:]) + crates = args.crates or ["."] + f = prune_keep if args.keep else prune + for c in crates: + cfile = os.path.join(c, ".cargo-checksum.json") if os.path.isdir(c) else c + f(cfile) |