diff options
Diffstat (limited to 'taskcluster/docker/funsize-update-generator/scripts/mbsdiff_hook.sh')
| -rwxr-xr-x | taskcluster/docker/funsize-update-generator/scripts/mbsdiff_hook.sh | 157 |
1 files changed, 157 insertions, 0 deletions
diff --git a/taskcluster/docker/funsize-update-generator/scripts/mbsdiff_hook.sh b/taskcluster/docker/funsize-update-generator/scripts/mbsdiff_hook.sh new file mode 100755 index 0000000000..965d938247 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/scripts/mbsdiff_hook.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# This tool contains functions that are to be used to handle/enable funsize +# Author: Mihai Tabara +# + +HOOK= +AWS_BUCKET_NAME= +LOCAL_CACHE_DIR= + +# Don't cache files smaller than this, as it's slower with S3 +# Bug 1437473 +CACHE_THRESHOLD=500000 + +S3_CACHE_HITS=0 +S3_CACHE_MISSES=0 + +getsha512(){ + openssl sha512 "${1}" | awk '{print $2}' +} + +print_usage(){ + echo "$(basename "$0") [-S S3-BUCKET-NAME] [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] PATH-FROM-URL PATH-TO-URL PATH-PATCH" + echo "Script that saves/retrieves from cache presumptive patches as args" + echo "" + echo "-A SERVER-URL - host where to send the files" + echo "-c LOCAL-CACHE-DIR-PATH local path to which patches are cached" + echo "-g pre hook - tests whether patch already in cache" + echo "-u post hook - upload patch to cache for future use" + echo "" + echo "PATH-FROM-URL : path on disk for source file" + echo "PATH-TO-URL : path on disk for destination file" + echo "PATH-PATCH : path on disk for patch between source and destination" +} + +upload_patch(){ + if [ "$(stat -c "%s" "$2")" -lt ${CACHE_THRESHOLD} ] + then + return 0 + fi + sha_from=$(getsha512 "$1") + sha_to=$(getsha512 "$2") + patch_path="$3" + patch_filename="$(basename "$3")" + + # save to local cache first + if [ -n "$LOCAL_CACHE_DIR" ]; then + local_cmd="mkdir -p "$LOCAL_CACHE_DIR/$sha_from"" + if $local_cmd >&2; then + cp -avf "${patch_path}" "$LOCAL_CACHE_DIR/$sha_from/$sha_to" + echo "${patch_path} saved on local cache." + fi + fi + + if [ -n "${AWS_BUCKET_NAME}" ]; then + BUCKET_PATH="s3://${AWS_BUCKET_NAME}${sha_from}/${sha_to}/${patch_filename}" + if aws s3 cp "${patch_path}" "${BUCKET_PATH}"; then + echo "${patch_path} saved on s://${AWS_BUCKET_NAME}" + return 0 + fi + echo "${patch_path} failed to be uploaded to s3://${AWS_BUCKET_NAME}" + return 1 + fi + return 0 +} + +get_patch(){ + # $1 and $2 are the /path/to/filename + if [ "$(stat -c "%s" "$2")" -lt ${CACHE_THRESHOLD} ] + then + return 1 + fi + sha_from=$(getsha512 "$1") + sha_to=$(getsha512 "$2") + destination_file="$3" + s3_filename="$(basename "$3")" + + # Try to retrieve from local cache first. + if [ -n "$LOCAL_CACHE_DIR" ]; then + if [ -r "$LOCAL_CACHE_DIR/$sha_from/$sha_to" ]; then + cp -avf "$LOCAL_CACHE_DIR/$sha_from/$sha_to" "$destination_file" + echo "Successful retrieved ${destination_file} from local cache." + return 0 + fi + fi + # If not in the local cache, we might find it remotely. + + if [ -n "${AWS_BUCKET_NAME}" ]; then + BUCKET_PATH="s3://${AWS_BUCKET_NAME}${sha_from}/${sha_to}/${s3_filename}" + if aws s3 ls "${BUCKET_PATH}"; then + ((S3_CACHE_HITS++)) + echo "s3 cache hit for ${s3_filename} (${S3_CACHE_HITS} total hits)" + if aws s3 cp "${BUCKET_PATH}" "${destination_file}"; then + echo "Successful retrieved ${destination_file} from s3://${AWS_BUCKET_NAME}" + return 0 + else + echo "Failed to retrieve ${destination_file} from s3://${AWS_BUCKET_NAME}" + return 1 + fi + # Not found, fall through to default error + else + ((S3_CACHE_MISSES++)) + echo "s3 cache miss for ${s3_filename} (${S3_CACHE_MISSES} total misses)" + fi + fi + return 1 +} + +OPTIND=1 + +while getopts ":S:c:gu" option; do + case $option in + S) + # This will probably be bucketname/path/prefix but we can use it either way + AWS_BUCKET_NAME="$OPTARG" + # Ensure trailing slash is there. + if [[ ! $AWS_BUCKET_NAME =~ .*/$ ]]; then + AWS_BUCKET_NAME="${AWS_BUCKET_NAME}/" + fi + ;; + c) + LOCAL_CACHE_DIR="$OPTARG" + ;; + g) + HOOK="PRE" + ;; + u) + HOOK="POST" + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + print_usage + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + print_usage + exit 1 + ;; + *) + echo "Unimplemented option: -$OPTARG" >&2 + print_usage + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ "$HOOK" == "PRE" ]; then + get_patch "$1" "$2" "$3" +elif [ "$HOOK" == "POST" ]; then + upload_patch "$1" "$2" "$3" +fi |