diff options
Diffstat (limited to '')
| -rw-r--r-- | tools/update-verify/release/common/cached_download.sh | 40 | ||||
| -rw-r--r-- | tools/update-verify/release/common/check_updates.sh | 114 | ||||
| -rw-r--r-- | tools/update-verify/release/common/download_builds.sh | 36 | ||||
| -rw-r--r-- | tools/update-verify/release/common/download_mars.sh | 105 | ||||
| -rwxr-xr-x | tools/update-verify/release/common/installdmg.ex | 45 | ||||
| -rwxr-xr-x | tools/update-verify/release/common/unpack-diskimage.sh | 95 | ||||
| -rwxr-xr-x | tools/update-verify/release/common/unpack.sh | 121 |
7 files changed, 556 insertions, 0 deletions
diff --git a/tools/update-verify/release/common/cached_download.sh b/tools/update-verify/release/common/cached_download.sh new file mode 100644 index 0000000000..7cb3c42f8d --- /dev/null +++ b/tools/update-verify/release/common/cached_download.sh @@ -0,0 +1,40 @@ +# this library works like a wrapper around wget, to allow downloads to be cached +# so that if later the same url is retrieved, the entry from the cache will be +# returned. + +pushd `dirname $0` &>/dev/null +cache_dir="$(pwd)/cache" +popd &>/dev/null + +# Deletes all files in the cache directory +# We don't support folders or .dot(hidden) files +# By not deleting the cache directory, it allows us to use Docker tmpfs mounts, +# which are the only workaround to poor mount r/w performance on MacOS +# Reference: https://forums.docker.com/t/file-access-in-mounted-volumes-extremely-slow-cpu-bound/8076/288 +clear_cache () { + rm -rf "${cache_dir}/*" +} + +# download method - you pass a filename to save the file under, and the url to call +cached_download () { + local output_file="${1}" + local url="${2}" + + if fgrep -x "${url}" "${cache_dir}/urls.list" >/dev/null; then + echo "Retrieving '${url}' from cache..." + local line_number="$(fgrep -nx "${url}" "${cache_dir}/urls.list" | sed 's/:.*//')" + cp "${cache_dir}/obj_$(printf "%05d\n" "${line_number}").cache" "${output_file}" + else + echo "Downloading '${url}' and placing in cache..." + rm -f "${output_file}" + $retry wget -O "${output_file}" --progress=dot:giga --server-response "${url}" 2>&1 + local exit_code=$? + if [ "${exit_code}" == 0 ]; then + echo "${url}" >> "${cache_dir}/urls.list" + local line_number="$(fgrep -nx "${url}" "${cache_dir}/urls.list" | sed 's/:.*//')" + cp "${output_file}" "${cache_dir}/obj_$(printf "%05d\n" "${line_number}").cache" + else + return "${exit_code}" + fi + fi +} diff --git a/tools/update-verify/release/common/check_updates.sh b/tools/update-verify/release/common/check_updates.sh new file mode 100644 index 0000000000..0623c02857 --- /dev/null +++ b/tools/update-verify/release/common/check_updates.sh @@ -0,0 +1,114 @@ +check_updates () { + # called with 9 args - platform, source package, target package, update package, old updater boolean, + # a path to the updater binary to use for the tests, a file to write diffs to, the update channel, + # and (sometimes) update-settings.ini values + update_platform=$1 + source_package=$2 + target_package=$3 + locale=$4 + use_old_updater=$5 + updater=$6 + diff_file=$7 + channel=$8 + mar_channel_IDs=$9 + + # cleanup + rm -rf source/* + rm -rf target/* + + unpack_build $update_platform source "$source_package" $locale '' $mar_channel_IDs + if [ "$?" != "0" ]; then + echo "FAILED: cannot unpack_build $update_platform source $source_package" + return 1 + fi + unpack_build $update_platform target "$target_package" $locale + if [ "$?" != "0" ]; then + echo "FAILED: cannot unpack_build $update_platform target $target_package" + return 1 + fi + + case $update_platform in + Darwin_ppc-gcc | Darwin_Universal-gcc3 | Darwin_x86_64-gcc3 | Darwin_x86-gcc3-u-ppc-i386 | Darwin_x86-gcc3-u-i386-x86_64 | Darwin_x86_64-gcc3-u-i386-x86_64 | Darwin_aarch64-gcc3) + platform_dirname="*.app" + ;; + WINNT*) + platform_dirname="bin" + ;; + Linux_x86-gcc | Linux_x86-gcc3 | Linux_x86_64-gcc3) + platform_dirname=`echo $product | tr '[A-Z]' '[a-z]'` + ;; + esac + + if [ -f update/update.status ]; then rm update/update.status; fi + if [ -f update/update.log ]; then rm update/update.log; fi + + if [ -d source/$platform_dirname ]; then + if [ `uname | cut -c-5` == "MINGW" ]; then + # windows + # change /c/path/to/pwd to c:\\path\\to\\pwd + four_backslash_pwd=$(echo $PWD | sed -e 's,^/\([a-zA-Z]\)/,\1:/,' | sed -e 's,/,\\\\,g') + two_backslash_pwd=$(echo $PWD | sed -e 's,^/\([a-zA-Z]\)/,\1:/,' | sed -e 's,/,\\,g') + cwd="$two_backslash_pwd\\source\\$platform_dirname" + update_abspath="$two_backslash_pwd\\update" + else + # not windows + # use ls here, because mac uses *.app, and we need to expand it + cwd=$(ls -d $PWD/source/$platform_dirname) + update_abspath="$PWD/update" + fi + + cd_dir=$(ls -d ${PWD}/source/${platform_dirname}) + cd "${cd_dir}" + set -x + "$updater" "$update_abspath" "$cwd" "$cwd" 0 + set +x + cd ../.. + else + echo "TEST-UNEXPECTED-FAIL: no dir in source/$platform_dirname" + return 1 + fi + + cat update/update.log + update_status=`cat update/update.status` + + if [ "$update_status" != "succeeded" ] + then + echo "TEST-UNEXPECTED-FAIL: update status was not successful: $update_status" + return 1 + fi + + # If we were testing an OS X mar on Linux, the unpack step copied the + # precomplete file from Contents/Resources to the root of the install + # to ensure the Linux updater binary could find it. However, only the + # precomplete file in Contents/Resources was updated, which means + # the copied version in the root of the install will usually have some + # differences between the source and target. To prevent this false + # positive from failing the tests, we simply remove it before diffing. + # The precomplete file in Contents/Resources is still diffed, so we + # don't lose any coverage by doing this. + cd `echo "source/$platform_dirname"` + if [[ -f "Contents/Resources/precomplete" && -f "precomplete" ]] + then + rm "precomplete" + fi + cd ../.. + cd `echo "target/$platform_dirname"` + if [[ -f "Contents/Resources/precomplete" && -f "precomplete" ]] + then + rm "precomplete" + fi + cd ../.. + + ../compare-directories.py source/${platform_dirname} target/${platform_dirname} ${channel} > "${diff_file}" + diffErr=$? + cat "${diff_file}" + if [ $diffErr == 2 ] + then + echo "TEST-UNEXPECTED-FAIL: differences found after update" + return 1 + elif [ $diffErr != 0 ] + then + echo "TEST-UNEXPECTED-FAIL: unknown error from diff: $diffErr" + return 3 + fi +} diff --git a/tools/update-verify/release/common/download_builds.sh b/tools/update-verify/release/common/download_builds.sh new file mode 100644 index 0000000000..e279c808db --- /dev/null +++ b/tools/update-verify/release/common/download_builds.sh @@ -0,0 +1,36 @@ +pushd `dirname $0` &>/dev/null +MY_DIR=$(pwd) +popd &>/dev/null +retry="$MY_DIR/../../../../mach python -m redo.cmd -s 1 -a 3" + +download_builds() { + # cleanup + mkdir -p downloads/ + rm -rf downloads/* + + source_url="$1" + target_url="$2" + + if [ -z "$source_url" ] || [ -z "$target_url" ] + then + "download_builds usage: <source_url> <target_url>" + exit 1 + fi + + for url in "$source_url" "$target_url" + do + source_file=`basename "$url"` + if [ -f "$source_file" ]; then rm "$source_file"; fi + cd downloads + if [ -f "$source_file" ]; then rm "$source_file"; fi + cached_download "${source_file}" "${url}" + status=$? + if [ $status != 0 ]; then + echo "TEST-UNEXPECTED-FAIL: Could not download source $source_file from $url" + echo "skipping.." + cd ../ + return $status + fi + cd ../ + done +} diff --git a/tools/update-verify/release/common/download_mars.sh b/tools/update-verify/release/common/download_mars.sh new file mode 100644 index 0000000000..d2dab107d2 --- /dev/null +++ b/tools/update-verify/release/common/download_mars.sh @@ -0,0 +1,105 @@ +download_mars () { + update_url="$1" + only="$2" + test_only="$3" + to_build_id="$4" + to_app_version="$5" + to_display_version="$6" + + max_tries=5 + try=1 + # retrying until we get offered an update + while [ "$try" -le "$max_tries" ]; do + echo "Using $update_url" + # retrying until AUS gives us any response at all + cached_download update.xml "${update_url}" + + echo "Got this response:" + cat update.xml + # If the first line after <updates> is </updates> then we have an + # empty snippet. Otherwise we're done + if [ "$(grep -A1 '<updates>' update.xml | tail -1)" != "</updates>" ]; then + break; + fi + echo "Empty response, sleeping" + sleep 5 + try=$(($try+1)) + done + + echo; echo; # padding + + update_line=`fgrep "<update " update.xml` + grep_rv=$? + if [ 0 -ne $grep_rv ]; then + echo "TEST-UNEXPECTED-FAIL: no <update/> found for $update_url" + return 1 + fi + command=`echo $update_line | sed -e 's/^.*<update //' -e 's:>.*$::' -e 's:\&:\&:g'` + eval "export $command" + + if [ ! -z "$to_build_id" -a "$buildID" != "$to_build_id" ]; then + echo "TEST-UNEXPECTED-FAIL: expected buildID $to_build_id does not match actual $buildID" + return 1 + fi + + if [ ! -z "$to_display_version" -a "$displayVersion" != "$to_display_version" ]; then + echo "TEST-UNEXPECTED-FAIL: expected displayVersion $to_display_version does not match actual $displayVersion" + return 1 + fi + + if [ ! -z "$to_app_version" -a "$appVersion" != "$to_app_version" ]; then + echo "TEST-UNEXPECTED-FAIL: expected appVersion $to_app_version does not match actual $appVersion" + return 1 + fi + + mkdir -p update/ + if [ -z $only ]; then + only="partial complete" + fi + for patch_type in $only + do + line=`fgrep "patch type=\"$patch_type" update.xml` + grep_rv=$? + + if [ 0 -ne $grep_rv ]; then + echo "TEST-UNEXPECTED-FAIL: no $patch_type update found for $update_url" + return 1 + fi + + command=`echo $line | sed -e 's/^.*<patch //' -e 's:/>.*$::' -e 's:\&:\&:g'` + eval "export $command" + + if [ "$test_only" == "1" ] + then + echo "Testing $URL" + curl -s -I -L $URL + return + else + cached_download "update/${patch_type}.mar" "${URL}" + fi + if [ "$?" != 0 ]; then + echo "Could not download $patch_type!" + echo "from: $URL" + fi + actual_size=`perl -e "printf \"%d\n\", (stat(\"update/$patch_type.mar\"))[7]"` + actual_hash=`openssl dgst -$hashFunction update/$patch_type.mar | sed -e 's/^.*= //'` + + if [ $actual_size != $size ]; then + echo "TEST-UNEXPECTED-FAIL: $patch_type from $update_url wrong size" + echo "TEST-UNEXPECTED-FAIL: update.xml size: $size" + echo "TEST-UNEXPECTED-FAIL: actual size: $actual_size" + return 1 + fi + + if [ $actual_hash != $hashValue ]; then + echo "TEST-UNEXPECTED-FAIL: $patch_type from $update_url wrong hash" + echo "TEST-UNEXPECTED-FAIL: update.xml hash: $hashValue" + echo "TEST-UNEXPECTED-FAIL: actual hash: $actual_hash" + return 1 + fi + + cp update/$patch_type.mar update/update.mar + echo $actual_size > update/$patch_type.size + + done +} diff --git a/tools/update-verify/release/common/installdmg.ex b/tools/update-verify/release/common/installdmg.ex new file mode 100755 index 0000000000..08bcf9a201 --- /dev/null +++ b/tools/update-verify/release/common/installdmg.ex @@ -0,0 +1,45 @@ +#!/usr/bin/expect +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is Mozilla Corporation Code. +# +# The Initial Developer of the Original Code is +# Clint Talbert. +# Portions created by the Initial Developer are Copyright (C) 2007 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Armen Zambrano Gasparnian <armenzg@mozilla.com> +# Axel Hecht <l10n@mozilla.com> +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** +#send_user $argv +spawn hdiutil attach -readonly -mountroot /tmp -private -noautoopen [lindex $argv 0] +expect { +"byte" {send "G"; exp_continue} +"END" {send "\r"; exp_continue} +"Y/N?" {send "Y\r"; exp_continue} +} diff --git a/tools/update-verify/release/common/unpack-diskimage.sh b/tools/update-verify/release/common/unpack-diskimage.sh new file mode 100755 index 0000000000..b647a69c4d --- /dev/null +++ b/tools/update-verify/release/common/unpack-diskimage.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is the installdmg.sh script from taols utilities +# +# The Initial Developer of the Original Code is +# Mozilla Corporation. +# Portions created by the Initial Developer are Copyright (C) 2009 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Chris AtLee <catlee@mozilla.com> +# Robert Kaiser <kairo@kairo.at> +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +# Unpack a disk image to a specified target folder +# +# Usage: unpack-diskimage <image_file> +# <mountpoint> +# <target_path> + +DMG_PATH=$1 +MOUNTPOINT=$2 +TARGETPATH=$3 +LOGFILE=unpack.output + +# How long to wait before giving up waiting for the mount to fininsh +TIMEOUT=90 + +# If the mount point already exists, then the previous run may not have cleaned +# up properly. We should try to umount and remove the its directory. +if [ -d $MOUNTPOINT ]; then + echo "$MOUNTPOINT already exists, trying to clean up" + hdiutil detach $MOUNTPOINT -force + rm -rdfv $MOUNTPOINT +fi + +# Install an on-exit handler that will unmount and remove the '$MOUNTPOINT' directory +trap "{ if [ -d $MOUNTPOINT ]; then hdiutil detach $MOUNTPOINT -force; rm -rdfv $MOUNTPOINT; fi; }" EXIT + +mkdir -p $MOUNTPOINT + +hdiutil attach -verbose -noautoopen -mountpoint $MOUNTPOINT "$DMG_PATH" &> $LOGFILE +# Wait for files to show up +# hdiutil uses a helper process, diskimages-helper, which isn't always done its +# work by the time hdiutil exits. So we wait until something shows up in the +# mount point directory. +i=0 +while [ "$(echo $MOUNTPOINT/*)" == "$MOUNTPOINT/*" ]; do + if [ $i -gt $TIMEOUT ]; then + echo "No files found, exiting" + exit 1 + fi + sleep 1 + i=$(expr $i + 1) +done +# Now we can copy everything out of the $MOUNTPOINT directory into the target directory +rsync -av $MOUNTPOINT/* $MOUNTPOINT/.DS_Store $MOUNTPOINT/.background $MOUNTPOINT/.VolumeIcon.icns $TARGETPATH/ > $LOGFILE +# sometimes hdiutil fails with "Resource busy" +hdiutil detach $MOUNTPOINT || { sleep 10; \ + if [ -d $MOUNTPOINT ]; then hdiutil detach $MOUNTPOINT -force; fi; } +i=0 +while [ "$(echo $MOUNTPOINT/*)" != "$MOUNTPOINT/*" ]; do + if [ $i -gt $TIMEOUT ]; then + echo "Cannot umount, exiting" + exit 1 + fi + sleep 1 + i=$(expr $i + 1) +done +rm -rdf $MOUNTPOINT diff --git a/tools/update-verify/release/common/unpack.sh b/tools/update-verify/release/common/unpack.sh new file mode 100755 index 0000000000..3249936493 --- /dev/null +++ b/tools/update-verify/release/common/unpack.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +function cleanup() { + hdiutil detach ${DEV_NAME} || + { sleep 5 && hdiutil detach ${DEV_NAME} -force; }; + return $1 && $?; +}; + +unpack_build () { + unpack_platform="$1" + dir_name="$2" + pkg_file="$3" + locale=$4 + unpack_jars=$5 + update_settings_string=$6 + + if [ ! -f "$pkg_file" ]; then + return 1 + fi + mkdir -p $dir_name + pushd $dir_name > /dev/null + case $unpack_platform in + # $unpack_platform is either + # - a balrog platform name (from testing/mozharness/scripts/release/update-verify-config-creator.py) + # - a simple platform name (from tools/update-verify/release/updates/verify.sh) + mac|Darwin_*) + os=`uname` + # How we unpack a dmg differs depending on which platform we're on. + if [[ "$os" == "Darwin" ]] + then + cd ../ + echo "installing $pkg_file" + ../common/unpack-diskimage.sh "$pkg_file" mnt $dir_name + else + 7z x ../"$pkg_file" > /dev/null + if [ `ls -1 | wc -l` -ne 1 ] + then + echo "Couldn't find .app package" + return 1 + fi + unpack_dir=$(ls -1) + unpack_dir=$(ls -d "${unpack_dir}") + mv "${unpack_dir}"/*.app . + rm -rf "${unpack_dir}" + appdir=$(ls -1) + appdir=$(ls -d *.app) + # The updater guesses the location of these files based on + # its own target architecture, not the mar. If we're not + # unpacking mac-on-mac, we need to copy them so it can find + # them. It's important to copy (and not move), because when + # we diff the installer vs updated build afterwards, the + # installer version will have them in their original place. + cp "${appdir}/Contents/Resources/update-settings.ini" "${appdir}/update-settings.ini" + cp "${appdir}/Contents/Resources/precomplete" "${appdir}/precomplete" + fi + update_settings_file="${appdir}/update-settings.ini" + ;; + win32|WINNT_*) + 7z x ../"$pkg_file" > /dev/null + if [ -d localized ] + then + mkdir bin/ + cp -rp nonlocalized/* bin/ + cp -rp localized/* bin/ + rm -rf nonlocalized + rm -rf localized + if [ $(find optional/ | wc -l) -gt 1 ] + then + cp -rp optional/* bin/ + rm -rf optional + fi + elif [ -d core ] + then + mkdir bin/ + cp -rp core/* bin/ + rm -rf core + else + for file in *.xpi + do + unzip -o $file > /dev/null + done + unzip -o ${locale}.xpi > /dev/null + fi + update_settings_file='bin/update-settings.ini' + ;; + linux|Linux_*) + if `echo $pkg_file | grep -q "tar.gz"` + then + tar xfz ../"$pkg_file" > /dev/null + elif `echo $pkg_file | grep -q "tar.bz2"` + then + tar xfj ../"$pkg_file" > /dev/null + else + echo "Unknown package type for file: $pkg_file" + exit 1 + fi + update_settings_file=`echo $product | tr '[A-Z]' '[a-z]'`'/update-settings.ini' + ;; + *) + echo "Unknown platform to unpack: $unpack_platform" + exit 1 + esac + + if [ ! -z $unpack_jars ]; then + for f in `find . -name '*.jar' -o -name '*.ja'`; do + unzip -o "$f" -d "$f.dir" > /dev/null + done + fi + + if [ ! -z $update_settings_string ]; then + echo "Modifying update-settings.ini" + cat "${update_settings_file}" | sed -e "s/^ACCEPTED_MAR_CHANNEL_IDS.*/ACCEPTED_MAR_CHANNEL_IDS=${update_settings_string}/" > "${update_settings_file}.new" + diff -u "${update_settings_file}" "${update_settings_file}.new" + echo " " + rm "${update_settings_file}" + mv "${update_settings_file}.new" "${update_settings_file}" + fi + + popd > /dev/null + +} |