diff options
Diffstat (limited to 'tools/update-verify')
31 files changed, 2486 insertions, 0 deletions
diff --git a/tools/update-verify/README.md b/tools/update-verify/README.md new file mode 100644 index 0000000000..14eb2a5f9a --- /dev/null +++ b/tools/update-verify/README.md @@ -0,0 +1,118 @@ +Mozilla Build Verification Scripts +================================== + +Contents +-------- + +updates -> AUS and update verification + +l10n -> l10n vs. en-US verification + +common -> useful utility scripts + +Update Verification +------------------- + +`verify.sh` + +> Does a low-level check of all advertised MAR files. Expects to have a +> file named all-locales, but does not (yet) handle platform exceptions, so +> these should be removed from the locales file. +> +> Prints errors on both STDOUT and STDIN, the intention is to run the +> script with STDOUT redirected to an output log. If there is not output +> on the console and an exit code of 0 then all tests pass; otherwise one +> or more tests failed. +> +> Does the following: +> +> 1) download update.xml from AUS for a particular release +> 2) download the partial and full mar advertised +> 3) check that the partial and full match the advertised size and sha1sum +> 4) downloads the latest release, and an older release +> 5) applies MAR to the older release, and compares the two releases. +> +> Step 5 is repeated for both the complete and partial MAR. +> +> Expects to have an updates.cfg file, describing all releases to try updating +> from. + +Valid Platforms for AUS +----------------------- +- Linux_x86-gcc3 +- Darwin_Universal-gcc3 +- Linux_x86-gcc3 +- WINNT_x86-msvc +- Darwin_ppc-gcc3 + +--- +Running it locally +================== + +Requirements: +------------- + +- [Docker](https://docs.docker.com/get-docker/) +- [optional | Mac] zstd (`brew install zst`) + +Docker Image +------------ + +1. [Ship-it](https://shipit.mozilla-releng.net/recent) holds the latest builds. +1. Clicking on "Ship task" of latest build will open the task group in +Taskcluster. +1. On the "Name contains" lookup box, search for `release-update-verify-firefox` +and open a `update-verify` task +1. Make note of the `CHANNEL` under Payload. ie: `beta-localtest` +1. Click "See more" under Task Details and open the `docker-image-update-verify` +task. + +Download the image artifact from *docker-image-update-verify* task and load it +manually +``` +zstd -d image.tar.zst +docker image load -i image.tar +``` + +**OR** + +Load docker image using mach and a task +``` +# Replace TASK-ID with the ID of a docker-image-update-verify task +./mach taskcluster-load-image --task-id=<TASK-ID> +``` + +Update Verify Config +-------------------- + +1. Open Taskcluster Task Group +1. Search for `update-verify-config` and open the task +1. Under Artifacts, download `update-verify.cfg` file + +Run Docker +---------- + +To run the container interactively: +> Replace `<MOZ DIRECTORY>` with gecko repository path on local host <br /> +> Replace `<UVC PATH>` with path to `update-verify.cfg` file on local host. +ie.: `~/Downloads/update-verify.cfg` +> Replace `<CHANNEL>` with value from `update-verify` task (Docker steps) + +``` +docker run \ + -it \ + --rm \ + -e CHANNEL=beta-localtest \ + -e MOZ_FETCHES_DIR=/builds/worker/fetches \ + -e MOZBUILD_STATE_PATH=/builds/worker/.mozbuild \ + -v <UVC PATH>:/builds/worker/fetches/update-verify.cfg + -v <MOZ DIRECTORY>:/builds/worker/checkouts/gecko \ + -w /builds/worker/checkouts/gecko \ + update-verify +``` +> Note that `MOZ_FETCHES_DIR` here is different from what is used in production. + +`total-chunks` and `this-chunk` refer to the number of lines in `update-verify.cfg` +``` +./tools/update-verify/scripts/chunked-verify.sh --total-chunks=228 --this-chunk=4 +``` diff --git a/tools/update-verify/python/util/__init__.py b/tools/update-verify/python/util/__init__.py new file mode 100644 index 0000000000..c580d191c1 --- /dev/null +++ b/tools/update-verify/python/util/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/update-verify/python/util/commands.py b/tools/update-verify/python/util/commands.py new file mode 100644 index 0000000000..e53464e6f8 --- /dev/null +++ b/tools/update-verify/python/util/commands.py @@ -0,0 +1,57 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Functions for running commands""" + +import logging +import os +import subprocess +import time + +import six + +log = logging.getLogger(__name__) + + +# timeout message, used in TRANSIENT_HG_ERRORS and in tests. +TERMINATED_PROCESS_MSG = "timeout, process terminated" + + +def log_cmd(cmd, **kwargs): + # cwd is special in that we always want it printed, even if it's not + # explicitly chosen + kwargs = kwargs.copy() + if "cwd" not in kwargs: + kwargs["cwd"] = os.getcwd() + log.info("command: START") + log.info("command: %s" % subprocess.list2cmdline(cmd)) + for key, value in six.iteritems(kwargs): + log.info("command: %s: %s", key, str(value)) + + +def merge_env(env): + new_env = os.environ.copy() + new_env.update(env) + return new_env + + +def run_cmd(cmd, **kwargs): + """Run cmd (a list of arguments). Raise subprocess.CalledProcessError if + the command exits with non-zero. If the command returns successfully, + return 0.""" + log_cmd(cmd, **kwargs) + # We update this after logging because we don't want all of the inherited + # env vars muddling up the output + if "env" in kwargs: + kwargs["env"] = merge_env(kwargs["env"]) + try: + t = time.monotonic() + log.info("command: output:") + return subprocess.check_call(cmd, **kwargs) + except subprocess.CalledProcessError: + log.info("command: ERROR", exc_info=True) + raise + finally: + elapsed = time.monotonic() - t + log.info("command: END (%.2fs elapsed)\n", elapsed) diff --git a/tools/update-verify/release/common/cached_download.sh b/tools/update-verify/release/common/cached_download.sh new file mode 100644 index 0000000000..7cb3c42f8d --- /dev/null +++ b/tools/update-verify/release/common/cached_download.sh @@ -0,0 +1,40 @@ +# this library works like a wrapper around wget, to allow downloads to be cached +# so that if later the same url is retrieved, the entry from the cache will be +# returned. + +pushd `dirname $0` &>/dev/null +cache_dir="$(pwd)/cache" +popd &>/dev/null + +# Deletes all files in the cache directory +# We don't support folders or .dot(hidden) files +# By not deleting the cache directory, it allows us to use Docker tmpfs mounts, +# which are the only workaround to poor mount r/w performance on MacOS +# Reference: https://forums.docker.com/t/file-access-in-mounted-volumes-extremely-slow-cpu-bound/8076/288 +clear_cache () { + rm -rf "${cache_dir}/*" +} + +# download method - you pass a filename to save the file under, and the url to call +cached_download () { + local output_file="${1}" + local url="${2}" + + if fgrep -x "${url}" "${cache_dir}/urls.list" >/dev/null; then + echo "Retrieving '${url}' from cache..." + local line_number="$(fgrep -nx "${url}" "${cache_dir}/urls.list" | sed 's/:.*//')" + cp "${cache_dir}/obj_$(printf "%05d\n" "${line_number}").cache" "${output_file}" + else + echo "Downloading '${url}' and placing in cache..." + rm -f "${output_file}" + $retry wget -O "${output_file}" --progress=dot:giga --server-response "${url}" 2>&1 + local exit_code=$? + if [ "${exit_code}" == 0 ]; then + echo "${url}" >> "${cache_dir}/urls.list" + local line_number="$(fgrep -nx "${url}" "${cache_dir}/urls.list" | sed 's/:.*//')" + cp "${output_file}" "${cache_dir}/obj_$(printf "%05d\n" "${line_number}").cache" + else + return "${exit_code}" + fi + fi +} diff --git a/tools/update-verify/release/common/check_updates.sh b/tools/update-verify/release/common/check_updates.sh new file mode 100644 index 0000000000..acf06d8b4e --- /dev/null +++ b/tools/update-verify/release/common/check_updates.sh @@ -0,0 +1,125 @@ +check_updates () { + # called with 10 args - platform, source package, target package, update package, old updater boolean, + # a path to the updater binary to use for the tests, a file to write diffs to, the update channel, + # update-settings.ini values, and a flag to indicate the target is dep-signed + update_platform=$1 + source_package=$2 + target_package=$3 + locale=$4 + use_old_updater=$5 + updater=$6 + diff_file=$7 + channel=$8 + mar_channel_IDs=$9 + update_to_dep=${10} + + # cleanup + rm -rf source/* + rm -rf target/* + + unpack_build $update_platform source "$source_package" $locale '' $mar_channel_IDs + if [ "$?" != "0" ]; then + echo "FAILED: cannot unpack_build $update_platform source $source_package" + return 1 + fi + unpack_build $update_platform target "$target_package" $locale + if [ "$?" != "0" ]; then + echo "FAILED: cannot unpack_build $update_platform target $target_package" + return 1 + fi + + case $update_platform in + Darwin_ppc-gcc | Darwin_Universal-gcc3 | Darwin_x86_64-gcc3 | Darwin_x86-gcc3-u-ppc-i386 | Darwin_x86-gcc3-u-i386-x86_64 | Darwin_x86_64-gcc3-u-i386-x86_64 | Darwin_aarch64-gcc3) + platform_dirname="*.app" + ;; + WINNT*) + platform_dirname="bin" + ;; + Linux_x86-gcc | Linux_x86-gcc3 | Linux_x86_64-gcc3) + platform_dirname=`echo $product | tr '[A-Z]' '[a-z]'` + ;; + esac + + if [ -f update/update.status ]; then rm update/update.status; fi + if [ -f update/update.log ]; then rm update/update.log; fi + + if [ -d source/$platform_dirname ]; then + if [ `uname | cut -c-5` == "MINGW" ]; then + # windows + # change /c/path/to/pwd to c:\\path\\to\\pwd + four_backslash_pwd=$(echo $PWD | sed -e 's,^/\([a-zA-Z]\)/,\1:/,' | sed -e 's,/,\\\\,g') + two_backslash_pwd=$(echo $PWD | sed -e 's,^/\([a-zA-Z]\)/,\1:/,' | sed -e 's,/,\\,g') + cwd="$two_backslash_pwd\\source\\$platform_dirname" + update_abspath="$two_backslash_pwd\\update" + else + # not windows + # use ls here, because mac uses *.app, and we need to expand it + cwd=$(ls -d $PWD/source/$platform_dirname) + update_abspath="$PWD/update" + fi + + cd_dir=$(ls -d ${PWD}/source/${platform_dirname}) + cd "${cd_dir}" + set -x + "$updater" "$update_abspath" "$cwd" "$cwd" 0 + set +x + cd ../.. + else + echo "TEST-UNEXPECTED-FAIL: no dir in source/$platform_dirname" + return 1 + fi + + cat update/update.log + update_status=`cat update/update.status` + + if [ "$update_status" != "succeeded" ] + then + echo "TEST-UNEXPECTED-FAIL: update status was not successful: $update_status" + return 1 + fi + + # If we were testing an OS X mar on Linux, the unpack step copied the + # precomplete file from Contents/Resources to the root of the install + # to ensure the Linux updater binary could find it. However, only the + # precomplete file in Contents/Resources was updated, which means + # the copied version in the root of the install will usually have some + # differences between the source and target. To prevent this false + # positive from failing the tests, we simply remove it before diffing. + # The precomplete file in Contents/Resources is still diffed, so we + # don't lose any coverage by doing this. + cd `echo "source/$platform_dirname"` + if [[ -f "Contents/Resources/precomplete" && -f "precomplete" ]] + then + rm "precomplete" + fi + cd ../.. + cd `echo "target/$platform_dirname"` + if [[ -f "Contents/Resources/precomplete" && -f "precomplete" ]] + then + rm "precomplete" + fi + cd ../.. + + # If we are testing an OSX mar to update from a production-signed/notarized + # build to a dep-signed one, ignore Contents/CodeResources which won't be + # present in the target, to avoid spurious failures + # Same applies to provisioning profiles, since we don't have them outside of prod + if ${update_to_dep}; then + ignore_coderesources="--ignore-missing=Contents/CodeResources --ignore-missing=Contents/embedded.provisionprofile" + else + ignore_coderesources= + fi + + ../compare-directories.py source/${platform_dirname} target/${platform_dirname} ${channel} ${ignore_coderesources} > "${diff_file}" + diffErr=$? + cat "${diff_file}" + if [ $diffErr == 2 ] + then + echo "TEST-UNEXPECTED-FAIL: differences found after update" + return 1 + elif [ $diffErr != 0 ] + then + echo "TEST-UNEXPECTED-FAIL: unknown error from diff: $diffErr" + return 3 + fi +} diff --git a/tools/update-verify/release/common/download_builds.sh b/tools/update-verify/release/common/download_builds.sh new file mode 100644 index 0000000000..e279c808db --- /dev/null +++ b/tools/update-verify/release/common/download_builds.sh @@ -0,0 +1,36 @@ +pushd `dirname $0` &>/dev/null +MY_DIR=$(pwd) +popd &>/dev/null +retry="$MY_DIR/../../../../mach python -m redo.cmd -s 1 -a 3" + +download_builds() { + # cleanup + mkdir -p downloads/ + rm -rf downloads/* + + source_url="$1" + target_url="$2" + + if [ -z "$source_url" ] || [ -z "$target_url" ] + then + "download_builds usage: <source_url> <target_url>" + exit 1 + fi + + for url in "$source_url" "$target_url" + do + source_file=`basename "$url"` + if [ -f "$source_file" ]; then rm "$source_file"; fi + cd downloads + if [ -f "$source_file" ]; then rm "$source_file"; fi + cached_download "${source_file}" "${url}" + status=$? + if [ $status != 0 ]; then + echo "TEST-UNEXPECTED-FAIL: Could not download source $source_file from $url" + echo "skipping.." + cd ../ + return $status + fi + cd ../ + done +} diff --git a/tools/update-verify/release/common/download_mars.sh b/tools/update-verify/release/common/download_mars.sh new file mode 100644 index 0000000000..d2dab107d2 --- /dev/null +++ b/tools/update-verify/release/common/download_mars.sh @@ -0,0 +1,105 @@ +download_mars () { + update_url="$1" + only="$2" + test_only="$3" + to_build_id="$4" + to_app_version="$5" + to_display_version="$6" + + max_tries=5 + try=1 + # retrying until we get offered an update + while [ "$try" -le "$max_tries" ]; do + echo "Using $update_url" + # retrying until AUS gives us any response at all + cached_download update.xml "${update_url}" + + echo "Got this response:" + cat update.xml + # If the first line after <updates> is </updates> then we have an + # empty snippet. Otherwise we're done + if [ "$(grep -A1 '<updates>' update.xml | tail -1)" != "</updates>" ]; then + break; + fi + echo "Empty response, sleeping" + sleep 5 + try=$(($try+1)) + done + + echo; echo; # padding + + update_line=`fgrep "<update " update.xml` + grep_rv=$? + if [ 0 -ne $grep_rv ]; then + echo "TEST-UNEXPECTED-FAIL: no <update/> found for $update_url" + return 1 + fi + command=`echo $update_line | sed -e 's/^.*<update //' -e 's:>.*$::' -e 's:\&:\&:g'` + eval "export $command" + + if [ ! -z "$to_build_id" -a "$buildID" != "$to_build_id" ]; then + echo "TEST-UNEXPECTED-FAIL: expected buildID $to_build_id does not match actual $buildID" + return 1 + fi + + if [ ! -z "$to_display_version" -a "$displayVersion" != "$to_display_version" ]; then + echo "TEST-UNEXPECTED-FAIL: expected displayVersion $to_display_version does not match actual $displayVersion" + return 1 + fi + + if [ ! -z "$to_app_version" -a "$appVersion" != "$to_app_version" ]; then + echo "TEST-UNEXPECTED-FAIL: expected appVersion $to_app_version does not match actual $appVersion" + return 1 + fi + + mkdir -p update/ + if [ -z $only ]; then + only="partial complete" + fi + for patch_type in $only + do + line=`fgrep "patch type=\"$patch_type" update.xml` + grep_rv=$? + + if [ 0 -ne $grep_rv ]; then + echo "TEST-UNEXPECTED-FAIL: no $patch_type update found for $update_url" + return 1 + fi + + command=`echo $line | sed -e 's/^.*<patch //' -e 's:/>.*$::' -e 's:\&:\&:g'` + eval "export $command" + + if [ "$test_only" == "1" ] + then + echo "Testing $URL" + curl -s -I -L $URL + return + else + cached_download "update/${patch_type}.mar" "${URL}" + fi + if [ "$?" != 0 ]; then + echo "Could not download $patch_type!" + echo "from: $URL" + fi + actual_size=`perl -e "printf \"%d\n\", (stat(\"update/$patch_type.mar\"))[7]"` + actual_hash=`openssl dgst -$hashFunction update/$patch_type.mar | sed -e 's/^.*= //'` + + if [ $actual_size != $size ]; then + echo "TEST-UNEXPECTED-FAIL: $patch_type from $update_url wrong size" + echo "TEST-UNEXPECTED-FAIL: update.xml size: $size" + echo "TEST-UNEXPECTED-FAIL: actual size: $actual_size" + return 1 + fi + + if [ $actual_hash != $hashValue ]; then + echo "TEST-UNEXPECTED-FAIL: $patch_type from $update_url wrong hash" + echo "TEST-UNEXPECTED-FAIL: update.xml hash: $hashValue" + echo "TEST-UNEXPECTED-FAIL: actual hash: $actual_hash" + return 1 + fi + + cp update/$patch_type.mar update/update.mar + echo $actual_size > update/$patch_type.size + + done +} diff --git a/tools/update-verify/release/common/installdmg.ex b/tools/update-verify/release/common/installdmg.ex new file mode 100755 index 0000000000..08bcf9a201 --- /dev/null +++ b/tools/update-verify/release/common/installdmg.ex @@ -0,0 +1,45 @@ +#!/usr/bin/expect +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is Mozilla Corporation Code. +# +# The Initial Developer of the Original Code is +# Clint Talbert. +# Portions created by the Initial Developer are Copyright (C) 2007 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Armen Zambrano Gasparnian <armenzg@mozilla.com> +# Axel Hecht <l10n@mozilla.com> +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** +#send_user $argv +spawn hdiutil attach -readonly -mountroot /tmp -private -noautoopen [lindex $argv 0] +expect { +"byte" {send "G"; exp_continue} +"END" {send "\r"; exp_continue} +"Y/N?" {send "Y\r"; exp_continue} +} diff --git a/tools/update-verify/release/common/unpack-diskimage.sh b/tools/update-verify/release/common/unpack-diskimage.sh new file mode 100755 index 0000000000..b647a69c4d --- /dev/null +++ b/tools/update-verify/release/common/unpack-diskimage.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is the installdmg.sh script from taols utilities +# +# The Initial Developer of the Original Code is +# Mozilla Corporation. +# Portions created by the Initial Developer are Copyright (C) 2009 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Chris AtLee <catlee@mozilla.com> +# Robert Kaiser <kairo@kairo.at> +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +# Unpack a disk image to a specified target folder +# +# Usage: unpack-diskimage <image_file> +# <mountpoint> +# <target_path> + +DMG_PATH=$1 +MOUNTPOINT=$2 +TARGETPATH=$3 +LOGFILE=unpack.output + +# How long to wait before giving up waiting for the mount to fininsh +TIMEOUT=90 + +# If the mount point already exists, then the previous run may not have cleaned +# up properly. We should try to umount and remove the its directory. +if [ -d $MOUNTPOINT ]; then + echo "$MOUNTPOINT already exists, trying to clean up" + hdiutil detach $MOUNTPOINT -force + rm -rdfv $MOUNTPOINT +fi + +# Install an on-exit handler that will unmount and remove the '$MOUNTPOINT' directory +trap "{ if [ -d $MOUNTPOINT ]; then hdiutil detach $MOUNTPOINT -force; rm -rdfv $MOUNTPOINT; fi; }" EXIT + +mkdir -p $MOUNTPOINT + +hdiutil attach -verbose -noautoopen -mountpoint $MOUNTPOINT "$DMG_PATH" &> $LOGFILE +# Wait for files to show up +# hdiutil uses a helper process, diskimages-helper, which isn't always done its +# work by the time hdiutil exits. So we wait until something shows up in the +# mount point directory. +i=0 +while [ "$(echo $MOUNTPOINT/*)" == "$MOUNTPOINT/*" ]; do + if [ $i -gt $TIMEOUT ]; then + echo "No files found, exiting" + exit 1 + fi + sleep 1 + i=$(expr $i + 1) +done +# Now we can copy everything out of the $MOUNTPOINT directory into the target directory +rsync -av $MOUNTPOINT/* $MOUNTPOINT/.DS_Store $MOUNTPOINT/.background $MOUNTPOINT/.VolumeIcon.icns $TARGETPATH/ > $LOGFILE +# sometimes hdiutil fails with "Resource busy" +hdiutil detach $MOUNTPOINT || { sleep 10; \ + if [ -d $MOUNTPOINT ]; then hdiutil detach $MOUNTPOINT -force; fi; } +i=0 +while [ "$(echo $MOUNTPOINT/*)" != "$MOUNTPOINT/*" ]; do + if [ $i -gt $TIMEOUT ]; then + echo "Cannot umount, exiting" + exit 1 + fi + sleep 1 + i=$(expr $i + 1) +done +rm -rdf $MOUNTPOINT diff --git a/tools/update-verify/release/common/unpack.sh b/tools/update-verify/release/common/unpack.sh new file mode 100755 index 0000000000..3249936493 --- /dev/null +++ b/tools/update-verify/release/common/unpack.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +function cleanup() { + hdiutil detach ${DEV_NAME} || + { sleep 5 && hdiutil detach ${DEV_NAME} -force; }; + return $1 && $?; +}; + +unpack_build () { + unpack_platform="$1" + dir_name="$2" + pkg_file="$3" + locale=$4 + unpack_jars=$5 + update_settings_string=$6 + + if [ ! -f "$pkg_file" ]; then + return 1 + fi + mkdir -p $dir_name + pushd $dir_name > /dev/null + case $unpack_platform in + # $unpack_platform is either + # - a balrog platform name (from testing/mozharness/scripts/release/update-verify-config-creator.py) + # - a simple platform name (from tools/update-verify/release/updates/verify.sh) + mac|Darwin_*) + os=`uname` + # How we unpack a dmg differs depending on which platform we're on. + if [[ "$os" == "Darwin" ]] + then + cd ../ + echo "installing $pkg_file" + ../common/unpack-diskimage.sh "$pkg_file" mnt $dir_name + else + 7z x ../"$pkg_file" > /dev/null + if [ `ls -1 | wc -l` -ne 1 ] + then + echo "Couldn't find .app package" + return 1 + fi + unpack_dir=$(ls -1) + unpack_dir=$(ls -d "${unpack_dir}") + mv "${unpack_dir}"/*.app . + rm -rf "${unpack_dir}" + appdir=$(ls -1) + appdir=$(ls -d *.app) + # The updater guesses the location of these files based on + # its own target architecture, not the mar. If we're not + # unpacking mac-on-mac, we need to copy them so it can find + # them. It's important to copy (and not move), because when + # we diff the installer vs updated build afterwards, the + # installer version will have them in their original place. + cp "${appdir}/Contents/Resources/update-settings.ini" "${appdir}/update-settings.ini" + cp "${appdir}/Contents/Resources/precomplete" "${appdir}/precomplete" + fi + update_settings_file="${appdir}/update-settings.ini" + ;; + win32|WINNT_*) + 7z x ../"$pkg_file" > /dev/null + if [ -d localized ] + then + mkdir bin/ + cp -rp nonlocalized/* bin/ + cp -rp localized/* bin/ + rm -rf nonlocalized + rm -rf localized + if [ $(find optional/ | wc -l) -gt 1 ] + then + cp -rp optional/* bin/ + rm -rf optional + fi + elif [ -d core ] + then + mkdir bin/ + cp -rp core/* bin/ + rm -rf core + else + for file in *.xpi + do + unzip -o $file > /dev/null + done + unzip -o ${locale}.xpi > /dev/null + fi + update_settings_file='bin/update-settings.ini' + ;; + linux|Linux_*) + if `echo $pkg_file | grep -q "tar.gz"` + then + tar xfz ../"$pkg_file" > /dev/null + elif `echo $pkg_file | grep -q "tar.bz2"` + then + tar xfj ../"$pkg_file" > /dev/null + else + echo "Unknown package type for file: $pkg_file" + exit 1 + fi + update_settings_file=`echo $product | tr '[A-Z]' '[a-z]'`'/update-settings.ini' + ;; + *) + echo "Unknown platform to unpack: $unpack_platform" + exit 1 + esac + + if [ ! -z $unpack_jars ]; then + for f in `find . -name '*.jar' -o -name '*.ja'`; do + unzip -o "$f" -d "$f.dir" > /dev/null + done + fi + + if [ ! -z $update_settings_string ]; then + echo "Modifying update-settings.ini" + cat "${update_settings_file}" | sed -e "s/^ACCEPTED_MAR_CHANNEL_IDS.*/ACCEPTED_MAR_CHANNEL_IDS=${update_settings_string}/" > "${update_settings_file}.new" + diff -u "${update_settings_file}" "${update_settings_file}.new" + echo " " + rm "${update_settings_file}" + mv "${update_settings_file}.new" "${update_settings_file}" + fi + + popd > /dev/null + +} diff --git a/tools/update-verify/release/compare-directories.py b/tools/update-verify/release/compare-directories.py new file mode 100755 index 0000000000..ea70d79b31 --- /dev/null +++ b/tools/update-verify/release/compare-directories.py @@ -0,0 +1,276 @@ +#! /usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import difflib +import hashlib +import logging +import os +import sys + +""" Define the transformations needed to make source + update == target + +Required: +The files list describes the files which a transform may be used on. +The 'side' is one of ('source', 'target') and defines where each transform is applied +The 'channel_prefix' list controls which channels a transform may be used for, where a value of +'beta' means all of beta, beta-localtest, beta-cdntest, etc. + +One or more: +A 'deletion' specifies a start of line to match on, removing the whole line +A 'substitution' is a list of full string to match and its replacement +""" +TRANSFORMS = [ + # channel-prefs.js + { + # preprocessor comments, eg //@line 6 "/builds/worker/workspace/... + # this can be removed once each channel has a watershed above 59.0b2 (from bug 1431342) + "files": [ + "defaults/pref/channel-prefs.js", + "Contents/Resources/defaults/pref/channel-prefs.js", + ], + "channel_prefix": ["aurora", "beta", "release", "esr"], + "side": "source", + "deletion": '//@line 6 "', + }, + { + # updates from a beta to an RC build, the latter specifies the release channel + "files": [ + "defaults/pref/channel-prefs.js", + "Contents/Resources/defaults/pref/channel-prefs.js", + ], + "channel_prefix": ["beta"], + "side": "target", + "substitution": [ + 'pref("app.update.channel", "release");\n', + 'pref("app.update.channel", "beta");\n', + ], + }, + { + # updates from an RC to a beta build + "files": [ + "defaults/pref/channel-prefs.js", + "Contents/Resources/defaults/pref/channel-prefs.js", + ], + "channel_prefix": ["beta"], + "side": "source", + "substitution": [ + 'pref("app.update.channel", "release");\n', + 'pref("app.update.channel", "beta");\n', + ], + }, + { + # Warning comments from bug 1576546 + # When updating from a pre-70.0 build to 70.0+ this removes the new comments in + # the target side. In the 70.0+ --> 70.0+ case with a RC we won't need this, and + # the channel munging above will make channel-prefs.js identical, allowing the code + # to break before applying this transform. + "files": [ + "defaults/pref/channel-prefs.js", + "Contents/Resources/defaults/pref/channel-prefs.js", + ], + "channel_prefix": ["aurora", "beta", "release", "esr"], + "side": "target", + "deletion": "//", + }, + # update-settings.ini + { + # updates from a beta to an RC build, the latter specifies the release channel + # on mac, we actually have both files. The second location is the real + # one but we copy to the first to run the linux64 updater + "files": ["update-settings.ini", "Contents/Resources/update-settings.ini"], + "channel_prefix": ["beta"], + "side": "target", + "substitution": [ + "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-release\n", + "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-beta,firefox-mozilla-release\n", + ], + }, + { + # updates from an RC to a beta build + # on mac, we only need to modify the legit file this time. unpack_build + # handles the copy for the updater in both source and target + "files": ["Contents/Resources/update-settings.ini"], + "channel_prefix": ["beta"], + "side": "source", + "substitution": [ + "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-release\n", + "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-beta,firefox-mozilla-release\n", + ], + }, +] + + +def walk_dir(path): + all_files = [] + all_dirs = [] + + for root, dirs, files in os.walk(path): + all_dirs.extend([os.path.join(root, d) for d in dirs]) + all_files.extend([os.path.join(root, f) for f in files]) + + # trim off directory prefix for easier comparison + all_dirs = [d[len(path) + 1 :] for d in all_dirs] + all_files = [f[len(path) + 1 :] for f in all_files] + + return all_dirs, all_files + + +def compare_listings( + source_list, target_list, label, source_dir, target_dir, ignore_missing=None +): + obj1 = set(source_list) + obj2 = set(target_list) + difference_found = False + ignore_missing = ignore_missing or () + + if ignore_missing: + logging.warning("ignoring paths: {}".format(ignore_missing)) + + left_diff = obj1 - obj2 + if left_diff: + if left_diff - set(ignore_missing): + _log = logging.error + difference_found = True + else: + _log = logging.warning + _log("Ignoring missing files due to ignore_missing") + + _log("{} only in {}:".format(label, source_dir)) + for d in sorted(left_diff): + _log(" {}".format(d)) + + right_diff = obj2 - obj1 + if right_diff: + logging.error("{} only in {}:".format(label, target_dir)) + for d in sorted(right_diff): + logging.error(" {}".format(d)) + difference_found = True + + return difference_found + + +def hash_file(filename): + h = hashlib.sha256() + with open(filename, "rb", buffering=0) as f: + for b in iter(lambda: f.read(128 * 1024), b""): + h.update(b) + return h.hexdigest() + + +def compare_common_files(files, channel, source_dir, target_dir): + difference_found = False + for filename in files: + source_file = os.path.join(source_dir, filename) + target_file = os.path.join(target_dir, filename) + + if os.stat(source_file).st_size != os.stat(target_file).st_size or hash_file( + source_file + ) != hash_file(target_file): + logging.info("Difference found in {}".format(filename)) + file_contents = { + "source": open(source_file).readlines(), + "target": open(target_file).readlines(), + } + + transforms = [ + t + for t in TRANSFORMS + if filename in t["files"] + and channel.startswith(tuple(t["channel_prefix"])) + ] + logging.debug( + "Got {} transform(s) to consider for {}".format( + len(transforms), filename + ) + ) + for transform in transforms: + side = transform["side"] + + if "deletion" in transform: + d = transform["deletion"] + logging.debug( + "Trying deleting lines starting {} from {}".format(d, side) + ) + file_contents[side] = [ + l for l in file_contents[side] if not l.startswith(d) + ] + + if "substitution" in transform: + r = transform["substitution"] + logging.debug("Trying replacement for {} in {}".format(r, side)) + file_contents[side] = [ + l.replace(r[0], r[1]) for l in file_contents[side] + ] + + if file_contents["source"] == file_contents["target"]: + logging.info("Transforms removed all differences") + break + + if file_contents["source"] != file_contents["target"]: + difference_found = True + logging.error( + "{} still differs after transforms, residual diff:".format(filename) + ) + for l in difflib.unified_diff( + file_contents["source"], file_contents["target"] + ): + logging.error(l.rstrip()) + + return difference_found + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + "Compare two directories recursively, with transformations for expected diffs" + ) + parser.add_argument("source", help="Directory containing updated Firefox") + parser.add_argument("target", help="Directory containing expected Firefox") + parser.add_argument("channel", help="Update channel used") + parser.add_argument( + "--verbose", "-v", action="store_true", help="Enable verbose logging" + ) + parser.add_argument( + "--ignore-missing", + action="append", + metavar="<path>", + help="Ignore absence of <path> in the target", + ) + + args = parser.parse_args() + level = logging.INFO + if args.verbose: + level = logging.DEBUG + logging.basicConfig(level=level, format="%(message)s", stream=sys.stdout) + + source = args.source + target = args.target + if not os.path.exists(source) or not os.path.exists(target): + logging.error("Source and/or target directory doesn't exist") + sys.exit(3) + + logging.info("Comparing {} with {}...".format(source, target)) + source_dirs, source_files = walk_dir(source) + target_dirs, target_files = walk_dir(target) + + dir_list_diff = compare_listings( + source_dirs, target_dirs, "Directories", source, target + ) + file_list_diff = compare_listings( + source_files, target_files, "Files", source, target, args.ignore_missing + ) + file_diff = compare_common_files( + set(source_files) & set(target_files), args.channel, source, target + ) + + if file_diff: + # Use status of 2 since python will use 1 if there is an error running the script + sys.exit(2) + elif dir_list_diff or file_list_diff: + # this has traditionally been a WARN, but we don't have files on one + # side anymore so lets FAIL + sys.exit(2) + else: + logging.info("No differences found") diff --git a/tools/update-verify/release/final-verification.sh b/tools/update-verify/release/final-verification.sh new file mode 100755 index 0000000000..879c64697f --- /dev/null +++ b/tools/update-verify/release/final-verification.sh @@ -0,0 +1,519 @@ +#!/bin/bash + +function usage { + log "In the updates subdirectory of the directory this script is in," + log "there are a bunch of config files. You should call this script," + log "passing the names of one or more of those files as parameters" + log "to this script." + log "" + log "This will validate that the update.xml files all exist for the" + log "given config file, and that they report the correct file sizes" + log "for the associated mar files, and that the associated mar files" + log "are available on the update servers." + log "" + log "This script will spawn multiple curl processes to query the" + log "snippets (update.xml file downloads) and the download urls in" + log "parallel. The number of parallel curl processes can be managed" + log "with the -p MAX_PROCS option." + log "" + log "Only the first three bytes of the mar files are downloaded" + log "using curl -r 0-2 option to save time. GET requests are issued" + log "rather than HEAD requests, since Akamai (one of our CDN" + log "partners) caches GET and HEAD requests separately - therefore" + log "they can be out-of-sync, and it is important that we validate" + log "that the GET requests return the expected results." + log "" + log "Please note this script can run on linux and OS X. It has not" + log "been tested on Windows, but may also work. It can be run" + log "locally, and does not require access to the mozilla vpn or" + log "any other special network, since the update servers are" + log "available over the internet. However, it does require an" + log "up-to-date checkout of the tools repository, as the updates/" + log "subfolder changes over time, and reflects the currently" + log "available updates. It makes no changes to the update servers" + log "so there is no harm in running it. It simply generates a" + log "report. However, please try to avoid hammering the update" + log "servers aggressively, e.g. with thousands of parallel" + log "processes. For example, feel free to run the examples below," + log "first making sure that your source code checkout is up-to-" + log "date on your own machine, to get the latest configs in the" + log "updates/ subdirectory." + log "" + log "Usage:" + log " $(basename "${0}") [-p MAX_PROCS] config1 [config2 config3 config4 ...]" + log " $(basename "${0}") -h" + log "" + log "Examples:" + log " 1. $(basename "${0}") -p 128 mozBeta-thunderbird-linux.cfg mozBeta-thunderbird-linux64.cfg" + log " 2. $(basename "${0}") mozBeta-thunderbird-linux64.cfg" +} + +function log { + echo "$(date): ${1}" +} + +# subprocesses don't log in real time, due to synchronisation +# issues which can cause log entries to overwrite each other. +# therefore this function outputs log entries written to +# temporary files on disk, and then deletes them. +function flush_logs { + ls -1rt "${TMPDIR}" | grep '^log\.' | while read LOG + do + cat "${TMPDIR}/${LOG}" + rm "${TMPDIR}/${LOG}" + done +} + +# this function takes an update.xml url as an argument +# and then logs a list of config files and their line +# numbers, that led to this update.xml url being tested +function show_cfg_file_entries { + local update_xml_url="${1}" + cat "${update_xml_urls}" | cut -f1 -d' ' | grep -Fn "${update_xml_url}" | sed 's/:.*//' | while read match_line_no + do + cfg_file="$(sed -n -e "${match_line_no}p" "${update_xml_urls}" | cut -f3 -d' ')" + cfg_line_no="$(sed -n -e "${match_line_no}p" "${update_xml_urls}" | cut -f4 -d' ')" + log " ${cfg_file} line ${cfg_line_no}: $(sed -n -e "${cfg_line_no}p" "${cfg_file}")" + done +} + +# this function takes a mar url as an argument and then +# logs information about which update.xml urls referenced +# this mar url, and which config files referenced those +# mar urls - so you have a full understanding of why this +# mar url was ever tested +function show_update_xml_entries { + local mar_url="${1}" + grep -Frl "${mar_url}" "${TMPDIR}" | grep '/update_xml_to_mar\.' | while read update_xml_to_mar + do + mar_size="$(cat "${update_xml_to_mar}" | cut -f2 -d' ')" + update_xml_url="$(cat "${update_xml_to_mar}" | cut -f3 -d' ')" + patch_type="$(cat "${update_xml_to_mar}" | cut -f4 -d' ')" + update_xml_actual_url="$(cat "${update_xml_to_mar}" | cut -f5 -d' ')" + log " ${update_xml_url}" + [ -n "${update_xml_actual_url}" ] && log " which redirected to: ${update_xml_actual_url}" + log " This contained an entry for:" + log " patch type: ${patch_type}" + log " mar size: ${mar_size}" + log " mar url: ${mar_url}" + log " The update.xml url above was retrieved because of the following cfg file entries:" + show_cfg_file_entries "${update_xml_url}" | sed 's/ / /' + done +} + +echo -n "$(date): Command called:" +for ((INDEX=0; INDEX<=$#; INDEX+=1)) +do + echo -n " '${!INDEX}'" +done +echo '' +log "From directory: '$(pwd)'" +log '' +log "Parsing arguments..." + +# Max procs lowered in bug 894368 to try to avoid spurious failures +MAX_PROCS=48 +BAD_ARG=0 +BAD_FILE=0 +while getopts p:h OPT +do + case "${OPT}" in + p) MAX_PROCS="${OPTARG}";; + h) usage + exit;; + *) BAD_ARG=1;; + esac +done +shift "$((OPTIND - 1))" + +# invalid option specified +[ "${BAD_ARG}" == 1 ] && exit 66 + +log "Checking one or more config files have been specified..." +if [ $# -lt 1 ] +then + usage + log "ERROR: You must specify one or more config files" + exit 64 +fi + +log "Checking whether MAX_PROCS is a number..." +if ! let x=MAX_PROCS 2>/dev/null +then + usage + log "ERROR: MAX_PROCS must be a number (-p option); you specified '${MAX_PROCS}' - this is not a number." + exit 65 +fi + +# config files are in updates subdirectory below this script +if ! cd "$(dirname "${0}")/updates" 2>/dev/null +then + log "ERROR: Cannot cd into '$(dirname "${0}")/updates' from '$(pwd)'" + exit 68 +fi + +log "Checking specified config files (and downloading them if necessary):" +log '' +configs=() +for file in "${@}" +do + if [[ ${file} == http* ]] + then + log " Downloading config file '${file}'" + cfg=$(mktemp) + curl -fL --retry 5 --compressed "${file}" > "$cfg" + if [ "$?" != 0 ]; then + log "Error downloading config file '${file}'" + BAD_FILE=1 + else + log " * '${file}' ok, downloaded to '${cfg}'" + configs+=($cfg) + fi + elif [ -f "${file}" ] + then + log " * '${file}' ok" + configs+=(${file}) + else + log " * '${file}' missing" + BAD_FILE=1 + fi +done +log '' + +# invalid config specified +if [ "${BAD_FILE}" == 1 ] +then + log "ERROR: Unable to download config file(s) or config files are missing from repo - see above" + exit 67 +fi + +log "All checks completed successfully." +log '' +log "Starting stopwatch..." +log '' +log "Please be aware output will now be buffered up, and only displayed after completion." +log "Therefore do not be alarmed if you see no output for several minutes." +log "See https://bugzilla.mozilla.org/show_bug.cgi?id=863602#c5 for details". +log '' + +START_TIME="$(date +%s)" + +# Create a temporary directory for all temp files, that can easily be +# deleted afterwards. See https://bugzilla.mozilla.org/show_bug.cgi?id=863602 +# to understand why we write everything in distinct temporary files rather +# than writing to standard error/standard out or files shared across +# processes. +# Need to unset TMPDIR first since it affects mktemp behaviour on next line +unset TMPDIR +export TMPDIR="$(mktemp -d -t final_verification.XXXXXXXXXX)" + +# this temporary file will list all update urls that need to be checked, in this format: +# <update url> <comma separated list of patch types> <cfg file that requests it> <line number of config file> +# e.g. +# https://aus4.mozilla.org/update/3/Firefox/18.0/20130104154748/Linux_x86_64-gcc3/zh-TW/releasetest/default/default/default/update.xml?force=1 complete moz20-firefox-linux64-major.cfg 3 +# https://aus4.mozilla.org/update/3/Firefox/18.0/20130104154748/Linux_x86_64-gcc3/zu/releasetest/default/default/default/update.xml?force=1 complete moz20-firefox-linux64.cfg 7 +# https://aus4.mozilla.org/update/3/Firefox/19.0/20130215130331/Linux_x86_64-gcc3/ach/releasetest/default/default/default/update.xml?force=1 complete,partial moz20-firefox-linux64-major.cfg 11 +# https://aus4.mozilla.org/update/3/Firefox/19.0/20130215130331/Linux_x86_64-gcc3/af/releasetest/default/default/default/update.xml?force=1 complete,partial moz20-firefox-linux64.cfg 17 +update_xml_urls="$(mktemp -t update_xml_urls.XXXXXXXXXX)" + +#################################################################################### +# And now a summary of all temp files that will get generated during this process... +# +# 1) mktemp -t failure.XXXXXXXXXX +# +# Each failure will generate a one line temp file, which is a space separated +# output of the error code, and the instance data for the failure. +# e.g. +# +# PATCH_TYPE_MISSING https://aus4.mozilla.org/update/3/Firefox/4.0b12/20110222205441/Linux_x86-gcc3/dummy-locale/releasetest/update.xml?force=1 complete https://aus4.mozilla.org/update/3/Firefox/4.0b12/20110222205441/Linux_x86-gcc3/dummy-locale/releasetest/default/default/default/update.xml?force=1 +# +# 2) mktemp -t update_xml_to_mar.XXXXXXXXXX +# +# For each mar url referenced in an update.xml file, a temp file will be created to store the +# association between update.xml url and mar url. This is later used (e.g. in function +# show_update_xml_entries) to trace back the update.xml url(s) that led to a mar url being +# tested. It is also used to keep a full list of mar urls to test. +# e.g. +# +# <mar url> <mar size> <update.xml url> <patch type> <update.xml redirection url, if HTTP 302 returned> +# +# 3) mktemp -t log.XXXXXXXXXX +# +# For each log message logged by a subprocesses, we will create a temp log file with the +# contents of the log message, since we cannot safely output the log message from the subprocess +# and guarantee that it will be correctly output. By buffering log output in individual log files +# we guarantee that log messages will not interfere with each other. We then flush them when all +# forked subprocesses have completed. +# +# 4) mktemp -t mar_headers.XXXXXXXXXX +# +# We keep a copy of the mar url http headers retrieved in one file per mar url. +# +# 5) mktemp -t update.xml.headers.XXXXXXXXXX +# +# We keep a copy of the update.xml http headers retrieved in one file per update.xml url. +# +# 6) mktemp -t update.xml.XXXXXXXXXX +# +# We keep a copy of each update.xml file retrieved in individual files. +#################################################################################### + + +# generate full list of update.xml urls, followed by patch types, +# as defined in the specified config files - and write into "${update_xml_urls}" file +aus_server="https://aus5.mozilla.org" +for cfg_file in "${configs[@]}" +do + line_no=0 + sed -e 's/localtest/cdntest/' "${cfg_file}" | while read config_line + do + let line_no++ + # to avoid contamination between iterations, reset variables + # each loop in case they are not declared + # aus_server is not "cleared" each iteration - to be consistent with previous behaviour of old + # final-verification.sh script - might be worth reviewing if we really want this behaviour + release="" product="" platform="" build_id="" locales="" channel="" from="" patch_types="complete" + eval "${config_line}" + for locale in ${locales} + do + echo "${aus_server}/update/3/$product/$release/$build_id/$platform/$locale/$channel/default/default/default/update.xml?force=1" "${patch_types// /,}" "${cfg_file}" "${line_no}" + done + done +done > "${update_xml_urls}" + +# download update.xml files and grab the mar urls from downloaded file for each patch type required +cat "${update_xml_urls}" | cut -f1-2 -d' ' | sort -u | xargs -n2 "-P${MAX_PROCS}" ../get-update-xml.sh +if [ "$?" != 0 ]; then + flush_logs + log "Error generating update requests" + exit 70 +fi + +flush_logs + +# download http header for each mar url +find "${TMPDIR}" -name 'update_xml_to_mar.*' -type f | xargs cat | cut -f1-2 -d' ' | sort -u | xargs -n2 "-P${MAX_PROCS}" ../test-mar-url.sh +if [ "$?" != 0 ]; then + flush_logs + log "Error HEADing mar urls" + exit 71 +fi + +flush_logs + +log '' +log 'Stopping stopwatch...' +STOP_TIME="$(date +%s)" + +number_of_failures="$(find "${TMPDIR}" -name 'failure.*' -type f | wc -l | sed 's/ //g')" +number_of_update_xml_urls="$(cat "${update_xml_urls}" | cut -f1 -d' ' | sort -u | wc -l | sed 's/ //g')" +number_of_mar_urls="$(find "${TMPDIR}" -name "update_xml_to_mar.*" | xargs cat | cut -f1 -d' ' | sort -u | wc -l | sed 's/ //g')" + +if [ "${number_of_failures}" -eq 0 ] +then + log + log "All tests passed successfully." + log + exit_code=0 +else + log '' + log '====================================' + [ "${number_of_failures}" -gt 1 ] && log "${number_of_failures} FAILURES" || log '1 FAILURE' + failure=0 + ls -1tr "${TMPDIR}" | grep '^failure\.' | while read failure_file + do + while read failure_code entry1 entry2 entry3 entry4 entry5 entry6 entry7 + do + log '====================================' + log '' + case "${failure_code}" in + + UPDATE_XML_UNAVAILABLE) + update_xml_url="${entry1}" + update_xml="${entry2}" + update_xml_headers="${entry3}" + update_xml_debug="${entry4}" + update_xml_curl_exit_code="${entry5}" + log "FAILURE $((++failure)): Update xml file not available" + log "" + log " Download url: ${update_xml_url}" + log " Curl returned exit code: ${update_xml_curl_exit_code}" + log "" + log " The HTTP headers were:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml_headers}" + log "" + log " The full curl debug output was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml_debug}" + log "" + log " The returned update.xml file was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml}" + log "" + log " This url was tested because of the following cfg file entries:" + show_cfg_file_entries "${update_xml_url}" + log "" + + ;; + + UPDATE_XML_REDIRECT_FAILED) + update_xml_url="${entry1}" + update_xml_actual_url="${entry2}" + update_xml="${entry3}" + update_xml_headers="${entry4}" + update_xml_debug="${entry5}" + update_xml_curl_exit_code="${entry6}" + log "FAILURE $((++failure)): Update xml file not available at *redirected* location" + log "" + log " Download url: ${update_xml_url}" + log " Redirected to: ${update_xml_actual_url}" + log " It could not be downloaded from this url." + log " Curl returned exit code: ${update_xml_curl_exit_code}" + log "" + log " The HTTP headers were:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml_headers}" + log "" + log " The full curl debug output was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml_debug}" + log "" + log " The returned update.xml file was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml}" + log "" + log " This url was tested because of the following cfg file entries:" + show_cfg_file_entries "${update_xml_url}" + log "" + ;; + + PATCH_TYPE_MISSING) + update_xml_url="${entry1}" + patch_type="${entry2}" + update_xml="${entry3}" + update_xml_headers="${entry4}" + update_xml_debug="${entry5}" + update_xml_actual_url="${entry6}" + log "FAILURE $((++failure)): Patch type '${patch_type}' not present in the downloaded update.xml file." + log "" + log " Update xml file downloaded from: ${update_xml_url}" + [ -n "${update_xml_actual_url}" ] && log " This redirected to the download url: ${update_xml_actual_url}" + log " Curl returned exit code: 0 (success)" + log "" + log " The HTTP headers were:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml_headers}" + log "" + log " The full curl debug output was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml_debug}" + log "" + log " The returned update.xml file was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml}" + log "" + log " This url and patch type combination was tested due to the following cfg file entries:" + show_cfg_file_entries "${update_xml_url}" + log "" + ;; + + NO_MAR_FILE) + mar_url="${entry1}" + mar_headers_file="${entry2}" + mar_headers_debug_file="${entry3}" + mar_file_curl_exit_code="${entry4}" + mar_actual_url="${entry5}" + log "FAILURE $((++failure)): Could not retrieve mar file" + log "" + log " Mar file url: ${mar_url}" + [ -n "${mar_actual_url}" ] && log " This redirected to: ${mar_actual_url}" + log " The mar file could not be downloaded from this location." + log " Curl returned exit code: ${mar_file_curl_exit_code}" + log "" + log " The HTTP headers were:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${mar_headers_file}" + log "" + log " The full curl debug output was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${mar_headers_debug_file}" + log "" + log " The mar download was tested because it was referenced in the following update xml file(s):" + show_update_xml_entries "${mar_url}" + log "" + ;; + + MAR_FILE_WRONG_SIZE) + mar_url="${entry1}" + mar_required_size="${entry2}" + mar_actual_size="${entry3}" + mar_headers_file="${entry4}" + mar_headers_debug_file="${entry5}" + mar_file_curl_exit_code="${entry6}" + mar_actual_url="${entry7}" + log "FAILURE $((++failure)): Mar file is wrong size" + log "" + log " Mar file url: ${mar_url}" + [ -n "${mar_actual_url}" ] && log " This redirected to: ${mar_actual_url}" + log " The http header of the mar file url says that the mar file is ${mar_actual_size} bytes." + log " One or more of the following update.xml file(s) says that the file should be ${mar_required_size} bytes." + log "" + log " These are the update xml file(s) that referenced this mar:" + show_update_xml_entries "${mar_url}" + log "" + log " Curl returned exit code: ${mar_file_curl_exit_code}" + log "" + log " The HTTP headers were:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${mar_headers_file}" + log "" + log " The full curl debug output was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${mar_headers_debug_file}" + log "" + ;; + + BAD_HTTP_RESPONSE_CODE_FOR_MAR) + mar_url="${entry1}" + mar_headers_file="${entry2}" + mar_headers_debug_file="${entry3}" + mar_file_curl_exit_code="${entry4}" + mar_actual_url="${entry5}" + http_response_code="$(sed -e "s/$(printf '\r')//" -n -e '/^HTTP\//p' "${mar_headers_file}" | tail -1)" + log "FAILURE $((++failure)): '${http_response_code}' for mar file" + log "" + log " Mar file url: ${mar_url}" + [ -n "${mar_actual_url}" ] && log " This redirected to: ${mar_actual_url}" + log "" + log " These are the update xml file(s) that referenced this mar:" + show_update_xml_entries "${mar_url}" + log "" + log " Curl returned exit code: ${mar_file_curl_exit_code}" + log "" + log " The HTTP headers were:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${mar_headers_file}" + log "" + log " The full curl debug output was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${mar_headers_debug_file}" + log "" + ;; + + *) + log "ERROR: Unknown failure code - '${failure_code}'" + log "ERROR: This is a serious bug in this script." + log "ERROR: Only known failure codes are: UPDATE_XML_UNAVAILABLE, UPDATE_XML_REDIRECT_FAILED, PATCH_TYPE_MISSING, NO_MAR_FILE, MAR_FILE_WRONG_SIZE, BAD_HTTP_RESPONSE_CODE_FOR_MAR" + log "" + log "FAILURE $((++failure)): Data from failure is: ${entry1} ${entry2} ${entry3} ${entry4} ${entry5} ${entry6}" + log "" + ;; + + esac + done < "${TMPDIR}/${failure_file}" + done + exit_code=1 +fi + + +log '' +log '====================================' +log 'KEY STATS' +log '====================================' +log '' +log "Config files scanned: ${#@}" +log "Update xml files downloaded and parsed: ${number_of_update_xml_urls}" +log "Unique mar urls found: ${number_of_mar_urls}" +log "Failures: ${number_of_failures}" +log "Parallel processes used (maximum limit): ${MAX_PROCS}" +log "Execution time: $((STOP_TIME-START_TIME)) seconds" +log '' + +rm -rf "${TMPDIR}" +exit ${exit_code} diff --git a/tools/update-verify/release/get-update-xml.sh b/tools/update-verify/release/get-update-xml.sh new file mode 100755 index 0000000000..4c1fa724a8 --- /dev/null +++ b/tools/update-verify/release/get-update-xml.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +update_xml_url="${1}" +patch_types="${2}" +update_xml="$(mktemp -t update.xml.XXXXXXXXXX)" +update_xml_headers="$(mktemp -t update.xml.headers.XXXXXXXXXX)" +update_xml_debug="$(mktemp -t update.xml.debug.XXXXXXXXXX)" +curl --retry 50 --retry-max-time 300 -s --show-error -D "${update_xml_headers}" -L -v -H "Cache-Control: max-stale=0" "${update_xml_url}" > "${update_xml}" 2>"${update_xml_debug}" +update_xml_curl_exit_code=$? +if [ "${update_xml_curl_exit_code}" == 0 ] +then + update_xml_actual_url="$(sed -e "s/$(printf '\r')//" -n -e 's/^Location: //p' "${update_xml_headers}" | tail -1)" + [ -n "${update_xml_actual_url}" ] && update_xml_url_with_redirects="${update_xml_url} => ${update_xml_actual_url}" || update_xml_url_with_redirects="${update_xml_url}" + echo "$(date): Downloaded update.xml file from ${update_xml_url_with_redirects}" > "$(mktemp -t log.XXXXXXXXXX)" + for patch_type in ${patch_types//,/ } + do + mar_url_and_size="$(sed -e 's/\&/\&/g' -n -e 's/.*<patch .*type="'"${patch_type}"'".* URL="\([^"]*\)".*size="\([^"]*\)".*/\1 \2/p' "${update_xml}" | tail -1)" + if [ -z "${mar_url_and_size}" ] + then + echo "$(date): FAILURE: No patch type '${patch_type}' found in update.xml from ${update_xml_url_with_redirects}" > "$(mktemp -t log.XXXXXXXXXX)" + echo "PATCH_TYPE_MISSING ${update_xml_url} ${patch_type} ${update_xml} ${update_xml_headers} ${update_xml_debug} ${update_xml_actual_url}" > "$(mktemp -t failure.XXXXXXXXXX)" + else + echo "$(date): Mar url and file size for patch type '${patch_type}' extracted from ${update_xml_url_with_redirects} (${mar_url_and_size})" > "$(mktemp -t log.XXXXXXXXXX)" + echo "${mar_url_and_size} ${update_xml_url} ${patch_type} ${update_xml_actual_url}" > "$(mktemp -t update_xml_to_mar.XXXXXXXXXX)" + fi + done +else + if [ -z "${update_xml_actual_url}" ] + then + echo "$(date): FAILURE: Could not retrieve update.xml from ${update_xml_url} for patch type(s) '${patch_types}'" > "$(mktemp -t log.XXXXXXXXXX)" + echo "UPDATE_XML_UNAVAILABLE ${update_xml_url} ${update_xml} ${update_xml_headers} ${update_xml_debug} ${update_xml_curl_exit_code}" > "$(mktemp -t failure.XXXXXXXXXX)" + else + echo "$(date): FAILURE: update.xml from ${update_xml_url} redirected to ${update_xml_actual_url} but could not retrieve update.xml from here" > "$(mktemp -t log.XXXXXXXXXX)" + echo "UPDATE_XML_REDIRECT_FAILED ${update_xml_url} ${update_xml_actual_url} ${update_xml} ${update_xml_headers} ${update_xml_debug} ${update_xml_curl_exit_code}" > "$(mktemp -t failure.XXXXXXXXXX)" + fi +fi diff --git a/tools/update-verify/release/mar_certs/README b/tools/update-verify/release/mar_certs/README new file mode 100644 index 0000000000..dd931ef1d3 --- /dev/null +++ b/tools/update-verify/release/mar_certs/README @@ -0,0 +1,29 @@ +These certificates are imported from mozilla-central (https://hg.mozilla.org/mozilla-central/file/tip/toolkit/mozapps/update/updater) +and used to support staging update verify jobs. These jobs end up replacing the certificates within the binaries +(through a binary search and replace), and must all be the same length for this to work correctly. If we recreate +these certificates, and the resulting public certificates are not the same length anymore, the commonName may be +changed to line them up again. https://github.com/google/der-ascii is a useful tool for doing this. For example: + +To convert the certificate to ascii: +der2ascii -i dep1.der -o dep1.ascii + +Then use your favourite editor to change the commonName field. That block will look something like: + SEQUENCE { + SET { + SEQUENCE { + # commonName + OBJECT_IDENTIFIER { 2.5.4.3 } + PrintableString { "CI MAR signing key 1" } + } + } + } + +You can pad the PrintableString with spaces to increase the length of the cert (1 space = 1 byte). + +Then, convert back to der: +ascii2der -i dep1.ascii -o newdep1.der + +The certificats in the sha1 subdirectory are from +https://hg.mozilla.org/mozilla-central/file/0fcbe72581bc/toolkit/mozapps/update/updater +which are the SHA-1 certs from before they where updated in Bug 1105689. They only include the release +certs, since the nightly certs are different length, and we only care about updates from old ESRs. diff --git a/tools/update-verify/release/mar_certs/dep1.der b/tools/update-verify/release/mar_certs/dep1.der Binary files differnew file mode 100644 index 0000000000..5320f41dfa --- /dev/null +++ b/tools/update-verify/release/mar_certs/dep1.der diff --git a/tools/update-verify/release/mar_certs/dep2.der b/tools/update-verify/release/mar_certs/dep2.der Binary files differnew file mode 100644 index 0000000000..f3eb568425 --- /dev/null +++ b/tools/update-verify/release/mar_certs/dep2.der diff --git a/tools/update-verify/release/mar_certs/nightly_aurora_level3_primary.der b/tools/update-verify/release/mar_certs/nightly_aurora_level3_primary.der Binary files differnew file mode 100644 index 0000000000..44fd95dcff --- /dev/null +++ b/tools/update-verify/release/mar_certs/nightly_aurora_level3_primary.der diff --git a/tools/update-verify/release/mar_certs/nightly_aurora_level3_secondary.der b/tools/update-verify/release/mar_certs/nightly_aurora_level3_secondary.der Binary files differnew file mode 100644 index 0000000000..90f8e6e82c --- /dev/null +++ b/tools/update-verify/release/mar_certs/nightly_aurora_level3_secondary.der diff --git a/tools/update-verify/release/mar_certs/release_primary.der b/tools/update-verify/release/mar_certs/release_primary.der Binary files differnew file mode 100644 index 0000000000..1d94f88ad7 --- /dev/null +++ b/tools/update-verify/release/mar_certs/release_primary.der diff --git a/tools/update-verify/release/mar_certs/release_secondary.der b/tools/update-verify/release/mar_certs/release_secondary.der Binary files differnew file mode 100644 index 0000000000..474706c4b7 --- /dev/null +++ b/tools/update-verify/release/mar_certs/release_secondary.der diff --git a/tools/update-verify/release/mar_certs/sha1/dep1.der b/tools/update-verify/release/mar_certs/sha1/dep1.der Binary files differnew file mode 100644 index 0000000000..ec8ce6184d --- /dev/null +++ b/tools/update-verify/release/mar_certs/sha1/dep1.der diff --git a/tools/update-verify/release/mar_certs/sha1/dep2.der b/tools/update-verify/release/mar_certs/sha1/dep2.der Binary files differnew file mode 100644 index 0000000000..4d0f244df2 --- /dev/null +++ b/tools/update-verify/release/mar_certs/sha1/dep2.der diff --git a/tools/update-verify/release/mar_certs/sha1/release_primary.der b/tools/update-verify/release/mar_certs/sha1/release_primary.der Binary files differnew file mode 100644 index 0000000000..11417c35e7 --- /dev/null +++ b/tools/update-verify/release/mar_certs/sha1/release_primary.der diff --git a/tools/update-verify/release/mar_certs/sha1/release_secondary.der b/tools/update-verify/release/mar_certs/sha1/release_secondary.der Binary files differnew file mode 100644 index 0000000000..16a7ef6d91 --- /dev/null +++ b/tools/update-verify/release/mar_certs/sha1/release_secondary.der diff --git a/tools/update-verify/release/mar_certs/xpcshellCertificate.der b/tools/update-verify/release/mar_certs/xpcshellCertificate.der Binary files differnew file mode 100644 index 0000000000..ea1fd47faa --- /dev/null +++ b/tools/update-verify/release/mar_certs/xpcshellCertificate.der diff --git a/tools/update-verify/release/replace-updater-certs.py b/tools/update-verify/release/replace-updater-certs.py new file mode 100644 index 0000000000..9e981fbfe0 --- /dev/null +++ b/tools/update-verify/release/replace-updater-certs.py @@ -0,0 +1,41 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os.path +import sys + +cert_dir = sys.argv[1] +# Read twice, because strings cannot be copied +updater_data = open(sys.argv[2], "rb").read() +new_updater = open(sys.argv[2], "rb").read() +outfile = sys.argv[3] + +cert_pairs = sys.argv[4:] + +if (len(cert_pairs) % 2) != 0: + print("Certs must be provided in pairs") + sys.exit(1) + +for find_cert, replace_cert in zip(*[iter(cert_pairs)] * 2): + find = open(os.path.join(cert_dir, find_cert), "rb").read() + replace = open(os.path.join(cert_dir, replace_cert), "rb").read() + print("Looking for {}...".format(find_cert)) + if find in new_updater: + print("Replacing {} with {}".format(find_cert, replace_cert)) + new_updater = new_updater.replace(find, replace) + else: + print("Didn't find {}...".format(find_cert)) + +if len(updater_data) != len(new_updater): + print( + "WARNING: new updater is not the same length as the old one (old: {}, new: {})".format( + len(updater_data), len(new_updater) + ) + ) + +if updater_data == new_updater: + print("WARNING: updater is unchanged") + +with open(outfile, "wb+") as f: + f.write(new_updater) diff --git a/tools/update-verify/release/test-mar-url.sh b/tools/update-verify/release/test-mar-url.sh new file mode 100755 index 0000000000..217c03d72a --- /dev/null +++ b/tools/update-verify/release/test-mar-url.sh @@ -0,0 +1,46 @@ +#!/bin/bash +mar_url="${1}" +mar_required_size="${2}" + +mar_headers_file="$(mktemp -t mar_headers.XXXXXXXXXX)" +mar_headers_debug_file="$(mktemp -t mar_headers_debug.XXXXXXXXXX)" +curl --retry 50 --retry-max-time 300 -s -i -r 0-2 -L -v "${mar_url}" > "${mar_headers_file}" 2>"${mar_headers_debug_file}" +mar_file_curl_exit_code=$? + +# Bug 894368 - HTTP 408's are not handled by the "curl --retry" mechanism; in this case retry in bash +attempts=1 +while [ "$((++attempts))" -lt 50 ] && grep 'HTTP/1\.1 408 Request Timeout' "${mar_headers_file}" &>/dev/null +do + sleep 1 + curl --retry 50 --retry-max-time 300 -s -i -r 0-2 -L -v "${mar_url}" > "${mar_headers_file}" 2>"${mar_headers_debug_file}" + mar_file_curl_exit_code=$? +done + +# check file size matches what was written in update.xml +# strip out dos line returns from header if they occur +# note: below, using $(printf '\r') for Darwin compatibility, rather than simple '\r' +# (i.e. shell interprets '\r' rather than sed interpretting '\r') +mar_actual_size="$(sed -e "s/$(printf '\r')//" -n -e 's/^Content-Range: bytes 0-2\///ip' "${mar_headers_file}" | tail -1)" +mar_actual_url="$(sed -e "s/$(printf '\r')//" -n -e 's/^Location: //p' "${mar_headers_file}" | tail -1)" +# note: below, sed -n '/^HTTP\//p' acts as grep '^HTTP/', but requires less overhead as sed already running +http_response_code="$(sed -e "s/$(printf '\r')//" -n -e '/^HTTP\//p' "${mar_headers_file}" | tail -1)" + +[ -n "${mar_actual_url}" ] && mar_url_with_redirects="${mar_url} => ${mar_actual_url}" || mar_url_with_redirects="${mar_url}" + +if [ "${mar_actual_size}" == "${mar_required_size}" ] +then + echo "$(date): Mar file ${mar_url_with_redirects} available with correct size (${mar_actual_size} bytes)" > "$(mktemp -t log.XXXXXXXXXX)" +elif [ -z "${mar_actual_size}" ] +then + echo "$(date): FAILURE: Could not retrieve http header for mar file from ${mar_url}" > "$(mktemp -t log.XXXXXXXXXX)" + echo "NO_MAR_FILE ${mar_url} ${mar_headers_file} ${mar_headers_debug_file} ${mar_file_curl_exit_code} ${mar_actual_url}" > "$(mktemp -t failure.XXXXXXXXXX)" + # If we get a response code (i.e. not an empty string), it better contain "206 Partial Content" or we should report on it. + # If response code is empty, this should be caught by a different block to this one (e.g. "could not retrieve http header"). +elif [ -n "${http_response_code}" ] && [ "${http_response_code}" == "${http_response_code/206 Partial Content/}" ] +then + echo "$(date): FAILURE: received a '${http_response_code}' response for mar file from ${mar_url} (expected HTTP 206 Partial Content)" > "$(mktemp -t log.XXXXXXXXXX)" + echo "BAD_HTTP_RESPONSE_CODE_FOR_MAR ${mar_url} ${mar_headers_file} ${mar_headers_debug_file} ${mar_file_curl_exit_code} ${mar_actual_url}" > "$(mktemp -t failure.XXXXXXXXXX)" +else + echo "$(date): FAILURE: Mar file incorrect size - should be ${mar_required_size} bytes, but is ${mar_actual_size} bytes - ${mar_url_with_redirects}" > "$(mktemp -t log.XXXXXXXXXX)" + echo "MAR_FILE_WRONG_SIZE ${mar_url} ${mar_required_size} ${mar_actual_size} ${mar_headers_file} ${mar_headers_debug_file} ${mar_file_curl_exit_code} ${mar_actual_url}" > "$(mktemp -t failure.XXXXXXXXXX)" +fi diff --git a/tools/update-verify/release/updates/verify.sh b/tools/update-verify/release/updates/verify.sh new file mode 100755 index 0000000000..3f8556b424 --- /dev/null +++ b/tools/update-verify/release/updates/verify.sh @@ -0,0 +1,292 @@ +#!/bin/bash +#set -x + +. ../common/cached_download.sh +. ../common/unpack.sh +. ../common/download_mars.sh +. ../common/download_builds.sh +. ../common/check_updates.sh + +# Cache init being handled by new async_download.py +# clear_cache +# create_cache + +ftp_server_to="http://stage.mozilla.org/pub/mozilla.org" +ftp_server_from="http://stage.mozilla.org/pub/mozilla.org" +aus_server="https://aus4.mozilla.org" +to="" +to_build_id="" +to_app_version="" +to_display_version="" +override_certs="" +diff_summary_log=${DIFF_SUMMARY_LOG:-"$PWD/diff-summary.log"} +if [ -e ${diff_summary_log} ]; then + rm ${diff_summary_log} +fi +touch ${diff_summary_log} + +pushd `dirname $0` &>/dev/null +MY_DIR=$(pwd) +popd &>/dev/null +retry="$MY_DIR/../../../../mach python -m redo.cmd -s 1 -a 3" +cert_replacer="$MY_DIR/../replace-updater-certs.py" + +dep_overrides="nightly_aurora_level3_primary.der dep1.der nightly_aurora_level3_secondary.der dep2.der release_primary.der dep1.der release_secondary.der dep2.der sha1/release_primary.der sha1/dep1.der sha1/release_secondary.der sha1/dep2.der" +nightly_overrides="dep1.der nightly_aurora_level3_primary.der dep2.der nightly_aurora_level3_secondary.der release_primary.der nightly_aurora_level3_primary.der release_secondary.der nightly_aurora_level3_secondary.der" +release_overrides="dep1.der release_primary.der dep2.der release_secondary.der nightly_aurora_level3_primary.der release_primary.der nightly_aurora_level3_secondary.der release_secondary.der" + +runmode=0 +config_file="updates.cfg" +UPDATE_ONLY=1 +TEST_ONLY=2 +MARS_ONLY=3 +COMPLETE=4 + +usage() +{ + echo "Usage: verify.sh [OPTION] [CONFIG_FILE]" + echo " -u, --update-only only download update.xml" + echo " -t, --test-only only test that MARs exist" + echo " -m, --mars-only only test MARs" + echo " -c, --complete complete upgrade test" +} + +if [ -z "$*" ] +then + usage + exit 0 +fi + +pass_arg_count=0 +while [ "$#" -gt "$pass_arg_count" ] +do + case "$1" in + -u | --update-only) + runmode=$UPDATE_ONLY + shift + ;; + -t | --test-only) + runmode=$TEST_ONLY + shift + ;; + -m | --mars-only) + runmode=$MARS_ONLY + shift + ;; + -c | --complete) + runmode=$COMPLETE + shift + ;; + *) + # Move the unrecognized arg to the end of the list + arg="$1" + shift + set -- "$@" "$arg" + pass_arg_count=`expr $pass_arg_count + 1` + esac +done + +if [ -n "$arg" ] +then + config_file=$arg + echo "Using config file $config_file" +else + echo "Using default config file $config_file" +fi + +if [ "$runmode" == "0" ] +then + usage + exit 0 +fi + +while read entry +do + # initialize all config variables + release="" + product="" + platform="" + build_id="" + locales="" + channel="" + from="" + patch_types="complete" + use_old_updater=0 + mar_channel_IDs="" + updater_package="" + eval $entry + + # the arguments for updater changed in Gecko 34/SeaMonkey 2.31 + major_version=`echo $release | cut -f1 -d.` + if [[ "$product" == "seamonkey" ]]; then + minor_version=`echo $release | cut -f2 -d.` + if [[ $major_version -le 2 && $minor_version -lt 31 ]]; then + use_old_updater=1 + fi + elif [[ $major_version -lt 34 ]]; then + use_old_updater=1 + fi + + # Note: cross platform tests seem to work for everything except Mac-on-Windows. + # We probably don't care about this use case. + if [[ "$updater_package" == "" ]]; then + updater_package="$from" + fi + + for locale in $locales + do + rm -f update/partial.size update/complete.size + for patch_type in $patch_types + do + update_path="${product}/${release}/${build_id}/${platform}/${locale}/${channel}/default/default/default" + if [ "$runmode" == "$MARS_ONLY" ] || [ "$runmode" == "$COMPLETE" ] || + [ "$runmode" == "$TEST_ONLY" ] + then + if [ "$runmode" == "$TEST_ONLY" ] + then + download_mars "${aus_server}/update/3/${update_path}/default/update.xml?force=1" ${patch_type} 1 \ + "${to_build_id}" "${to_app_version}" "${to_display_version}" + err=$? + else + download_mars "${aus_server}/update/3/${update_path}/update.xml?force=1" ${patch_type} 0 \ + "${to_build_id}" "${to_app_version}" "${to_display_version}" + err=$? + fi + if [ "$err" != "0" ]; then + echo "TEST-UNEXPECTED-FAIL: [${release} ${locale} ${patch_type}] download_mars returned non-zero exit code: ${err}" + continue + fi + else + mkdir -p updates/${update_path}/complete + mkdir -p updates/${update_path}/partial + $retry wget -q -O ${patch_type} updates/${update_path}/${patch_type}/update.xml "${aus_server}/update/3/${update_path}/update.xml?force=1" + + fi + if [ "$runmode" == "$COMPLETE" ] + then + if [ -z "$from" ] || [ -z "$to" ] + then + continue + fi + + updater_platform="" + updater_package_url=`echo "${ftp_server_from}${updater_package}" | sed "s/%locale%/${locale}/"` + updater_package_filename=`basename "$updater_package_url"` + case $updater_package_filename in + *dmg) + platform_dirname="*.app" + updater_bins="Contents/MacOS/updater.app/Contents/MacOS/updater Contents/MacOS/updater.app/Contents/MacOS/org.mozilla.updater" + updater_platform="mac" + ;; + *exe) + updater_package_url=`echo "${updater_package_url}" | sed "s/ja-JP-mac/ja/"` + platform_dirname="bin" + updater_bins="updater.exe" + updater_platform="win32" + ;; + *bz2) + updater_package_url=`echo "${updater_package_url}" | sed "s/ja-JP-mac/ja/"` + platform_dirname=`echo $product | tr '[A-Z]' '[a-z]'` + updater_bins="updater" + updater_platform="linux" + ;; + *) + echo "Couldn't detect updater platform" + exit 1 + ;; + esac + + rm -rf updater/* + cached_download "${updater_package_filename}" "${updater_package_url}" + unpack_build "$updater_platform" updater "$updater_package_filename" "$locale" + + # Even on Windows, we want Unix-style paths for the updater, because of MSYS. + cwd=$(\ls -d $PWD/updater/$platform_dirname) + # Bug 1209376. Linux updater linked against other libraries in the installation directory + export LD_LIBRARY_PATH=$cwd + updater="null" + for updater_bin in $updater_bins; do + if [ -e "$cwd/$updater_bin" ]; then + echo "Found updater at $updater_bin" + updater="$cwd/$updater_bin" + break + fi + done + + update_to_dep=false + if [ ! -z "$override_certs" ]; then + echo "Replacing certs in updater binary" + cp "${updater}" "${updater}.orig" + case ${override_certs} in + dep) + overrides=${dep_overrides} + update_to_dep=true + ;; + nightly) + overrides=${nightly_overrides} + ;; + release) + overrides=${release_overrides} + ;; + *) + echo "Unknown override cert - skipping" + ;; + esac + python3 "${cert_replacer}" "${MY_DIR}/../mar_certs" "${updater}.orig" "${updater}" ${overrides} + else + echo "override_certs is '${override_certs}', not replacing any certificates" + fi + + if [ "$updater" == "null" ]; then + echo "Couldn't find updater binary" + continue + fi + + from_path=`echo $from | sed "s/%locale%/${locale}/"` + to_path=`echo $to | sed "s/%locale%/${locale}/"` + download_builds "${ftp_server_from}${from_path}" "${ftp_server_to}${to_path}" + err=$? + if [ "$err" != "0" ]; then + echo "TEST-UNEXPECTED-FAIL: [$release $locale $patch_type] download_builds returned non-zero exit code: $err" + continue + fi + source_file=`basename "$from_path"` + target_file=`basename "$to_path"` + diff_file="results.diff" + if [ -e ${diff_file} ]; then + rm ${diff_file} + fi + check_updates "${platform}" "downloads/${source_file}" "downloads/${target_file}" ${locale} ${use_old_updater} ${updater} ${diff_file} ${channel} "${mar_channel_IDs}" ${update_to_dep} + err=$? + if [ "$err" == "0" ]; then + continue + elif [ "$err" == "1" ]; then + echo "TEST-UNEXPECTED-FAIL: [$release $locale $patch_type] check_updates returned failure for $platform downloads/$source_file vs. downloads/$target_file: $err" + elif [ "$err" == "2" ]; then + echo "WARN: [$release $locale $patch_type] check_updates returned warning for $platform downloads/$source_file vs. downloads/$target_file: $err" + else + echo "TEST-UNEXPECTED-FAIL: [$release $locale $patch_type] check_updates returned unknown error for $platform downloads/$source_file vs. downloads/$target_file: $err" + fi + + if [ -s ${diff_file} ]; then + echo "Found diffs for ${patch_type} update from ${aus_server}/update/3/${update_path}/update.xml?force=1" >> ${diff_summary_log} + cat ${diff_file} >> ${diff_summary_log} + echo "" >> ${diff_summary_log} + fi + fi + done + if [ -f update/partial.size ] && [ -f update/complete.size ]; then + partial_size=`cat update/partial.size` + complete_size=`cat update/complete.size` + if [ $partial_size -gt $complete_size ]; then + echo "TEST-UNEXPECTED-FAIL: [$release $locale $patch_type] partial updates are larger than complete updates" + elif [ $partial_size -eq $complete_size ]; then + echo "WARN: [$release $locale $patch_type] partial updates are the same size as complete updates, this should only happen for major updates" + else + echo "SUCCESS: [$release $locale $patch_type] partial updates are smaller than complete updates, all is well in the universe" + fi + fi + done +done < $config_file + +clear_cache diff --git a/tools/update-verify/scripts/async_download.py b/tools/update-verify/scripts/async_download.py new file mode 100644 index 0000000000..efedc8295f --- /dev/null +++ b/tools/update-verify/scripts/async_download.py @@ -0,0 +1,362 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import asyncio +import glob +import logging +import os +import sys +import xml.etree.ElementTree as ET +from os import path + +import aiohttp + +logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s") +log = logging.getLogger(__name__) + +UV_CACHE_PATH = os.getenv( + "UV_CACHE_PATH", os.path.join(path.dirname(__file__), "../release/updates/cache/") +) +UV_PARALLEL_DOWNLOADS = os.getenv("UV_PARALLEL_DOWNLOADS", 20) + +FTP_SERVER_TO = os.getenv("ftp_server_to", "http://stage.mozilla.org/pub/mozilla.org") +FTP_SERVER_FROM = os.getenv( + "ftp_server_from", "http://stage.mozilla.org/pub/mozilla.org" +) +AUS_SERVER = os.getenv("aus_server", "https://aus5.mozilla.org") + + +def create_cache(): + if not os.path.isdir(UV_CACHE_PATH): + os.mkdir(UV_CACHE_PATH) + + +def remove_cache(): + """ + Removes all files in the cache folder + We don't support folders or .dot(hidden) files + By not deleting the cache directory, it allows us to use Docker tmpfs mounts, + which are the only workaround to poor mount r/w performance on MacOS + Bug Reference: + https://forums.docker.com/t/file-access-in-mounted-volumes-extremely-slow-cpu-bound/8076/288 + """ + files = glob.glob(f"{UV_CACHE_PATH}/*") + for f in files: + os.remove(f) + + +def _cachepath(i, ext): + # Helper function: given an index, return a cache file path + return path.join(UV_CACHE_PATH, f"obj_{i:0>5}.{ext}") + + +async def fetch_url(url, path, connector): + """ + Fetch/download a file to a specific path + + Parameters + ---------- + url : str + URL to be fetched + + path : str + Path to save binary + + Returns + ------- + dict + Request result. If error result['error'] is True + """ + + def _result(response, error=False): + data = { + "headers": dict(response.headers), + "status": response.status, + "reason": response.reason, + "_request_info": str(response._request_info), + "url": url, + "path": path, + "error": error, + } + return data + + # Set connection timeout to 15 minutes + timeout = aiohttp.ClientTimeout(total=900) + + try: + async with aiohttp.ClientSession( + connector=connector, connector_owner=False, timeout=timeout + ) as session: + log.info(f"Retrieving {url}") + async with session.get( + url, headers={"Cache-Control": "max-stale=0"} + ) as response: + # Any response code > 299 means something went wrong + if response.status > 299: + log.info(f"Failed to download {url} with status {response.status}") + return _result(response, True) + + with open(path, "wb") as fd: + while True: + chunk = await response.content.read() + if not chunk: + break + fd.write(chunk) + result = _result(response) + log.info(f'Finished downloading {url}\n{result["headers"]}') + return result + + except ( + UnicodeDecodeError, # Data parsing + asyncio.TimeoutError, # Async timeout + aiohttp.ClientError, # aiohttp error + ) as e: + log.error("=============") + log.error(f"Error downloading {url}") + log.error(e) + log.error("=============") + return {"path": path, "url": url, "error": True} + + +async def download_multi(targets, sourceFunc): + """ + Download list of targets + + Parameters + ---------- + targets : list + List of urls to download + + sourceFunc : str + Source function name (for filename) + + Returns + ------- + tuple + List of responses (Headers) + """ + + targets = set(targets) + amount = len(targets) + + connector = aiohttp.TCPConnector( + limit=UV_PARALLEL_DOWNLOADS, # Simultaneous connections, per host + ttl_dns_cache=600, # Cache DNS for 10 mins + ) + + log.info(f"\nDownloading {amount} files ({UV_PARALLEL_DOWNLOADS} async limit)") + + # Transform targets into {url, path} objects + payloads = [ + {"url": url, "path": _cachepath(i, sourceFunc)} + for (i, url) in enumerate(targets) + ] + + downloads = [] + + fetches = [fetch_url(t["url"], t["path"], connector) for t in payloads] + + downloads.extend(await asyncio.gather(*fetches)) + connector.close() + + results = [] + # Remove file if download failed + for fetch in downloads: + # If there's an error, try to remove the file, but keep going if file not present + if fetch["error"]: + try: + os.unlink(fetch.get("path", None)) + except (TypeError, FileNotFoundError) as e: + log.info(f"Unable to cleanup error file: {e} continuing...") + continue + + results.append(fetch) + + return results + + +async def download_builds(verifyConfig): + """ + Given UpdateVerifyConfig, download and cache all necessary updater files + Include "to" and "from"/"updater_pacakge" + + Parameters + ---------- + verifyConfig : UpdateVerifyConfig + Chunked config + + Returns + ------- + list : List of file paths and urls to each updater file + """ + + updaterUrls = set() + for release in verifyConfig.releases: + ftpServerFrom = release["ftp_server_from"] + ftpServerTo = release["ftp_server_to"] + + for locale in release["locales"]: + toUri = verifyConfig.to + if toUri is not None and ftpServerTo is not None: + toUri = toUri.replace("%locale%", locale) + updaterUrls.add(f"{ftpServerTo}{toUri}") + + for reference in ("updater_package", "from"): + uri = release.get(reference, None) + if uri is None: + continue + uri = uri.replace("%locale%", locale) + # /ja-JP-mac/ locale is replaced with /ja/ for updater packages + uri = uri.replace("ja-JP-mac", "ja") + updaterUrls.add(f"{ftpServerFrom}{uri}") + + log.info(f"About to download {len(updaterUrls)} updater packages") + + updaterResults = await download_multi(list(updaterUrls), "updater.async.cache") + return updaterResults + + +def get_mar_urls_from_update(path): + """ + Given an update.xml file, return MAR URLs + + If update.xml doesn't have URLs, returns empty list + + Parameters + ---------- + path : str + Path to update.xml file + + Returns + ------- + list : List of URLs + """ + + result = [] + root = ET.parse(path).getroot() + for patch in root.findall("update/patch"): + url = patch.get("URL") + if url: + result.append(url) + return result + + +async def download_mars(updatePaths): + """ + Given list of update.xml paths, download MARs for each + + Parameters + ---------- + update_paths : list + List of paths to update.xml files + """ + + patchUrls = set() + for updatePath in updatePaths: + for url in get_mar_urls_from_update(updatePath): + patchUrls.add(url) + + log.info(f"About to download {len(patchUrls)} MAR packages") + marResults = await download_multi(list(patchUrls), "mar.async.cache") + return marResults + + +async def download_update_xml(verifyConfig): + """ + Given UpdateVerifyConfig, download and cache all necessary update.xml files + + Parameters + ---------- + verifyConfig : UpdateVerifyConfig + Chunked config + + Returns + ------- + list : List of file paths and urls to each update.xml file + """ + + xmlUrls = set() + product = verifyConfig.product + urlTemplate = ( + "{server}/update/3/{product}/{release}/{build}/{platform}/" + "{locale}/{channel}/default/default/default/update.xml?force=1" + ) + + for release in verifyConfig.releases: + for locale in release["locales"]: + xmlUrls.add( + urlTemplate.format( + server=AUS_SERVER, + product=product, + release=release["release"], + build=release["build_id"], + platform=release["platform"], + locale=locale, + channel=verifyConfig.channel, + ) + ) + + log.info(f"About to download {len(xmlUrls)} update.xml files") + xmlResults = await download_multi(list(xmlUrls), "update.xml.async.cache") + return xmlResults + + +async def _download_from_config(verifyConfig): + """ + Given an UpdateVerifyConfig object, download all necessary files to cache + + Parameters + ---------- + verifyConfig : UpdateVerifyConfig + The config - already chunked + """ + remove_cache() + create_cache() + + downloadList = [] + ################## + # Download files # + ################## + xmlFiles = await download_update_xml(verifyConfig) + downloadList.extend(xmlFiles) + downloadList += await download_mars(x["path"] for x in xmlFiles) + downloadList += await download_builds(verifyConfig) + + ##################### + # Create cache.list # + ##################### + cacheLinks = [] + + # Rename files and add to cache_links + for download in downloadList: + cacheLinks.append(download["url"]) + fileIndex = len(cacheLinks) + os.rename(download["path"], _cachepath(fileIndex, "cache")) + + cacheIndexPath = path.join(UV_CACHE_PATH, "urls.list") + with open(cacheIndexPath, "w") as cache: + cache.writelines(f"{l}\n" for l in cacheLinks) + + # Log cache + log.info("Cache index urls.list contents:") + with open(cacheIndexPath, "r") as cache: + for ln, url in enumerate(cache.readlines()): + line = url.replace("\n", "") + log.info(f"Line {ln+1}: {line}") + + return None + + +def download_from_config(verifyConfig): + """ + Given an UpdateVerifyConfig object, download all necessary files to cache + (sync function that calls the async one) + + Parameters + ---------- + verifyConfig : UpdateVerifyConfig + The config - already chunked + """ + return asyncio.run(_download_from_config(verifyConfig)) diff --git a/tools/update-verify/scripts/chunked-verify.py b/tools/update-verify/scripts/chunked-verify.py new file mode 100644 index 0000000000..8c4320d4cc --- /dev/null +++ b/tools/update-verify/scripts/chunked-verify.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import os +import sys +from os import path +from tempfile import mkstemp + +sys.path.append(path.join(path.dirname(__file__), "../python")) +logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s") +log = logging.getLogger(__name__) + +from async_download import download_from_config +from mozrelease.update_verify import UpdateVerifyConfig +from util.commands import run_cmd + +UPDATE_VERIFY_COMMAND = ["bash", "verify.sh", "-c"] +UPDATE_VERIFY_DIR = path.join(path.dirname(__file__), "../release/updates") + + +if __name__ == "__main__": + from argparse import ArgumentParser + + parser = ArgumentParser("") + + parser.set_defaults( + chunks=None, + thisChunk=None, + ) + parser.add_argument("--verify-config", required=True, dest="verifyConfig") + parser.add_argument("--verify-channel", required=True, dest="verify_channel") + parser.add_argument("--chunks", required=True, dest="chunks", type=int) + parser.add_argument("--this-chunk", required=True, dest="thisChunk", type=int) + parser.add_argument("--diff-summary", required=True, type=str) + + options = parser.parse_args() + assert options.chunks and options.thisChunk, "chunks and this-chunk are required" + assert path.isfile(options.verifyConfig), "Update verify config must exist!" + verifyConfigFile = options.verifyConfig + + fd, configFile = mkstemp() + # Needs to be opened in "bytes" mode because we perform relative seeks on it + fh = os.fdopen(fd, "wb") + try: + verifyConfig = UpdateVerifyConfig() + verifyConfig.read(path.join(UPDATE_VERIFY_DIR, verifyConfigFile)) + myVerifyConfig = verifyConfig.getChunk(options.chunks, options.thisChunk) + # override the channel if explicitly set + if options.verify_channel: + myVerifyConfig.channel = options.verify_channel + myVerifyConfig.write(fh) + fh.close() + run_cmd(["cat", configFile]) + + # Before verifying, we want to download and cache all required files + download_from_config(myVerifyConfig) + + run_cmd( + UPDATE_VERIFY_COMMAND + [configFile], + cwd=UPDATE_VERIFY_DIR, + env={"DIFF_SUMMARY_LOG": path.abspath(options.diff_summary)}, + ) + finally: + if path.exists(configFile): + os.unlink(configFile) diff --git a/tools/update-verify/scripts/chunked-verify.sh b/tools/update-verify/scripts/chunked-verify.sh new file mode 100755 index 0000000000..ad6af19080 --- /dev/null +++ b/tools/update-verify/scripts/chunked-verify.sh @@ -0,0 +1,72 @@ +#!/bin/bash +set -ex +set -o pipefail +# This ugly hack is a cross-platform (Linux/Mac/Windows+MSYS) way to get the +# absolute path to the directory containing this script +pushd `dirname $0` &>/dev/null +MY_DIR=$(pwd) +popd &>/dev/null +SCRIPTS_DIR="$MY_DIR/.." +PYTHON='./mach python' +VERIFY_CONFIG="$MOZ_FETCHES_DIR/update-verify.cfg" + +while [ "$#" -gt 0 ]; do + case $1 in + # Parse total-chunks + --total-chunks=*) chunks="${1#*=}"; shift 1;; + --total-chunks) chunks="${2}"; shift 2;; + + # Parse this-chunk + --this-chunk=*) thisChunk="${1#*=}"; shift 1;; + --this-chunk) thisChunk="${2}"; shift 2;; + + # Stop if other parameters are sent + *) echo "Unknown parameter: ${1}"; exit 1;; + esac +done + +# Validate parameters +if [ -z "${chunks}" ]; then echo "Required parameter: --total-chunks"; exit 1; fi +if [ -z "${thisChunk}" ]; then echo "Required parameter: --this-chunk"; exit 1; fi + +# release promotion +if [ -n "$CHANNEL" ]; then + EXTRA_PARAMS="--verify-channel $CHANNEL" +else + EXTRA_PARAMS="" +fi +$PYTHON $MY_DIR/chunked-verify.py --chunks $chunks --this-chunk $thisChunk \ +--verify-config $VERIFY_CONFIG --diff-summary $PWD/diff-summary.log $EXTRA_PARAMS \ +2>&1 | tee $SCRIPTS_DIR/../verify_log.txt + +print_failed_msg() +{ + echo "-------------------------" + echo "This run has failed, see the above log" + echo + return 1 +} + +print_warning_msg() +{ + echo "-------------------------" + echo "This run has warnings, see the above log" + echo + return 2 +} + +set +x + +echo "Scanning log for failures and warnings" +echo "--------------------------------------" + +# Test for a failure, note we are set -e. +# Grep returns 0 on a match and 1 on no match +# Testing for failures first is important because it's OK to to mark as failed +# when there's failures+warnings, but not OK to mark as warnings in the same +# situation. +( ! grep 'TEST-UNEXPECTED-FAIL:' $SCRIPTS_DIR/../verify_log.txt ) || print_failed_msg +( ! grep 'WARN:' $SCRIPTS_DIR/../verify_log.txt ) || print_warning_msg + +echo "-------------------------" +echo "All is well" |