7 files changed, 566 insertions, 0 deletions
diff --git a/tools/update-verify/release/common/cached_download.sh b/tools/update-verify/release/common/cached_download.sh
new file mode 100644
index 0000000000..7cb3c42f8d
--- /dev/null
+++ b/tools/update-verify/release/common/cached_download.sh
@@ -0,0 +1,40 @@
+# this library works like a wrapper around wget, to allow downloads to be cached
+# so that if later the same url is retrieved, the entry from the cache will be
+# returned.
+
+pushd `dirname $0` &>/dev/null
+cache_dir="$(pwd)/cache"
+popd &>/dev/null
+
+# Deletes all files in the cache directory
+# We don't support folders or .dot(hidden) files
+# By not deleting the cache directory, it allows us to use Docker tmpfs mounts,
+# which are the only workaround to poor mount r/w performance on MacOS
+# Reference: https://forums.docker.com/t/file-access-in-mounted-volumes-extremely-slow-cpu-bound/8076/288
+clear_cache () {
+    rm -rf "${cache_dir}/*"
+}
+
+# download method - you pass a filename to save the file under, and the url to call
+cached_download () {
+    local output_file="${1}"
+    local url="${2}"
+
+    if fgrep -x "${url}" "${cache_dir}/urls.list" >/dev/null; then
+        echo "Retrieving '${url}' from cache..."
+        local line_number="$(fgrep -nx  "${url}" "${cache_dir}/urls.list" | sed 's/:.*//')"
+        cp "${cache_dir}/obj_$(printf "%05d\n" "${line_number}").cache" "${output_file}"
+    else
+        echo "Downloading '${url}' and placing in cache..."
+        rm -f "${output_file}"
+        $retry wget -O "${output_file}" --progress=dot:giga --server-response "${url}" 2>&1
+        local exit_code=$?
+        if [ "${exit_code}" == 0 ]; then
+            echo "${url}" >> "${cache_dir}/urls.list"
+            local line_number="$(fgrep -nx  "${url}" "${cache_dir}/urls.list" | sed 's/:.*//')"
+            cp "${output_file}" "${cache_dir}/obj_$(printf "%05d\n" "${line_number}").cache"
+        else
+            return "${exit_code}"
+        fi
+    fi
+}
diff --git a/tools/update-verify/release/common/check_updates.sh b/tools/update-verify/release/common/check_updates.sh
new file mode 100644
index 0000000000..6479f2f1f1
--- /dev/null
+++ b/tools/update-verify/release/common/check_updates.sh
@@ -0,0 +1,124 @@
+check_updates () {
+  # called with 10 args - platform, source package, target package, update package, old updater boolean,
+  # a path to the updater binary to use for the tests, a file to write diffs to, the update channel,
+  # update-settings.ini values, and a flag to indicate the target is dep-signed
+  update_platform=$1
+  source_package=$2
+  target_package=$3
+  locale=$4
+  use_old_updater=$5
+  updater=$6
+  diff_file=$7
+  channel=$8
+  mar_channel_IDs=$9
+  update_to_dep=${10}
+
+  # cleanup
+  rm -rf source/*
+  rm -rf target/*
+
+  unpack_build $update_platform source "$source_package" $locale '' $mar_channel_IDs
+  if [ "$?" != "0" ]; then
+    echo "FAILED: cannot unpack_build $update_platform source $source_package"
+    return 1
+  fi
+  unpack_build $update_platform target "$target_package" $locale 
+  if [ "$?" != "0" ]; then
+    echo "FAILED: cannot unpack_build $update_platform target $target_package"
+    return 1
+  fi
+  
+  case $update_platform in
+      Darwin_ppc-gcc | Darwin_Universal-gcc3 | Darwin_x86_64-gcc3 | Darwin_x86-gcc3-u-ppc-i386 | Darwin_x86-gcc3-u-i386-x86_64 | Darwin_x86_64-gcc3-u-i386-x86_64 | Darwin_aarch64-gcc3) 
+          platform_dirname="*.app"
+          ;;
+      WINNT*) 
+          platform_dirname="bin"
+          ;;
+      Linux_x86-gcc | Linux_x86-gcc3 | Linux_x86_64-gcc3) 
+          platform_dirname=`echo $product | tr '[A-Z]' '[a-z]'`
+          ;;
+  esac
+
+  if [ -f update/update.status ]; then rm update/update.status; fi
+  if [ -f update/update.log ]; then rm update/update.log; fi
+
+  if [ -d source/$platform_dirname ]; then
+    if [ `uname | cut -c-5` == "MINGW" ]; then
+      # windows
+      # change /c/path/to/pwd to c:\\path\\to\\pwd
+      four_backslash_pwd=$(echo $PWD | sed -e 's,^/\([a-zA-Z]\)/,\1:/,' | sed -e 's,/,\\\\,g')
+      two_backslash_pwd=$(echo $PWD | sed -e 's,^/\([a-zA-Z]\)/,\1:/,' | sed -e 's,/,\\,g')
+      cwd="$two_backslash_pwd\\source\\$platform_dirname"
+      update_abspath="$two_backslash_pwd\\update"
+    else
+      # not windows
+      # use ls here, because mac uses *.app, and we need to expand it
+      cwd=$(ls -d $PWD/source/$platform_dirname)
+      update_abspath="$PWD/update"
+    fi
+
+    cd_dir=$(ls -d ${PWD}/source/${platform_dirname})
+    cd "${cd_dir}"
+    set -x
+    "$updater" "$update_abspath" "$cwd" "$cwd" 0
+    set +x
+    cd ../..
+  else
+    echo "TEST-UNEXPECTED-FAIL: no dir in source/$platform_dirname"
+    return 1
+  fi
+
+  cat update/update.log
+  update_status=`cat update/update.status`
+
+  if [ "$update_status" != "succeeded" ]
+  then
+    echo "TEST-UNEXPECTED-FAIL: update status was not successful: $update_status"
+    return 1
+  fi
+
+  # If we were testing an OS X mar on Linux, the unpack step copied the
+  # precomplete file from Contents/Resources to the root of the install
+  # to ensure the Linux updater binary could find it. However, only the
+  # precomplete file in Contents/Resources was updated, which means
+  # the copied version in the root of the install will usually have some
+  # differences between the source and target. To prevent this false
+  # positive from failing the tests, we simply remove it before diffing.
+  # The precomplete file in Contents/Resources is still diffed, so we
+  # don't lose any coverage by doing this.
+  cd `echo "source/$platform_dirname"`
+  if [[ -f "Contents/Resources/precomplete" && -f "precomplete" ]]
+  then
+    rm "precomplete"
+  fi
+  cd ../..
+  cd `echo "target/$platform_dirname"`
+  if [[ -f "Contents/Resources/precomplete" && -f "precomplete" ]]
+  then
+    rm "precomplete"
+  fi
+  cd ../..
+
+  # If we are testing an OSX mar to update from a production-signed/notarized
+  # build to a dep-signed one, ignore Contents/CodeResources which won't be
+  # present in the target, to avoid spurious failures
+  if ${update_to_dep}; then
+    ignore_coderesources=--ignore-missing=Contents/CodeResources
+  else
+    ignore_coderesources=
+  fi
+
+  ../compare-directories.py source/${platform_dirname} target/${platform_dirname} ${channel} ${ignore_coderesources} > "${diff_file}"
+  diffErr=$?
+  cat "${diff_file}"
+  if [ $diffErr == 2 ]
+  then
+    echo "TEST-UNEXPECTED-FAIL: differences found after update"
+    return 1
+  elif [ $diffErr != 0 ]
+  then
+    echo "TEST-UNEXPECTED-FAIL: unknown error from diff: $diffErr"
+    return 3
+  fi
+}
diff --git a/tools/update-verify/release/common/download_builds.sh b/tools/update-verify/release/common/download_builds.sh
new file mode 100644
index 0000000000..e279c808db
--- /dev/null
+++ b/tools/update-verify/release/common/download_builds.sh
@@ -0,0 +1,36 @@
+pushd `dirname $0` &>/dev/null
+MY_DIR=$(pwd)
+popd &>/dev/null
+retry="$MY_DIR/../../../../mach python -m redo.cmd -s 1 -a 3"
+
+download_builds() {
+  # cleanup
+  mkdir -p downloads/
+  rm -rf downloads/*
+
+  source_url="$1"
+  target_url="$2"
+
+  if [ -z "$source_url" ] || [ -z "$target_url" ]
+  then
+    "download_builds usage: <source_url> <target_url>"
+    exit 1
+  fi
+
+  for url in "$source_url" "$target_url"
+    do
+    source_file=`basename "$url"`
+    if [ -f "$source_file" ]; then rm "$source_file"; fi
+    cd downloads 
+    if [ -f "$source_file" ]; then rm "$source_file"; fi
+    cached_download "${source_file}" "${url}"
+    status=$?
+    if [ $status != 0 ]; then
+      echo "TEST-UNEXPECTED-FAIL: Could not download source $source_file from $url"
+      echo "skipping.."
+      cd ../
+      return $status
+    fi
+    cd ../
+  done
+}
diff --git a/tools/update-verify/release/common/download_mars.sh b/tools/update-verify/release/common/download_mars.sh
new file mode 100644
index 0000000000..d2dab107d2
--- /dev/null
+++ b/tools/update-verify/release/common/download_mars.sh
@@ -0,0 +1,105 @@
+download_mars () {
+    update_url="$1"
+    only="$2"
+    test_only="$3"
+    to_build_id="$4"
+    to_app_version="$5"
+    to_display_version="$6"
+
+    max_tries=5
+    try=1
+    # retrying until we get offered an update
+    while [ "$try" -le "$max_tries" ]; do
+        echo "Using  $update_url"
+        # retrying until AUS gives us any response at all
+        cached_download update.xml "${update_url}"
+
+        echo "Got this response:"
+        cat update.xml
+        # If the first line after <updates> is </updates> then we have an
+        # empty snippet. Otherwise we're done
+        if [ "$(grep -A1 '<updates>' update.xml | tail -1)" != "</updates>" ]; then
+            break;
+        fi
+        echo "Empty response, sleeping"
+        sleep 5
+        try=$(($try+1))
+    done
+
+    echo; echo;  # padding
+
+    update_line=`fgrep "<update " update.xml`
+    grep_rv=$?
+    if [ 0 -ne $grep_rv ]; then
+        echo "TEST-UNEXPECTED-FAIL: no <update/> found for $update_url"
+        return 1
+    fi
+    command=`echo $update_line | sed -e 's/^.*<update //' -e 's:>.*$::' -e 's:\&amp;:\&:g'`
+    eval "export $command"
+
+    if [ ! -z "$to_build_id" -a "$buildID" != "$to_build_id" ]; then
+        echo "TEST-UNEXPECTED-FAIL: expected buildID $to_build_id does not match actual $buildID"
+        return 1
+    fi
+
+    if [ ! -z "$to_display_version" -a "$displayVersion" != "$to_display_version" ]; then
+        echo "TEST-UNEXPECTED-FAIL: expected displayVersion $to_display_version does not match actual $displayVersion"
+        return 1
+    fi
+
+    if [ ! -z "$to_app_version" -a "$appVersion" != "$to_app_version" ]; then
+        echo "TEST-UNEXPECTED-FAIL: expected appVersion $to_app_version does not match actual $appVersion"
+        return 1
+    fi
+
+    mkdir -p update/
+    if [ -z $only ]; then
+      only="partial complete"
+    fi
+    for patch_type in $only
+      do
+      line=`fgrep "patch type=\"$patch_type" update.xml`
+      grep_rv=$?
+
+      if [ 0 -ne $grep_rv ]; then
+        echo "TEST-UNEXPECTED-FAIL: no $patch_type update found for $update_url"
+        return 1
+      fi
+
+      command=`echo $line | sed -e 's/^.*<patch //' -e 's:/>.*$::' -e 's:\&amp;:\&:g'`
+      eval "export $command"
+
+      if [ "$test_only" == "1" ]
+      then
+        echo "Testing $URL"
+        curl -s -I -L $URL
+        return
+      else
+        cached_download "update/${patch_type}.mar" "${URL}"
+      fi
+      if [ "$?" != 0 ]; then
+        echo "Could not download $patch_type!"
+        echo "from: $URL"
+      fi
+      actual_size=`perl -e "printf \"%d\n\", (stat(\"update/$patch_type.mar\"))[7]"`
+      actual_hash=`openssl dgst -$hashFunction update/$patch_type.mar | sed -e 's/^.*= //'`
+
+      if [ $actual_size != $size ]; then
+          echo "TEST-UNEXPECTED-FAIL: $patch_type from $update_url wrong size"
+          echo "TEST-UNEXPECTED-FAIL: update.xml size: $size"
+          echo "TEST-UNEXPECTED-FAIL: actual size: $actual_size"
+          return 1
+      fi
+
+      if [ $actual_hash != $hashValue ]; then
+          echo "TEST-UNEXPECTED-FAIL: $patch_type from $update_url wrong hash"
+          echo "TEST-UNEXPECTED-FAIL: update.xml hash: $hashValue"
+          echo "TEST-UNEXPECTED-FAIL: actual hash: $actual_hash"
+          return 1
+      fi
+
+      cp update/$patch_type.mar update/update.mar
+      echo $actual_size > update/$patch_type.size
+
+    done
+}
diff --git a/tools/update-verify/release/common/installdmg.ex b/tools/update-verify/release/common/installdmg.ex
new file mode 100755
index 0000000000..08bcf9a201
--- /dev/null
+++ b/tools/update-verify/release/common/installdmg.ex
@@ -0,0 +1,45 @@
+#!/usr/bin/expect
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is Mozilla Corporation Code.
+#
+# The Initial Developer of the Original Code is
+# Clint Talbert.
+# Portions created by the Initial Developer are Copyright (C) 2007
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#  Armen Zambrano Gasparnian <armenzg@mozilla.com>
+#  Axel Hecht <l10n@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+#send_user $argv
+spawn hdiutil attach -readonly -mountroot /tmp -private -noautoopen [lindex $argv 0]
+expect {
+"byte" {send "G"; exp_continue}
+"END" {send "\r"; exp_continue}
+"Y/N?" {send "Y\r"; exp_continue}
+}
diff --git a/tools/update-verify/release/common/unpack-diskimage.sh b/tools/update-verify/release/common/unpack-diskimage.sh
new file mode 100755
index 0000000000..b647a69c4d
--- /dev/null
+++ b/tools/update-verify/release/common/unpack-diskimage.sh
@@ -0,0 +1,95 @@
+#!/bin/bash
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is the installdmg.sh script from taols utilities
+#
+# The Initial Developer of the Original Code is
+# Mozilla Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2009
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#  Chris AtLee <catlee@mozilla.com>
+#  Robert Kaiser <kairo@kairo.at>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+# Unpack a disk image to a specified target folder
+#
+# Usage: unpack-diskimage <image_file>
+#                         <mountpoint>
+#                         <target_path>
+
+DMG_PATH=$1
+MOUNTPOINT=$2
+TARGETPATH=$3
+LOGFILE=unpack.output
+
+# How long to wait before giving up waiting for the mount to fininsh
+TIMEOUT=90
+
+# If the mount point already exists, then the previous run may not have cleaned
+# up properly.  We should try to umount and remove the its directory.
+if [ -d $MOUNTPOINT ]; then
+    echo "$MOUNTPOINT already exists, trying to clean up"
+    hdiutil detach $MOUNTPOINT -force
+    rm -rdfv $MOUNTPOINT
+fi
+
+# Install an on-exit handler that will unmount and remove the '$MOUNTPOINT' directory
+trap "{ if [ -d $MOUNTPOINT ]; then hdiutil detach $MOUNTPOINT -force; rm -rdfv $MOUNTPOINT; fi; }" EXIT
+
+mkdir -p $MOUNTPOINT
+
+hdiutil attach -verbose -noautoopen -mountpoint $MOUNTPOINT "$DMG_PATH" &> $LOGFILE
+# Wait for files to show up
+# hdiutil uses a helper process, diskimages-helper, which isn't always done its
+# work by the time hdiutil exits.  So we wait until something shows up in the
+# mount point directory.
+i=0
+while [ "$(echo $MOUNTPOINT/*)" == "$MOUNTPOINT/*" ]; do
+    if [ $i -gt $TIMEOUT ]; then
+        echo "No files found, exiting"
+        exit 1
+    fi
+    sleep 1
+    i=$(expr $i + 1)
+done
+# Now we can copy everything out of the $MOUNTPOINT directory into the target directory
+rsync -av $MOUNTPOINT/* $MOUNTPOINT/.DS_Store $MOUNTPOINT/.background $MOUNTPOINT/.VolumeIcon.icns $TARGETPATH/ > $LOGFILE
+# sometimes hdiutil fails with "Resource busy"
+hdiutil detach $MOUNTPOINT || { sleep  10; \
+    if [ -d $MOUNTPOINT ]; then hdiutil detach $MOUNTPOINT -force; fi; }
+i=0
+while [ "$(echo $MOUNTPOINT/*)" != "$MOUNTPOINT/*" ]; do
+    if [ $i -gt $TIMEOUT ]; then
+        echo "Cannot umount, exiting"
+        exit 1
+    fi
+    sleep 1
+    i=$(expr $i + 1)
+done
+rm -rdf $MOUNTPOINT
diff --git a/tools/update-verify/release/common/unpack.sh b/tools/update-verify/release/common/unpack.sh
new file mode 100755
index 0000000000..3249936493
--- /dev/null
+++ b/tools/update-verify/release/common/unpack.sh
@@ -0,0 +1,121 @@
+#!/bin/bash
+
+function cleanup() { 
+    hdiutil detach ${DEV_NAME} || 
+      { sleep 5 && hdiutil detach ${DEV_NAME} -force; }; 
+    return $1 && $?; 
+};
+
+unpack_build () {
+    unpack_platform="$1"
+    dir_name="$2"
+    pkg_file="$3"
+    locale=$4
+    unpack_jars=$5
+    update_settings_string=$6
+
+    if [ ! -f "$pkg_file" ]; then
+      return 1
+    fi 
+    mkdir -p $dir_name
+    pushd $dir_name > /dev/null
+    case $unpack_platform in
+        # $unpack_platform is either
+        # - a balrog platform name (from testing/mozharness/scripts/release/update-verify-config-creator.py)
+        # - a simple platform name (from tools/update-verify/release/updates/verify.sh)
+        mac|Darwin_*)
+            os=`uname`
+            # How we unpack a dmg differs depending on which platform we're on.
+            if [[ "$os" == "Darwin" ]]
+            then
+                cd ../
+                echo "installing $pkg_file"
+                ../common/unpack-diskimage.sh "$pkg_file" mnt $dir_name
+            else
+                7z x ../"$pkg_file" > /dev/null
+                if [ `ls -1 | wc -l` -ne 1 ]
+                then
+                    echo "Couldn't find .app package"
+                    return 1
+                fi
+                unpack_dir=$(ls -1)
+                unpack_dir=$(ls -d "${unpack_dir}")
+                mv "${unpack_dir}"/*.app .
+                rm -rf "${unpack_dir}"
+                appdir=$(ls -1)
+                appdir=$(ls -d *.app)
+                # The updater guesses the location of these files based on
+                # its own target architecture, not the mar. If we're not
+                # unpacking mac-on-mac, we need to copy them so it can find
+                # them. It's important to copy (and not move), because when
+                # we diff the installer vs updated build afterwards, the
+                # installer version will have them in their original place.
+                cp "${appdir}/Contents/Resources/update-settings.ini" "${appdir}/update-settings.ini"
+                cp "${appdir}/Contents/Resources/precomplete" "${appdir}/precomplete"
+            fi
+            update_settings_file="${appdir}/update-settings.ini"
+            ;;
+        win32|WINNT_*)
+            7z x ../"$pkg_file" > /dev/null
+            if [ -d localized ]
+            then
+              mkdir bin/
+              cp -rp nonlocalized/* bin/
+              cp -rp localized/*    bin/
+              rm -rf nonlocalized
+              rm -rf localized
+              if [ $(find optional/ | wc -l) -gt 1 ]
+              then 
+                cp -rp optional/*     bin/
+                rm -rf optional
+              fi
+            elif [ -d core ]
+            then
+              mkdir bin/
+              cp -rp core/* bin/
+              rm -rf core
+            else
+              for file in *.xpi
+              do
+                unzip -o $file > /dev/null
+              done
+              unzip -o ${locale}.xpi > /dev/null
+            fi
+            update_settings_file='bin/update-settings.ini'
+            ;;
+        linux|Linux_*)
+            if `echo $pkg_file | grep -q "tar.gz"`
+            then
+                tar xfz ../"$pkg_file" > /dev/null
+            elif `echo $pkg_file | grep -q "tar.bz2"`
+            then
+                tar xfj ../"$pkg_file" > /dev/null
+            else
+                echo "Unknown package type for file: $pkg_file"
+                exit 1
+            fi
+            update_settings_file=`echo $product | tr '[A-Z]' '[a-z]'`'/update-settings.ini'
+            ;;
+        *)
+            echo "Unknown platform to unpack: $unpack_platform"
+            exit 1
+    esac
+
+    if [ ! -z $unpack_jars ]; then
+        for f in `find . -name '*.jar' -o -name '*.ja'`; do
+            unzip -o "$f" -d "$f.dir" > /dev/null
+        done
+    fi
+
+    if [ ! -z $update_settings_string ]; then
+       echo "Modifying update-settings.ini"
+       cat  "${update_settings_file}" | sed -e "s/^ACCEPTED_MAR_CHANNEL_IDS.*/ACCEPTED_MAR_CHANNEL_IDS=${update_settings_string}/" > "${update_settings_file}.new"
+       diff -u "${update_settings_file}" "${update_settings_file}.new"
+       echo " "
+       rm "${update_settings_file}"
+       mv "${update_settings_file}.new" "${update_settings_file}"
+    fi
+
+    popd > /dev/null
+
+}