summaryrefslogtreecommitdiffstats
path: root/tools/update-verify
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-15 03:35:49 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-15 03:35:49 +0000
commitd8bbc7858622b6d9c278469aab701ca0b609cddf (patch)
treeeff41dc61d9f714852212739e6b3738b82a2af87 /tools/update-verify
parentReleasing progress-linux version 125.0.3-1~progress7.99u1. (diff)
downloadfirefox-d8bbc7858622b6d9c278469aab701ca0b609cddf.tar.xz
firefox-d8bbc7858622b6d9c278469aab701ca0b609cddf.zip
Merging upstream version 126.0.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'tools/update-verify')
-rw-r--r--tools/update-verify/release/common/cached_download.sh16
-rw-r--r--tools/update-verify/release/common/check_updates.sh80
-rw-r--r--tools/update-verify/release/common/download_builds.sh11
-rw-r--r--tools/update-verify/release/common/download_mars.sh55
-rwxr-xr-xtools/update-verify/release/common/unpack-diskimage.sh32
-rwxr-xr-xtools/update-verify/release/common/unpack.sh68
-rwxr-xr-xtools/update-verify/release/compare-directories.py38
-rwxr-xr-xtools/update-verify/release/updates/verify.sh106
8 files changed, 241 insertions, 165 deletions
diff --git a/tools/update-verify/release/common/cached_download.sh b/tools/update-verify/release/common/cached_download.sh
index 7cb3c42f8d..4522b5504d 100644
--- a/tools/update-verify/release/common/cached_download.sh
+++ b/tools/update-verify/release/common/cached_download.sh
@@ -1,10 +1,12 @@
+#!/bin/bash
# this library works like a wrapper around wget, to allow downloads to be cached
# so that if later the same url is retrieved, the entry from the cache will be
# returned.
-pushd `dirname $0` &>/dev/null
+pushd "$(dirname "$0")" &>/dev/null || exit
cache_dir="$(pwd)/cache"
-popd &>/dev/null
+popd &>/dev/null || exit
+retry="$MY_DIR/../../../../mach python -m redo.cmd -s 1 -a 3"
# Deletes all files in the cache directory
# We don't support folders or .dot(hidden) files
@@ -12,7 +14,7 @@ popd &>/dev/null
# which are the only workaround to poor mount r/w performance on MacOS
# Reference: https://forums.docker.com/t/file-access-in-mounted-volumes-extremely-slow-cpu-bound/8076/288
clear_cache () {
- rm -rf "${cache_dir}/*"
+ rm -rf "${cache_dir:?}/*"
}
# download method - you pass a filename to save the file under, and the url to call
@@ -20,9 +22,10 @@ cached_download () {
local output_file="${1}"
local url="${2}"
- if fgrep -x "${url}" "${cache_dir}/urls.list" >/dev/null; then
+ if grep -Fx "${url}" "${cache_dir}/urls.list" >/dev/null; then
echo "Retrieving '${url}' from cache..."
- local line_number="$(fgrep -nx "${url}" "${cache_dir}/urls.list" | sed 's/:.*//')"
+ local line_number
+ line_number="$(grep -Fnx "${url}" "${cache_dir}/urls.list" | sed 's/:.*//')"
cp "${cache_dir}/obj_$(printf "%05d\n" "${line_number}").cache" "${output_file}"
else
echo "Downloading '${url}' and placing in cache..."
@@ -31,7 +34,8 @@ cached_download () {
local exit_code=$?
if [ "${exit_code}" == 0 ]; then
echo "${url}" >> "${cache_dir}/urls.list"
- local line_number="$(fgrep -nx "${url}" "${cache_dir}/urls.list" | sed 's/:.*//')"
+ local line_number
+ line_number="$(grep -Fnx "${url}" "${cache_dir}/urls.list" | sed 's/:.*//')"
cp "${output_file}" "${cache_dir}/obj_$(printf "%05d\n" "${line_number}").cache"
else
return "${exit_code}"
diff --git a/tools/update-verify/release/common/check_updates.sh b/tools/update-verify/release/common/check_updates.sh
index acf06d8b4e..f03d8a069d 100644
--- a/tools/update-verify/release/common/check_updates.sh
+++ b/tools/update-verify/release/common/check_updates.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+
check_updates () {
# called with 10 args - platform, source package, target package, update package, old updater boolean,
# a path to the updater binary to use for the tests, a file to write diffs to, the update channel,
@@ -6,24 +8,34 @@ check_updates () {
source_package=$2
target_package=$3
locale=$4
- use_old_updater=$5
- updater=$6
- diff_file=$7
- channel=$8
- mar_channel_IDs=$9
- update_to_dep=${10}
+ updater=$5
+ diff_file=$6
+ channel=$7
+ mar_channel_IDs=$8
+ update_to_dep=$9
+ local mac_update_settings_dir_override
+ mac_update_settings_dir_override=${10}
+ local product
+ product=${11}
# cleanup
rm -rf source/*
rm -rf target/*
- unpack_build $update_platform source "$source_package" $locale '' $mar_channel_IDs
- if [ "$?" != "0" ]; then
+ # $mac_update_settings_dir_override allows unpack_build to find a host platform appropriate
+ # `update-settings.ini` file, which is needed to successfully run the updater later in this
+ # function.
+ if ! unpack_build "$update_platform" source "$source_package" "$locale" '' "$mar_channel_IDs" "$mac_update_settings_dir_override" "$product"; then
echo "FAILED: cannot unpack_build $update_platform source $source_package"
return 1
fi
- unpack_build $update_platform target "$target_package" $locale
- if [ "$?" != "0" ]; then
+ # Unlike unpacking the `source` build, we don't actually _need_ $mac_update_settings_dir_override
+ # here to succesfully apply the update, but its usage in `source` causes an `update-settings.ini`
+ # file to be present in the directory we diff, which means we either also need it present in the
+ # `target` directory, or to remove it after the update is applied. The latter was chosen
+ # because it keeps the workaround close together (as opposed to just above this, and then much
+ # further down).
+ if ! unpack_build "$update_platform" target "$target_package" "$locale" '' '' "$mac_update_settings_dir_override" "$product"; then
echo "FAILED: cannot unpack_build $update_platform target $target_package"
return 1
fi
@@ -36,30 +48,35 @@ check_updates () {
platform_dirname="bin"
;;
Linux_x86-gcc | Linux_x86-gcc3 | Linux_x86_64-gcc3)
- platform_dirname=`echo $product | tr '[A-Z]' '[a-z]'`
+ platform_dirname=$(echo "$product" | tr '[:upper:]' '[:lower:]')
;;
esac
if [ -f update/update.status ]; then rm update/update.status; fi
if [ -f update/update.log ]; then rm update/update.log; fi
+ # This check is disabled because we rely on glob expansion here
+ # shellcheck disable=SC2086
if [ -d source/$platform_dirname ]; then
- if [ `uname | cut -c-5` == "MINGW" ]; then
+ if [ "$(uname | cut -c-5)" == "MINGW" ]; then
# windows
# change /c/path/to/pwd to c:\\path\\to\\pwd
- four_backslash_pwd=$(echo $PWD | sed -e 's,^/\([a-zA-Z]\)/,\1:/,' | sed -e 's,/,\\\\,g')
- two_backslash_pwd=$(echo $PWD | sed -e 's,^/\([a-zA-Z]\)/,\1:/,' | sed -e 's,/,\\,g')
+ two_backslash_pwd=$(echo "$PWD" | sed -e 's,^/\([a-zA-Z]\)/,\1:/,' | sed -e 's,/,\\,g')
cwd="$two_backslash_pwd\\source\\$platform_dirname"
update_abspath="$two_backslash_pwd\\update"
else
# not windows
# use ls here, because mac uses *.app, and we need to expand it
+ # This check is disabled because we rely on glob expansion here
+ # shellcheck disable=SC2086
cwd=$(ls -d $PWD/source/$platform_dirname)
update_abspath="$PWD/update"
fi
+ # This check is disabled because we rely on glob expansion here
+ # shellcheck disable=SC2086
cd_dir=$(ls -d ${PWD}/source/${platform_dirname})
- cd "${cd_dir}"
+ cd "${cd_dir}" || (echo "TEST-UNEXPECTED-FAIL: couldn't cd to ${cd_dir}" && return 1)
set -x
"$updater" "$update_abspath" "$cwd" "$cwd" 0
set +x
@@ -70,7 +87,7 @@ check_updates () {
fi
cat update/update.log
- update_status=`cat update/update.status`
+ update_status=$(cat update/update.status)
if [ "$update_status" != "succeeded" ]
then
@@ -87,13 +104,17 @@ check_updates () {
# positive from failing the tests, we simply remove it before diffing.
# The precomplete file in Contents/Resources is still diffed, so we
# don't lose any coverage by doing this.
- cd `echo "source/$platform_dirname"`
+ # This check is disabled because we rely on glob expansion here
+ # shellcheck disable=SC2086
+ cd source/$platform_dirname || (echo "TEST-UNEXPECTED-FAIL: couldn't cd to source/${platform_dirname}" && exit 1)
if [[ -f "Contents/Resources/precomplete" && -f "precomplete" ]]
then
rm "precomplete"
fi
cd ../..
- cd `echo "target/$platform_dirname"`
+ # This check is disabled because we rely on glob expansion here
+ # shellcheck disable=SC2086
+ cd target/$platform_dirname || (echo "TEST-UNEXPECTED-FAIL: couldn't cd to target/${platform_dirname}" && exit 1)
if [[ -f "Contents/Resources/precomplete" && -f "precomplete" ]]
then
rm "precomplete"
@@ -110,7 +131,28 @@ check_updates () {
ignore_coderesources=
fi
- ../compare-directories.py source/${platform_dirname} target/${platform_dirname} ${channel} ${ignore_coderesources} > "${diff_file}"
+ # On Mac, there are two Frameworks that are not included with updates, and
+ # which change with every build. Because of this, we ignore differences in
+ # them in `compare-directories.py`. The best verification we can do for them
+ # is that they still exist.
+ if [[ $update_platform == Darwin_* ]]; then
+ # This check is disabled because we rely on glob expansion here
+ # shellcheck disable=SC2086
+ if ! compgen -G source/${platform_dirname}/Contents/MacOS/updater.app/Contents/Frameworks/UpdateSettings.framework >/dev/null; then
+ echo "TEST-UNEXPECTED-FAIL: UpdateSettings.framework doesn't exist after update"
+ return 4
+ fi
+ # This check is disabled because we rely on glob expansion here
+ # shellcheck disable=SC2086
+ if ! compgen -G source/${platform_dirname}/Contents/Frameworks/ChannelPrefs.framework >/dev/null; then
+ echo "TEST-UNEXPECTED-FAIL: ChannelPrefs.framework doesn't exist after update"
+ return 5
+ fi
+ fi
+
+ # This check is disabled because we rely on glob expansion here
+ # shellcheck disable=SC2086
+ ../compare-directories.py source/${platform_dirname} target/${platform_dirname} "${channel}" ${ignore_coderesources} > "${diff_file}"
diffErr=$?
cat "${diff_file}"
if [ $diffErr == 2 ]
diff --git a/tools/update-verify/release/common/download_builds.sh b/tools/update-verify/release/common/download_builds.sh
index e279c808db..0374ecaffc 100644
--- a/tools/update-verify/release/common/download_builds.sh
+++ b/tools/update-verify/release/common/download_builds.sh
@@ -1,7 +1,4 @@
-pushd `dirname $0` &>/dev/null
-MY_DIR=$(pwd)
-popd &>/dev/null
-retry="$MY_DIR/../../../../mach python -m redo.cmd -s 1 -a 3"
+#!/bin/bash
download_builds() {
# cleanup
@@ -13,15 +10,15 @@ download_builds() {
if [ -z "$source_url" ] || [ -z "$target_url" ]
then
- "download_builds usage: <source_url> <target_url>"
+ echo "download_builds usage: <source_url> <target_url>"
exit 1
fi
for url in "$source_url" "$target_url"
do
- source_file=`basename "$url"`
+ source_file=$(basename "$url")
if [ -f "$source_file" ]; then rm "$source_file"; fi
- cd downloads
+ cd downloads || exit
if [ -f "$source_file" ]; then rm "$source_file"; fi
cached_download "${source_file}" "${url}"
status=$?
diff --git a/tools/update-verify/release/common/download_mars.sh b/tools/update-verify/release/common/download_mars.sh
index d2dab107d2..84cbfe2ccc 100644
--- a/tools/update-verify/release/common/download_mars.sh
+++ b/tools/update-verify/release/common/download_mars.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+
download_mars () {
update_url="$1"
only="$2"
@@ -23,42 +25,49 @@ download_mars () {
fi
echo "Empty response, sleeping"
sleep 5
- try=$(($try+1))
+ try=$((try+1))
done
echo; echo; # padding
- update_line=`fgrep "<update " update.xml`
+ update_line=$(grep -F "<update " update.xml)
grep_rv=$?
if [ 0 -ne $grep_rv ]; then
echo "TEST-UNEXPECTED-FAIL: no <update/> found for $update_url"
return 1
fi
- command=`echo $update_line | sed -e 's/^.*<update //' -e 's:>.*$::' -e 's:\&amp;:\&:g'`
+ command=$(echo "$update_line" | sed -e 's/^.*<update //' -e 's:>.*$::' -e 's:\&amp;:\&:g')
eval "export $command"
- if [ ! -z "$to_build_id" -a "$buildID" != "$to_build_id" ]; then
+ # buildID and some other variables further down are gathered by eval'ing
+ # the massaged `update.xml` file a bit further up. Because of this, shellcheck
+ # cannot verify their existence, and gets grumpy. Ideally we would do this
+ # differently, but it's not worth the trouble at the time of writing.
+ # shellcheck disable=SC2154
+ if [ -n "$to_build_id" ] && [ "$buildID" != "$to_build_id" ]; then
echo "TEST-UNEXPECTED-FAIL: expected buildID $to_build_id does not match actual $buildID"
return 1
fi
- if [ ! -z "$to_display_version" -a "$displayVersion" != "$to_display_version" ]; then
+ # shellcheck disable=SC2154
+ if [ -n "$to_display_version" ] && [ "$displayVersion" != "$to_display_version" ]; then
echo "TEST-UNEXPECTED-FAIL: expected displayVersion $to_display_version does not match actual $displayVersion"
return 1
fi
- if [ ! -z "$to_app_version" -a "$appVersion" != "$to_app_version" ]; then
+ # shellcheck disable=SC2154
+ if [ -n "$to_app_version" ] && [ "$appVersion" != "$to_app_version" ]; then
echo "TEST-UNEXPECTED-FAIL: expected appVersion $to_app_version does not match actual $appVersion"
return 1
fi
mkdir -p update/
- if [ -z $only ]; then
+ if [ -z "$only" ]; then
only="partial complete"
fi
for patch_type in $only
do
- line=`fgrep "patch type=\"$patch_type" update.xml`
+ line=$(grep -F "patch type=\"$patch_type" update.xml)
grep_rv=$?
if [ 0 -ne $grep_rv ]; then
@@ -66,40 +75,44 @@ download_mars () {
return 1
fi
- command=`echo $line | sed -e 's/^.*<patch //' -e 's:/>.*$::' -e 's:\&amp;:\&:g'`
+ command=$(echo "$line" | sed -e 's/^.*<patch //' -e 's:/>.*$::' -e 's:\&amp;:\&:g')
eval "export $command"
if [ "$test_only" == "1" ]
then
echo "Testing $URL"
- curl -s -I -L $URL
+ curl -s -I -L "$URL"
return
else
- cached_download "update/${patch_type}.mar" "${URL}"
- fi
- if [ "$?" != 0 ]; then
- echo "Could not download $patch_type!"
- echo "from: $URL"
+ if ! cached_download "update/${patch_type}.mar" "${URL}"; then
+ echo "Could not download $patch_type!"
+ echo "from: $URL"
+ fi
fi
- actual_size=`perl -e "printf \"%d\n\", (stat(\"update/$patch_type.mar\"))[7]"`
- actual_hash=`openssl dgst -$hashFunction update/$patch_type.mar | sed -e 's/^.*= //'`
+ actual_size=$(perl -e "printf \"%d\n\", (stat(\"update/$patch_type.mar\"))[7]")
+ # shellcheck disable=SC2154
+ actual_hash=$(openssl dgst -"$hashFunction" update/"$patch_type".mar | sed -e 's/^.*= //')
- if [ $actual_size != $size ]; then
+ # shellcheck disable=SC2154
+ if [ "$actual_size" != "$size" ]; then
echo "TEST-UNEXPECTED-FAIL: $patch_type from $update_url wrong size"
+ # shellcheck disable=SC2154
echo "TEST-UNEXPECTED-FAIL: update.xml size: $size"
echo "TEST-UNEXPECTED-FAIL: actual size: $actual_size"
return 1
fi
- if [ $actual_hash != $hashValue ]; then
+ # shellcheck disable=SC2154
+ if [ "$actual_hash" != "$hashValue" ]; then
echo "TEST-UNEXPECTED-FAIL: $patch_type from $update_url wrong hash"
+ # shellcheck disable=SC2154
echo "TEST-UNEXPECTED-FAIL: update.xml hash: $hashValue"
echo "TEST-UNEXPECTED-FAIL: actual hash: $actual_hash"
return 1
fi
- cp update/$patch_type.mar update/update.mar
- echo $actual_size > update/$patch_type.size
+ cp update/"$patch_type".mar update/update.mar
+ echo "$actual_size" > update/"$patch_type".size
done
}
diff --git a/tools/update-verify/release/common/unpack-diskimage.sh b/tools/update-verify/release/common/unpack-diskimage.sh
index b647a69c4d..760fe0bd33 100755
--- a/tools/update-verify/release/common/unpack-diskimage.sh
+++ b/tools/update-verify/release/common/unpack-diskimage.sh
@@ -53,43 +53,43 @@ TIMEOUT=90
# If the mount point already exists, then the previous run may not have cleaned
# up properly. We should try to umount and remove the its directory.
-if [ -d $MOUNTPOINT ]; then
+if [ -d "$MOUNTPOINT" ]; then
echo "$MOUNTPOINT already exists, trying to clean up"
- hdiutil detach $MOUNTPOINT -force
- rm -rdfv $MOUNTPOINT
+ hdiutil detach "$MOUNTPOINT" -force
+ rm -rdfv "$MOUNTPOINT"
fi
# Install an on-exit handler that will unmount and remove the '$MOUNTPOINT' directory
-trap "{ if [ -d $MOUNTPOINT ]; then hdiutil detach $MOUNTPOINT -force; rm -rdfv $MOUNTPOINT; fi; }" EXIT
+trap '{ if [ -d $MOUNTPOINT ]; then hdiutil detach $MOUNTPOINT -force; rm -rdfv $MOUNTPOINT; fi; }' EXIT
-mkdir -p $MOUNTPOINT
+mkdir -p "$MOUNTPOINT"
-hdiutil attach -verbose -noautoopen -mountpoint $MOUNTPOINT "$DMG_PATH" &> $LOGFILE
+hdiutil attach -verbose -noautoopen -mountpoint "$MOUNTPOINT" "$DMG_PATH" &> $LOGFILE
# Wait for files to show up
# hdiutil uses a helper process, diskimages-helper, which isn't always done its
# work by the time hdiutil exits. So we wait until something shows up in the
# mount point directory.
i=0
-while [ "$(echo $MOUNTPOINT/*)" == "$MOUNTPOINT/*" ]; do
- if [ $i -gt $TIMEOUT ]; then
+while [ "$(echo "$MOUNTPOINT"/*)" == "$MOUNTPOINT/*" ]; do
+ if [ "$i" -gt $TIMEOUT ]; then
echo "No files found, exiting"
exit 1
fi
sleep 1
- i=$(expr $i + 1)
+ i=$((i+1))
done
# Now we can copy everything out of the $MOUNTPOINT directory into the target directory
-rsync -av $MOUNTPOINT/* $MOUNTPOINT/.DS_Store $MOUNTPOINT/.background $MOUNTPOINT/.VolumeIcon.icns $TARGETPATH/ > $LOGFILE
+rsync -av "$MOUNTPOINT"/* "$MOUNTPOINT"/.DS_Store "$MOUNTPOINT"/.background "$MOUNTPOINT"/.VolumeIcon.icns "$TARGETPATH"/ > $LOGFILE
# sometimes hdiutil fails with "Resource busy"
-hdiutil detach $MOUNTPOINT || { sleep 10; \
- if [ -d $MOUNTPOINT ]; then hdiutil detach $MOUNTPOINT -force; fi; }
+hdiutil detach "$MOUNTPOINT" || { sleep 10; \
+ if [ -d "$MOUNTPOINT" ]; then hdiutil detach "$MOUNTPOINT" -force; fi; }
i=0
-while [ "$(echo $MOUNTPOINT/*)" != "$MOUNTPOINT/*" ]; do
- if [ $i -gt $TIMEOUT ]; then
+while [ "$(echo "$MOUNTPOINT"/*)" != "$MOUNTPOINT/*" ]; do
+ if [ "$i" -gt $TIMEOUT ]; then
echo "Cannot umount, exiting"
exit 1
fi
sleep 1
- i=$(expr $i + 1)
+ i=$((i+1))
done
-rm -rdf $MOUNTPOINT
+rm -rdf "$MOUNTPOINT"
diff --git a/tools/update-verify/release/common/unpack.sh b/tools/update-verify/release/common/unpack.sh
index 3249936493..a0e1204487 100755
--- a/tools/update-verify/release/common/unpack.sh
+++ b/tools/update-verify/release/common/unpack.sh
@@ -1,11 +1,5 @@
#!/bin/bash
-function cleanup() {
- hdiutil detach ${DEV_NAME} ||
- { sleep 5 && hdiutil detach ${DEV_NAME} -force; };
- return $1 && $?;
-};
-
unpack_build () {
unpack_platform="$1"
dir_name="$2"
@@ -13,27 +7,35 @@ unpack_build () {
locale=$4
unpack_jars=$5
update_settings_string=$6
+ # If provided, must be a directory containing `update-settings.ini` which
+ # will be used instead of attempting to find this file in the unpacked
+ # build. `update_settings_string` modifications will still be performed on
+ # the file.
+ local mac_update_settings_dir_override
+ mac_update_settings_dir_override=$7
+ local product
+ product=$8
if [ ! -f "$pkg_file" ]; then
return 1
fi
- mkdir -p $dir_name
- pushd $dir_name > /dev/null
+ mkdir -p "$dir_name"
+ pushd "$dir_name" > /dev/null || exit
case $unpack_platform in
# $unpack_platform is either
# - a balrog platform name (from testing/mozharness/scripts/release/update-verify-config-creator.py)
# - a simple platform name (from tools/update-verify/release/updates/verify.sh)
mac|Darwin_*)
- os=`uname`
+ os=$(uname)
# How we unpack a dmg differs depending on which platform we're on.
if [[ "$os" == "Darwin" ]]
then
cd ../
echo "installing $pkg_file"
- ../common/unpack-diskimage.sh "$pkg_file" mnt $dir_name
+ ../common/unpack-diskimage.sh "$pkg_file" mnt "$dir_name"
else
7z x ../"$pkg_file" > /dev/null
- if [ `ls -1 | wc -l` -ne 1 ]
+ if [ "$(find . -mindepth 1 -maxdepth 1 | wc -l)" -ne 1 ]
then
echo "Couldn't find .app package"
return 1
@@ -43,14 +45,18 @@ unpack_build () {
mv "${unpack_dir}"/*.app .
rm -rf "${unpack_dir}"
appdir=$(ls -1)
- appdir=$(ls -d *.app)
- # The updater guesses the location of these files based on
- # its own target architecture, not the mar. If we're not
- # unpacking mac-on-mac, we need to copy them so it can find
- # them. It's important to copy (and not move), because when
- # we diff the installer vs updated build afterwards, the
- # installer version will have them in their original place.
- cp "${appdir}/Contents/Resources/update-settings.ini" "${appdir}/update-settings.ini"
+ appdir=$(ls -d ./*.app)
+ if [ -d "${mac_update_settings_dir_override}" ]; then
+ cp "${mac_update_settings_dir_override}/update-settings.ini" "${appdir}/update-settings.ini"
+ else
+ # The updater guesses the location of these files based on
+ # its own target architecture, not the mar. If we're not
+ # unpacking mac-on-mac, we need to copy them so it can find
+ # them. It's important to copy (and not move), because when
+ # we diff the installer vs updated build afterwards, the
+ # installer version will have them in their original place.
+ cp "${appdir}/Contents/Resources/update-settings.ini" "${appdir}/update-settings.ini"
+ fi
cp "${appdir}/Contents/Resources/precomplete" "${appdir}/precomplete"
fi
update_settings_file="${appdir}/update-settings.ini"
@@ -64,7 +70,7 @@ unpack_build () {
cp -rp localized/* bin/
rm -rf nonlocalized
rm -rf localized
- if [ $(find optional/ | wc -l) -gt 1 ]
+ if [ "$(find optional/ | wc -l)" -gt 1 ]
then
cp -rp optional/* bin/
rm -rf optional
@@ -77,45 +83,43 @@ unpack_build () {
else
for file in *.xpi
do
- unzip -o $file > /dev/null
+ unzip -o "$file" > /dev/null
done
- unzip -o ${locale}.xpi > /dev/null
+ unzip -o "${locale}".xpi > /dev/null
fi
update_settings_file='bin/update-settings.ini'
;;
linux|Linux_*)
- if `echo $pkg_file | grep -q "tar.gz"`
+ if echo "$pkg_file" | grep -q "tar.gz"
then
tar xfz ../"$pkg_file" > /dev/null
- elif `echo $pkg_file | grep -q "tar.bz2"`
+ elif echo "$pkg_file" | grep -q "tar.bz2"
then
tar xfj ../"$pkg_file" > /dev/null
else
echo "Unknown package type for file: $pkg_file"
exit 1
fi
- update_settings_file=`echo $product | tr '[A-Z]' '[a-z]'`'/update-settings.ini'
+ update_settings_file=$(echo "$product" | tr '[:upper:]' '[:lower:]')'/update-settings.ini'
;;
*)
echo "Unknown platform to unpack: $unpack_platform"
exit 1
esac
- if [ ! -z $unpack_jars ]; then
- for f in `find . -name '*.jar' -o -name '*.ja'`; do
- unzip -o "$f" -d "$f.dir" > /dev/null
- done
+ if [ -n "$unpack_jars" ]; then
+ find . \( -name '*.jar' -o -name '*.ja' \) -exec unzip -o {} -d {}.dir \; >/dev/null
fi
- if [ ! -z $update_settings_string ]; then
+ if [ -n "$update_settings_string" ]; then
echo "Modifying update-settings.ini"
- cat "${update_settings_file}" | sed -e "s/^ACCEPTED_MAR_CHANNEL_IDS.*/ACCEPTED_MAR_CHANNEL_IDS=${update_settings_string}/" > "${update_settings_file}.new"
+ sed -e "s/^ACCEPTED_MAR_CHANNEL_IDS.*/ACCEPTED_MAR_CHANNEL_IDS=${update_settings_string}/" < "${update_settings_file}" > "${update_settings_file}.new"
diff -u "${update_settings_file}" "${update_settings_file}.new"
echo " "
rm "${update_settings_file}"
mv "${update_settings_file}.new" "${update_settings_file}"
fi
- popd > /dev/null
+ popd > /dev/null || exit
}
diff --git a/tools/update-verify/release/compare-directories.py b/tools/update-verify/release/compare-directories.py
index ea70d79b31..714f02a66d 100755
--- a/tools/update-verify/release/compare-directories.py
+++ b/tools/update-verify/release/compare-directories.py
@@ -29,7 +29,6 @@ TRANSFORMS = [
# this can be removed once each channel has a watershed above 59.0b2 (from bug 1431342)
"files": [
"defaults/pref/channel-prefs.js",
- "Contents/Resources/defaults/pref/channel-prefs.js",
],
"channel_prefix": ["aurora", "beta", "release", "esr"],
"side": "source",
@@ -39,7 +38,6 @@ TRANSFORMS = [
# updates from a beta to an RC build, the latter specifies the release channel
"files": [
"defaults/pref/channel-prefs.js",
- "Contents/Resources/defaults/pref/channel-prefs.js",
],
"channel_prefix": ["beta"],
"side": "target",
@@ -52,7 +50,6 @@ TRANSFORMS = [
# updates from an RC to a beta build
"files": [
"defaults/pref/channel-prefs.js",
- "Contents/Resources/defaults/pref/channel-prefs.js",
],
"channel_prefix": ["beta"],
"side": "source",
@@ -69,7 +66,6 @@ TRANSFORMS = [
# to break before applying this transform.
"files": [
"defaults/pref/channel-prefs.js",
- "Contents/Resources/defaults/pref/channel-prefs.js",
],
"channel_prefix": ["aurora", "beta", "release", "esr"],
"side": "target",
@@ -80,7 +76,7 @@ TRANSFORMS = [
# updates from a beta to an RC build, the latter specifies the release channel
# on mac, we actually have both files. The second location is the real
# one but we copy to the first to run the linux64 updater
- "files": ["update-settings.ini", "Contents/Resources/update-settings.ini"],
+ "files": ["update-settings.ini"],
"channel_prefix": ["beta"],
"side": "target",
"substitution": [
@@ -88,20 +84,20 @@ TRANSFORMS = [
"ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-beta,firefox-mozilla-release\n",
],
},
- {
- # updates from an RC to a beta build
- # on mac, we only need to modify the legit file this time. unpack_build
- # handles the copy for the updater in both source and target
- "files": ["Contents/Resources/update-settings.ini"],
- "channel_prefix": ["beta"],
- "side": "source",
- "substitution": [
- "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-release\n",
- "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-beta,firefox-mozilla-release\n",
- ],
- },
]
+# Files that are expected to be different, but cannot be transformed to get a useful diff.
+# This should generally only be used for files that have unpredictable contents, eg:
+# things that are signed but not updated.
+IGNORE_FILES = (
+ "Contents/MacOS/updater.app/Contents/Frameworks/UpdateSettings.framework/Resources/Info.plist",
+ "Contents/MacOS/updater.app/Contents/Frameworks/UpdateSettings.framework/_CodeSignature/CodeResources",
+ "Contents/MacOS/updater.app/Contents/Frameworks/UpdateSettings.framework/UpdateSettings",
+ "Contents/Frameworks/ChannelPrefs.framework/Resources/Info.plist",
+ "Contents/Frameworks/ChannelPrefs.framework/_CodeSignature/CodeResources",
+ "Contents/Frameworks/ChannelPrefs.framework/ChannelPrefs",
+)
+
def walk_dir(path):
all_files = []
@@ -170,6 +166,14 @@ def compare_common_files(files, channel, source_dir, target_dir):
source_file
) != hash_file(target_file):
logging.info("Difference found in {}".format(filename))
+ if filename in IGNORE_FILES:
+ logging.info(
+ "Ignoring difference in {} because it is listed in IGNORE_FILES".format(
+ filename
+ )
+ )
+ continue
+
file_contents = {
"source": open(source_file).readlines(),
"target": open(target_file).readlines(),
diff --git a/tools/update-verify/release/updates/verify.sh b/tools/update-verify/release/updates/verify.sh
index 3f8556b424..4f7201c048 100755
--- a/tools/update-verify/release/updates/verify.sh
+++ b/tools/update-verify/release/updates/verify.sh
@@ -20,14 +20,14 @@ to_app_version=""
to_display_version=""
override_certs=""
diff_summary_log=${DIFF_SUMMARY_LOG:-"$PWD/diff-summary.log"}
-if [ -e ${diff_summary_log} ]; then
- rm ${diff_summary_log}
+if [ -e "${diff_summary_log}" ]; then
+ rm "${diff_summary_log}"
fi
-touch ${diff_summary_log}
+touch "${diff_summary_log}"
-pushd `dirname $0` &>/dev/null
+pushd "$(dirname "$0")" &>/dev/null || exit
MY_DIR=$(pwd)
-popd &>/dev/null
+popd &>/dev/null || exit
retry="$MY_DIR/../../../../mach python -m redo.cmd -s 1 -a 3"
cert_replacer="$MY_DIR/../replace-updater-certs.py"
@@ -82,7 +82,7 @@ do
arg="$1"
shift
set -- "$@" "$arg"
- pass_arg_count=`expr $pass_arg_count + 1`
+ pass_arg_count=$((pass_arg_count + 1))
esac
done
@@ -100,7 +100,7 @@ then
exit 0
fi
-while read entry
+while read -r entry
do
# initialize all config variables
release=""
@@ -111,21 +111,10 @@ do
channel=""
from=""
patch_types="complete"
- use_old_updater=0
mar_channel_IDs=""
updater_package=""
- eval $entry
-
- # the arguments for updater changed in Gecko 34/SeaMonkey 2.31
- major_version=`echo $release | cut -f1 -d.`
- if [[ "$product" == "seamonkey" ]]; then
- minor_version=`echo $release | cut -f2 -d.`
- if [[ $major_version -le 2 && $minor_version -lt 31 ]]; then
- use_old_updater=1
- fi
- elif [[ $major_version -lt 34 ]]; then
- use_old_updater=1
- fi
+ mac_update_settings_dir_override=""
+ eval "$entry"
# Note: cross platform tests seem to work for everything except Mac-on-Windows.
# We probably don't care about this use case.
@@ -144,11 +133,11 @@ do
then
if [ "$runmode" == "$TEST_ONLY" ]
then
- download_mars "${aus_server}/update/3/${update_path}/default/update.xml?force=1" ${patch_type} 1 \
+ download_mars "${aus_server}/update/3/${update_path}/default/update.xml?force=1" "${patch_type}" 1 \
"${to_build_id}" "${to_app_version}" "${to_display_version}"
err=$?
else
- download_mars "${aus_server}/update/3/${update_path}/update.xml?force=1" ${patch_type} 0 \
+ download_mars "${aus_server}/update/3/${update_path}/update.xml?force=1" "${patch_type}" 0 \
"${to_build_id}" "${to_app_version}" "${to_display_version}"
err=$?
fi
@@ -157,9 +146,9 @@ do
continue
fi
else
- mkdir -p updates/${update_path}/complete
- mkdir -p updates/${update_path}/partial
- $retry wget -q -O ${patch_type} updates/${update_path}/${patch_type}/update.xml "${aus_server}/update/3/${update_path}/update.xml?force=1"
+ mkdir -p updates/"${update_path}"/complete
+ mkdir -p updates/"${update_path}"/partial
+ $retry wget -q -O "${patch_type}" updates/"${update_path}"/"${patch_type}"/update.xml "${aus_server}/update/3/${update_path}/update.xml?force=1"
fi
if [ "$runmode" == "$COMPLETE" ]
@@ -170,25 +159,42 @@ do
fi
updater_platform=""
- updater_package_url=`echo "${ftp_server_from}${updater_package}" | sed "s/%locale%/${locale}/"`
- updater_package_filename=`basename "$updater_package_url"`
+ updater_package_url=$(echo "${ftp_server_from}${updater_package}" | sed "s/%locale%/${locale}/")
+ updater_package_filename=$(basename "$updater_package_url")
case $updater_package_filename in
*dmg)
platform_dirname="*.app"
updater_bins="Contents/MacOS/updater.app/Contents/MacOS/updater Contents/MacOS/updater.app/Contents/MacOS/org.mozilla.updater"
updater_platform="mac"
+ mac_update_settings_dir_override=""
;;
*exe)
- updater_package_url=`echo "${updater_package_url}" | sed "s/ja-JP-mac/ja/"`
+ updater_package_url=${updater_package_url/ja-JP-mac/ja}
platform_dirname="bin"
updater_bins="updater.exe"
updater_platform="win32"
+ case $platform in
+ Darwin_*)
+ mac_update_settings_dir_override="${PWD}/updater/${platform_dirname}"
+ ;;
+ *)
+ mac_update_settings_dir_override=""
+ ;;
+ esac
;;
*bz2)
- updater_package_url=`echo "${updater_package_url}" | sed "s/ja-JP-mac/ja/"`
- platform_dirname=`echo $product | tr '[A-Z]' '[a-z]'`
+ updater_package_url=${updater_package_url/ja-JP-mac/ja}
+ platform_dirname=$(echo "$product" | tr '[:upper:]' '[:lower:]')
updater_bins="updater"
updater_platform="linux"
+ case $platform in
+ Darwin_*)
+ mac_update_settings_dir_override="${PWD}/updater/${platform_dirname}"
+ ;;
+ *)
+ mac_update_settings_dir_override=""
+ ;;
+ esac
;;
*)
echo "Couldn't detect updater platform"
@@ -198,10 +204,10 @@ do
rm -rf updater/*
cached_download "${updater_package_filename}" "${updater_package_url}"
- unpack_build "$updater_platform" updater "$updater_package_filename" "$locale"
+ unpack_build "$updater_platform" updater "$updater_package_filename" "$locale" "$product"
# Even on Windows, we want Unix-style paths for the updater, because of MSYS.
- cwd=$(\ls -d $PWD/updater/$platform_dirname)
+ cwd=$(\ls -d "$PWD"/updater/"$platform_dirname")
# Bug 1209376. Linux updater linked against other libraries in the installation directory
export LD_LIBRARY_PATH=$cwd
updater="null"
@@ -214,7 +220,7 @@ do
done
update_to_dep=false
- if [ ! -z "$override_certs" ]; then
+ if [ -n "$override_certs" ]; then
echo "Replacing certs in updater binary"
cp "${updater}" "${updater}.orig"
case ${override_certs} in
@@ -232,6 +238,8 @@ do
echo "Unknown override cert - skipping"
;;
esac
+ # because we actually rely on $overrides being split up into separate args 🤦
+ # shellcheck disable=SC2086
python3 "${cert_replacer}" "${MY_DIR}/../mar_certs" "${updater}.orig" "${updater}" ${overrides}
else
echo "override_certs is '${override_certs}', not replacing any certificates"
@@ -242,21 +250,23 @@ do
continue
fi
- from_path=`echo $from | sed "s/%locale%/${locale}/"`
- to_path=`echo $to | sed "s/%locale%/${locale}/"`
+ # Quotes around %locale% needed to prevent bash from interpreting the
+ # "%" characters as special characters.
+ from_path=${from/"%locale%"/${locale}}
+ to_path=${to/"%locale%"/${locale}}
download_builds "${ftp_server_from}${from_path}" "${ftp_server_to}${to_path}"
err=$?
if [ "$err" != "0" ]; then
echo "TEST-UNEXPECTED-FAIL: [$release $locale $patch_type] download_builds returned non-zero exit code: $err"
continue
fi
- source_file=`basename "$from_path"`
- target_file=`basename "$to_path"`
+ source_file=$(basename "$from_path")
+ target_file=$(basename "$to_path")
diff_file="results.diff"
if [ -e ${diff_file} ]; then
rm ${diff_file}
fi
- check_updates "${platform}" "downloads/${source_file}" "downloads/${target_file}" ${locale} ${use_old_updater} ${updater} ${diff_file} ${channel} "${mar_channel_IDs}" ${update_to_dep}
+ check_updates "${platform}" "downloads/${source_file}" "downloads/${target_file}" "${locale}" "${updater}" ${diff_file} "${channel}" "${mar_channel_IDs}" ${update_to_dep} "${mac_update_settings_dir_override}" "${product}"
err=$?
if [ "$err" == "0" ]; then
continue
@@ -268,25 +278,27 @@ do
echo "TEST-UNEXPECTED-FAIL: [$release $locale $patch_type] check_updates returned unknown error for $platform downloads/$source_file vs. downloads/$target_file: $err"
fi
- if [ -s ${diff_file} ]; then
- echo "Found diffs for ${patch_type} update from ${aus_server}/update/3/${update_path}/update.xml?force=1" >> ${diff_summary_log}
- cat ${diff_file} >> ${diff_summary_log}
- echo "" >> ${diff_summary_log}
+ if [ -s "${diff_file}" ]; then
+ {
+ echo "Found diffs for ${patch_type} update from ${aus_server}/update/3/${update_path}/update.xml?force=1"
+ cat "${diff_file}"
+ echo ""
+ } >> "${diff_summary_log}"
fi
fi
done
if [ -f update/partial.size ] && [ -f update/complete.size ]; then
- partial_size=`cat update/partial.size`
- complete_size=`cat update/complete.size`
- if [ $partial_size -gt $complete_size ]; then
+ partial_size=$(cat update/partial.size)
+ complete_size=$(cat update/complete.size)
+ if [ "$partial_size" -gt "$complete_size" ]; then
echo "TEST-UNEXPECTED-FAIL: [$release $locale $patch_type] partial updates are larger than complete updates"
- elif [ $partial_size -eq $complete_size ]; then
+ elif [ "$partial_size" -eq "$complete_size" ]; then
echo "WARN: [$release $locale $patch_type] partial updates are the same size as complete updates, this should only happen for major updates"
else
echo "SUCCESS: [$release $locale $patch_type] partial updates are smaller than complete updates, all is well in the universe"
fi
fi
done
-done < $config_file
+done < "$config_file"
clear_cache