Adding upstream version 115.8.0esr.upstream/115.8.0esr

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 519 insertions, 0 deletions

diff --git a/tools/update-verify/release/final-verification.sh b/tools/update-verify/release/final-verification.sh
new file mode 100755
index 0000000000..879c64697f
--- /dev/null
+++ b/tools/update-verify/release/final-verification.sh
@@ -0,0 +1,519 @@
+#!/bin/bash
+
+function usage {
+    log "In the updates subdirectory of the directory this script is in,"
+    log "there are a bunch of config files. You should call this script,"
+    log "passing the names of one or more of those files as parameters"
+    log "to this script."
+    log ""
+    log "This will validate that the update.xml files all exist for the"
+    log "given config file, and that they report the correct file sizes"
+    log "for the associated mar files, and that the associated mar files"
+    log "are available on the update servers."
+    log ""
+    log "This script will spawn multiple curl processes to query the"
+    log "snippets (update.xml file downloads) and the download urls in"
+    log "parallel. The number of parallel curl processes can be managed"
+    log "with the -p MAX_PROCS option."
+    log ""
+    log "Only the first three bytes of the mar files are downloaded"
+    log "using curl -r 0-2 option to save time. GET requests are issued"
+    log "rather than HEAD requests, since Akamai (one of our CDN"
+    log "partners) caches GET and HEAD requests separately - therefore"
+    log "they can be out-of-sync, and it is important that we validate"
+    log "that the GET requests return the expected results."
+    log ""
+    log "Please note this script can run on linux and OS X. It has not"
+    log "been tested on Windows, but may also work. It can be run"
+    log "locally, and does not require access to the mozilla vpn or"
+    log "any other special network, since the update servers are"
+    log "available over the internet. However, it does require an"
+    log "up-to-date checkout of the tools repository, as the updates/"
+    log "subfolder changes over time, and reflects the currently"
+    log "available updates. It makes no changes to the update servers"
+    log "so there is no harm in running it. It simply generates a"
+    log "report. However, please try to avoid hammering the update"
+    log "servers aggressively, e.g. with thousands of parallel"
+    log "processes. For example, feel free to run the examples below,"
+    log "first making sure that your source code checkout is up-to-"
+    log "date on your own machine, to get the latest configs in the"
+    log "updates/ subdirectory."
+    log ""
+    log "Usage:"
+    log "    $(basename "${0}") [-p MAX_PROCS] config1 [config2 config3 config4 ...]"
+    log "    $(basename "${0}") -h"
+    log ""
+    log "Examples:"
+    log "    1. $(basename "${0}") -p 128 mozBeta-thunderbird-linux.cfg mozBeta-thunderbird-linux64.cfg"
+    log "    2. $(basename "${0}") mozBeta-thunderbird-linux64.cfg"
+}
+
+function log {
+    echo "$(date):  ${1}"
+}
+
+# subprocesses don't log in real time, due to synchronisation
+# issues which can cause log entries to overwrite each other.
+# therefore this function outputs log entries written to
+# temporary files on disk, and then deletes them.
+function flush_logs {
+    ls -1rt "${TMPDIR}" | grep '^log\.' | while read LOG
+    do
+        cat "${TMPDIR}/${LOG}"
+        rm "${TMPDIR}/${LOG}"
+    done
+}
+
+# this function takes an update.xml url as an argument
+# and then logs a list of config files and their line
+# numbers, that led to this update.xml url being tested
+function show_cfg_file_entries {
+    local update_xml_url="${1}"
+    cat "${update_xml_urls}" | cut -f1 -d' ' | grep -Fn "${update_xml_url}" | sed 's/:.*//' | while read match_line_no
+    do
+        cfg_file="$(sed -n -e "${match_line_no}p" "${update_xml_urls}" | cut -f3 -d' ')"
+        cfg_line_no="$(sed -n -e "${match_line_no}p" "${update_xml_urls}" | cut -f4 -d' ')"
+        log "        ${cfg_file} line ${cfg_line_no}: $(sed -n -e "${cfg_line_no}p" "${cfg_file}")"
+    done
+}
+
+# this function takes a mar url as an argument and then
+# logs information about which update.xml urls referenced
+# this mar url, and which config files referenced those
+# mar urls - so you have a full understanding of why this
+# mar url was ever tested
+function show_update_xml_entries {
+    local mar_url="${1}"
+    grep -Frl "${mar_url}" "${TMPDIR}" | grep '/update_xml_to_mar\.' | while read update_xml_to_mar
+    do
+        mar_size="$(cat "${update_xml_to_mar}" | cut -f2 -d' ')"
+        update_xml_url="$(cat "${update_xml_to_mar}" | cut -f3 -d' ')"
+        patch_type="$(cat "${update_xml_to_mar}" | cut -f4 -d' ')"
+        update_xml_actual_url="$(cat "${update_xml_to_mar}" | cut -f5 -d' ')"
+        log "        ${update_xml_url}"
+        [ -n "${update_xml_actual_url}" ] && log "            which redirected to: ${update_xml_actual_url}"
+        log "            This contained an entry for:"
+        log "                patch type: ${patch_type}"
+        log "                mar size: ${mar_size}"
+        log "                mar url: ${mar_url}"
+        log "            The update.xml url above was retrieved because of the following cfg file entries:"
+        show_cfg_file_entries "${update_xml_url}" | sed 's/        /                /'
+    done
+}
+
+echo -n "$(date):  Command called:"
+for ((INDEX=0; INDEX<=$#; INDEX+=1))
+do
+    echo -n " '${!INDEX}'"
+done
+echo ''
+log "From directory: '$(pwd)'"
+log ''
+log "Parsing arguments..."
+
+# Max procs lowered in bug 894368 to try to avoid spurious failures
+MAX_PROCS=48
+BAD_ARG=0
+BAD_FILE=0
+while getopts p:h OPT
+do
+    case "${OPT}" in
+        p) MAX_PROCS="${OPTARG}";;
+        h) usage
+           exit;;
+        *) BAD_ARG=1;;
+    esac
+done
+shift "$((OPTIND - 1))"
+
+# invalid option specified
+[ "${BAD_ARG}" == 1 ] && exit 66
+
+log "Checking one or more config files have been specified..."
+if [ $# -lt 1 ]
+then
+    usage
+    log "ERROR: You must specify one or more config files"
+    exit 64
+fi
+
+log "Checking whether MAX_PROCS is a number..."
+if ! let x=MAX_PROCS 2>/dev/null
+then
+    usage
+    log "ERROR: MAX_PROCS must be a number (-p option); you specified '${MAX_PROCS}' - this is not a number."
+    exit 65
+fi
+
+# config files are in updates subdirectory below this script
+if ! cd "$(dirname "${0}")/updates" 2>/dev/null
+then
+    log "ERROR: Cannot cd into '$(dirname "${0}")/updates' from '$(pwd)'"
+    exit 68
+fi
+
+log "Checking specified config files (and downloading them if necessary):"
+log ''
+configs=()
+for file in "${@}"
+do
+    if [[ ${file} == http* ]]
+    then
+        log "  Downloading config file '${file}'"
+        cfg=$(mktemp)
+        curl -fL --retry 5 --compressed "${file}" > "$cfg"
+        if [ "$?" != 0 ]; then
+            log "Error downloading config file '${file}'"
+            BAD_FILE=1
+        else
+            log "  * '${file}' ok, downloaded to '${cfg}'"
+            configs+=($cfg)
+        fi
+    elif [ -f "${file}" ]
+    then
+        log "  * '${file}' ok"
+        configs+=(${file})
+    else
+        log "  * '${file}' missing"
+        BAD_FILE=1
+    fi
+done
+log ''
+
+# invalid config specified
+if [ "${BAD_FILE}" == 1 ]
+then
+    log "ERROR: Unable to download config file(s) or config files are missing from repo - see above"
+    exit 67
+fi
+
+log "All checks completed successfully."
+log ''
+log "Starting stopwatch..."
+log ''
+log "Please be aware output will now be buffered up, and only displayed after completion."
+log "Therefore do not be alarmed if you see no output for several minutes."
+log "See https://bugzilla.mozilla.org/show_bug.cgi?id=863602#c5 for details".
+log ''
+
+START_TIME="$(date +%s)"
+
+# Create a temporary directory for all temp files, that can easily be
+# deleted afterwards. See https://bugzilla.mozilla.org/show_bug.cgi?id=863602
+# to understand why we write everything in distinct temporary files rather
+# than writing to standard error/standard out or files shared across
+# processes.
+# Need to unset TMPDIR first since it affects mktemp behaviour on next line
+unset TMPDIR
+export TMPDIR="$(mktemp -d -t final_verification.XXXXXXXXXX)"
+
+# this temporary file will list all update urls that need to be checked, in this format:
+# <update url> <comma separated list of patch types> <cfg file that requests it> <line number of config file>
+# e.g.
+# https://aus4.mozilla.org/update/3/Firefox/18.0/20130104154748/Linux_x86_64-gcc3/zh-TW/releasetest/default/default/default/update.xml?force=1 complete moz20-firefox-linux64-major.cfg 3
+# https://aus4.mozilla.org/update/3/Firefox/18.0/20130104154748/Linux_x86_64-gcc3/zu/releasetest/default/default/default/update.xml?force=1 complete moz20-firefox-linux64.cfg 7
+# https://aus4.mozilla.org/update/3/Firefox/19.0/20130215130331/Linux_x86_64-gcc3/ach/releasetest/default/default/default/update.xml?force=1 complete,partial moz20-firefox-linux64-major.cfg 11
+# https://aus4.mozilla.org/update/3/Firefox/19.0/20130215130331/Linux_x86_64-gcc3/af/releasetest/default/default/default/update.xml?force=1 complete,partial moz20-firefox-linux64.cfg 17
+update_xml_urls="$(mktemp -t update_xml_urls.XXXXXXXXXX)"
+
+####################################################################################
+# And now a summary of all temp files that will get generated during this process...
+#
+# 1) mktemp -t failure.XXXXXXXXXX
+#
+# Each failure will generate a one line temp file, which is a space separated
+# output of the error code, and the instance data for the failure.
+# e.g.
+#
+# PATCH_TYPE_MISSING https://aus4.mozilla.org/update/3/Firefox/4.0b12/20110222205441/Linux_x86-gcc3/dummy-locale/releasetest/update.xml?force=1 complete https://aus4.mozilla.org/update/3/Firefox/4.0b12/20110222205441/Linux_x86-gcc3/dummy-locale/releasetest/default/default/default/update.xml?force=1
+#
+# 2) mktemp -t update_xml_to_mar.XXXXXXXXXX
+#
+# For each mar url referenced in an update.xml file, a temp file will be created to store the
+# association between update.xml url and mar url. This is later used (e.g. in function
+# show_update_xml_entries) to trace back the update.xml url(s) that led to a mar url being
+# tested. It is also used to keep a full list of mar urls to test.
+# e.g.
+#
+# <mar url> <mar size> <update.xml url> <patch type> <update.xml redirection url, if HTTP 302 returned>
+#
+# 3) mktemp -t log.XXXXXXXXXX
+#
+# For each log message logged by a subprocesses, we will create a temp log file with the
+# contents of the log message, since we cannot safely output the log message from the subprocess
+# and guarantee that it will be correctly output. By buffering log output in individual log files
+# we guarantee that log messages will not interfere with each other. We then flush them when all
+# forked subprocesses have completed.
+#
+# 4) mktemp -t mar_headers.XXXXXXXXXX
+#
+# We keep a copy of the mar url http headers retrieved in one file per mar url.
+#
+# 5) mktemp -t update.xml.headers.XXXXXXXXXX
+#
+# We keep a copy of the update.xml http headers retrieved in one file per update.xml url.
+#
+# 6) mktemp -t update.xml.XXXXXXXXXX
+#
+# We keep a copy of each update.xml file retrieved in individual files.
+####################################################################################
+
+
+# generate full list of update.xml urls, followed by patch types,
+# as defined in the specified config files - and write into "${update_xml_urls}" file
+aus_server="https://aus5.mozilla.org"
+for cfg_file in "${configs[@]}"
+do
+    line_no=0
+    sed -e 's/localtest/cdntest/' "${cfg_file}" | while read config_line
+    do
+        let line_no++
+        # to avoid contamination between iterations, reset variables
+        # each loop in case they are not declared
+        # aus_server is not "cleared" each iteration - to be consistent with previous behaviour of old
+        # final-verification.sh script - might be worth reviewing if we really want this behaviour
+        release="" product="" platform="" build_id="" locales="" channel="" from="" patch_types="complete"
+        eval "${config_line}"
+        for locale in ${locales}
+        do
+            echo "${aus_server}/update/3/$product/$release/$build_id/$platform/$locale/$channel/default/default/default/update.xml?force=1" "${patch_types// /,}" "${cfg_file}" "${line_no}"
+        done
+    done
+done > "${update_xml_urls}"
+
+# download update.xml files and grab the mar urls from downloaded file for each patch type required
+cat "${update_xml_urls}" | cut -f1-2 -d' ' | sort -u | xargs -n2 "-P${MAX_PROCS}" ../get-update-xml.sh
+if [ "$?" != 0 ]; then
+    flush_logs
+    log "Error generating update requests"
+    exit 70
+fi
+
+flush_logs
+
+# download http header for each mar url
+find "${TMPDIR}" -name 'update_xml_to_mar.*' -type f | xargs cat | cut -f1-2 -d' ' | sort -u | xargs -n2 "-P${MAX_PROCS}" ../test-mar-url.sh
+if [ "$?" != 0 ]; then
+    flush_logs
+    log "Error HEADing mar urls"
+    exit 71
+fi
+
+flush_logs
+
+log ''
+log 'Stopping stopwatch...'
+STOP_TIME="$(date +%s)"
+
+number_of_failures="$(find "${TMPDIR}" -name 'failure.*' -type f | wc -l | sed 's/ //g')"
+number_of_update_xml_urls="$(cat "${update_xml_urls}" | cut -f1 -d' ' | sort -u | wc -l | sed 's/ //g')"
+number_of_mar_urls="$(find "${TMPDIR}" -name "update_xml_to_mar.*" | xargs cat | cut -f1 -d' ' | sort -u | wc -l | sed 's/ //g')"
+
+if [ "${number_of_failures}" -eq 0 ]
+then
+    log
+    log "All tests passed successfully."
+    log
+    exit_code=0
+else
+    log ''
+    log '===================================='
+    [ "${number_of_failures}" -gt 1 ] && log "${number_of_failures} FAILURES" || log '1 FAILURE'
+    failure=0
+    ls -1tr "${TMPDIR}" | grep '^failure\.' | while read failure_file
+    do
+        while read failure_code entry1 entry2 entry3 entry4 entry5 entry6 entry7
+        do
+            log '===================================='
+            log ''
+            case "${failure_code}" in
+
+                UPDATE_XML_UNAVAILABLE)
+                    update_xml_url="${entry1}"
+                    update_xml="${entry2}"
+                    update_xml_headers="${entry3}"
+                    update_xml_debug="${entry4}"
+                    update_xml_curl_exit_code="${entry5}"
+                    log "FAILURE $((++failure)): Update xml file not available"
+                    log ""
+                    log "    Download url: ${update_xml_url}"
+                    log "    Curl returned exit code: ${update_xml_curl_exit_code}"
+                    log ""
+                    log "    The HTTP headers were:"
+                    sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${update_xml_headers}"
+                    log ""
+                    log "    The full curl debug output was:"
+                    sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${update_xml_debug}"
+                    log ""
+                    log "    The returned update.xml file was:"
+                    sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${update_xml}"
+                    log ""
+                    log "    This url was tested because of the following cfg file entries:"
+                    show_cfg_file_entries "${update_xml_url}"
+                    log ""
+
+                    ;;
+
+                UPDATE_XML_REDIRECT_FAILED)
+                    update_xml_url="${entry1}"
+                    update_xml_actual_url="${entry2}"
+                    update_xml="${entry3}"
+                    update_xml_headers="${entry4}"
+                    update_xml_debug="${entry5}"
+                    update_xml_curl_exit_code="${entry6}"
+                    log "FAILURE $((++failure)): Update xml file not available at *redirected* location"
+                    log ""
+                    log "    Download url: ${update_xml_url}"
+                    log "    Redirected to: ${update_xml_actual_url}"
+                    log "    It could not be downloaded from this url."
+                    log "    Curl returned exit code: ${update_xml_curl_exit_code}"
+                    log ""
+                    log "    The HTTP headers were:"
+                    sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${update_xml_headers}"
+                    log ""
+                    log "    The full curl debug output was:"
+                    sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${update_xml_debug}"
+                    log ""
+                    log "    The returned update.xml file was:"
+                    sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${update_xml}"
+                    log ""
+                    log "    This url was tested because of the following cfg file entries:"
+                    show_cfg_file_entries "${update_xml_url}"
+                    log ""
+                    ;;
+
+                PATCH_TYPE_MISSING)
+                    update_xml_url="${entry1}"
+                    patch_type="${entry2}"
+                    update_xml="${entry3}"
+                    update_xml_headers="${entry4}"
+                    update_xml_debug="${entry5}"
+                    update_xml_actual_url="${entry6}"
+                    log "FAILURE $((++failure)): Patch type '${patch_type}' not present in the downloaded update.xml file."
+                    log ""
+                    log "    Update xml file downloaded from: ${update_xml_url}"
+                    [ -n "${update_xml_actual_url}" ] && log "    This redirected to the download url: ${update_xml_actual_url}"
+                    log "    Curl returned exit code: 0 (success)"
+                    log ""
+                    log "    The HTTP headers were:"
+                    sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${update_xml_headers}"
+                    log ""
+                    log "    The full curl debug output was:"
+                    sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${update_xml_debug}"
+                    log ""
+                    log "    The returned update.xml file was:"
+                    sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${update_xml}"
+                    log ""
+                    log "    This url and patch type combination was tested due to the following cfg file entries:"
+                    show_cfg_file_entries "${update_xml_url}"
+                    log ""
+                    ;;
+
+                NO_MAR_FILE)
+                    mar_url="${entry1}"
+                    mar_headers_file="${entry2}"
+                    mar_headers_debug_file="${entry3}"
+                    mar_file_curl_exit_code="${entry4}"
+                    mar_actual_url="${entry5}"
+                    log "FAILURE $((++failure)): Could not retrieve mar file"
+                    log ""
+                    log "    Mar file url: ${mar_url}"
+                    [ -n "${mar_actual_url}" ] && log "    This redirected to: ${mar_actual_url}"
+                    log "    The mar file could not be downloaded from this location."
+                    log "    Curl returned exit code: ${mar_file_curl_exit_code}"
+                    log ""
+                    log "    The HTTP headers were:"
+                    sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${mar_headers_file}"
+                    log ""
+                    log "    The full curl debug output was:"
+                    sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${mar_headers_debug_file}"
+                    log ""
+                    log "    The mar download was tested because it was referenced in the following update xml file(s):"
+                    show_update_xml_entries "${mar_url}"
+                    log ""
+                    ;;
+
+                MAR_FILE_WRONG_SIZE)
+                    mar_url="${entry1}"
+                    mar_required_size="${entry2}"
+                    mar_actual_size="${entry3}"
+                    mar_headers_file="${entry4}"
+                    mar_headers_debug_file="${entry5}"
+                    mar_file_curl_exit_code="${entry6}"
+                    mar_actual_url="${entry7}"
+                    log "FAILURE $((++failure)): Mar file is wrong size"
+                    log ""
+                    log "    Mar file url: ${mar_url}"
+                    [ -n "${mar_actual_url}" ] && log "    This redirected to: ${mar_actual_url}"
+                    log "    The http header of the mar file url says that the mar file is ${mar_actual_size} bytes."
+                    log "    One or more of the following update.xml file(s) says that the file should be ${mar_required_size} bytes."
+                    log ""
+                    log "    These are the update xml file(s) that referenced this mar:"
+                    show_update_xml_entries "${mar_url}"
+                    log ""
+                    log "    Curl returned exit code: ${mar_file_curl_exit_code}"
+                    log ""
+                    log "    The HTTP headers were:"
+                    sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${mar_headers_file}"
+                    log ""
+                    log "    The full curl debug output was:"
+                    sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${mar_headers_debug_file}"
+                    log ""
+                    ;;
+
+                BAD_HTTP_RESPONSE_CODE_FOR_MAR)
+                    mar_url="${entry1}"
+                    mar_headers_file="${entry2}"
+                    mar_headers_debug_file="${entry3}"
+                    mar_file_curl_exit_code="${entry4}"
+                    mar_actual_url="${entry5}"
+                    http_response_code="$(sed -e "s/$(printf '\r')//" -n -e '/^HTTP\//p' "${mar_headers_file}" | tail -1)"
+                    log "FAILURE $((++failure)): '${http_response_code}' for mar file"
+                    log ""
+                    log "    Mar file url: ${mar_url}"
+                    [ -n "${mar_actual_url}" ] && log "    This redirected to: ${mar_actual_url}"
+                    log ""
+                    log "    These are the update xml file(s) that referenced this mar:"
+                    show_update_xml_entries "${mar_url}"
+                    log ""
+                    log "    Curl returned exit code: ${mar_file_curl_exit_code}"
+                    log ""
+                    log "    The HTTP headers were:"
+                    sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${mar_headers_file}"
+                    log ""
+                    log "    The full curl debug output was:"
+                    sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${mar_headers_debug_file}"
+                    log ""
+                    ;;
+
+                *)
+                    log "ERROR: Unknown failure code - '${failure_code}'"
+                    log "ERROR: This is a serious bug in this script."
+                    log "ERROR: Only known failure codes are: UPDATE_XML_UNAVAILABLE, UPDATE_XML_REDIRECT_FAILED, PATCH_TYPE_MISSING, NO_MAR_FILE, MAR_FILE_WRONG_SIZE, BAD_HTTP_RESPONSE_CODE_FOR_MAR"
+                    log ""
+                    log "FAILURE $((++failure)): Data from failure is: ${entry1} ${entry2} ${entry3} ${entry4} ${entry5} ${entry6}"
+                    log ""
+                    ;;
+
+            esac
+        done < "${TMPDIR}/${failure_file}"
+    done
+    exit_code=1
+fi
+
+
+log ''
+log '===================================='
+log 'KEY STATS'
+log '===================================='
+log ''
+log "Config files scanned:                       ${#@}"
+log "Update xml files downloaded and parsed:     ${number_of_update_xml_urls}"
+log "Unique mar urls found:                      ${number_of_mar_urls}"
+log "Failures:                                   ${number_of_failures}"
+log "Parallel processes used (maximum limit):    ${MAX_PROCS}"
+log "Execution time:                             $((STOP_TIME-START_TIME)) seconds"
+log ''
+
+rm -rf "${TMPDIR}"
+exit ${exit_code}