path: root/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh

#!/bin/bash

set -ex

function usage {
  cat <<EOF

Usage: $(basename "$0") -h # Displays this usage/help text
Usage: $(basename "$0") -x # lists exit codes
Usage: $(basename "$0") [-p product]
           [-r existing_repo_dir]
           # Use mozilla-central builds to check HSTS & HPKP
           [--use-mozilla-central]
           # Use archive.m.o instead of the taskcluster index to get xpcshell
           [--use-ftp-builds]
           # One (or more) of the following actions must be specified.
           --hsts | --hpkp | --remote-settings | --suffix-list
           -b branch

EOF
}

PRODUCT="firefox"
BRANCH=""
PLATFORM_EXT="tar.bz2"
UNPACK_CMD="tar jxf"
CLOSED_TREE=false
DONTBUILD=false
APPROVAL=false
COMMIT_AUTHOR='ffxbld <ffxbld@mozilla.com>'
REPODIR=''
HGHOST="hg.mozilla.org"
STAGEHOST="archive.mozilla.org"
WGET="wget -nv"
UNTAR="tar -zxf"
DIFF="$(command -v diff) -u"
BASEDIR="${HOME}"

SCRIPTDIR="$(realpath "$(dirname "$0")")"
HG="$(command -v hg)"
DATADIR="${BASEDIR}/data"
mkdir -p "${DATADIR}"

USE_MC=false
USE_TC=true
JQ="$(command -v jq)"

DO_HSTS=false
HSTS_PRELOAD_SCRIPT="${SCRIPTDIR}/getHSTSPreloadList.js"
HSTS_PRELOAD_ERRORS="nsSTSPreloadList.errors"
HSTS_PRELOAD_INC_OLD="${DATADIR}/nsSTSPreloadList.inc"
HSTS_PRELOAD_INC_NEW="${BASEDIR}/${PRODUCT}/nsSTSPreloadList.inc"
HSTS_UPDATED=false

DO_HPKP=false
HPKP_PRELOAD_SCRIPT="${SCRIPTDIR}/genHPKPStaticPins.js"
HPKP_PRELOAD_ERRORS="StaticHPKPins.errors"
HPKP_PRELOAD_JSON="${DATADIR}/PreloadedHPKPins.json"
HPKP_PRELOAD_INC="StaticHPKPins.h"
HPKP_PRELOAD_INPUT="${DATADIR}/${HPKP_PRELOAD_INC}"
HPKP_PRELOAD_OUTPUT="${DATADIR}/${HPKP_PRELOAD_INC}.out"
HPKP_UPDATED=false

DO_REMOTE_SETTINGS=false
REMOTE_SETTINGS_SERVER=''
REMOTE_SETTINGS_INPUT="${DATADIR}/remote-settings.in"
REMOTE_SETTINGS_OUTPUT="${DATADIR}/remote-settings.out"
REMOTE_SETTINGS_DIR="/services/settings/dumps"
REMOTE_SETTINGS_UPDATED=false

DO_SUFFIX_LIST=false
GITHUB_SUFFIX_URL="https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat"
GITHUB_SUFFIX_LOCAL="public_suffix_list.dat"
HG_SUFFIX_LOCAL="effective_tld_names.dat"
HG_SUFFIX_PATH="/netwerk/dns/${HG_SUFFIX_LOCAL}"
SUFFIX_LIST_UPDATED=false

ARTIFACTS_DIR="${ARTIFACTS_DIR:-.}"
# Defaults
HSTS_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${HSTS_DIFF_ARTIFACT:-"nsSTSPreloadList.diff"}"
HPKP_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${HPKP_DIFF_ARTIFACT:-"StaticHPKPins.h.diff"}"
REMOTE_SETTINGS_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${REMOTE_SETTINGS_DIFF_ARTIFACT:-"remote-settings.diff"}"
SUFFIX_LIST_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${SUFFIX_LIST_DIFF_ARTIFACT:-"effective_tld_names.diff"}"

# duplicate the functionality of taskcluster-lib-urls, but in bash..
queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1"
index_base="$TASKCLUSTER_ROOT_URL/api/index/v1"

# Cleanup common artifacts.
function preflight_cleanup {
  cd "${BASEDIR}"
  rm -rf "${PRODUCT}" tests "${BROWSER_ARCHIVE}" "${TESTS_ARCHIVE}"
}

function download_shared_artifacts_from_ftp {
  cd "${BASEDIR}"

  # Download everything we need to run js with xpcshell
  echo "INFO: Downloading all the necessary pieces from ${STAGEHOST}..."
  ARTIFACT_DIR="nightly/latest-${REPODIR}"
  if [ "${USE_MC}" == "true" ]; then
    ARTIFACT_DIR="nightly/latest-mozilla-central"
  fi

  BROWSER_ARCHIVE_URL="https://${STAGEHOST}/pub/mozilla.org/${PRODUCT}/${ARTIFACT_DIR}/${BROWSER_ARCHIVE}"
  TESTS_ARCHIVE_URL="https://${STAGEHOST}/pub/mozilla.org/${PRODUCT}/${ARTIFACT_DIR}/${TESTS_ARCHIVE}"

  echo "INFO: ${WGET} ${BROWSER_ARCHIVE_URL}"
  ${WGET} "${BROWSER_ARCHIVE_URL}"
  echo "INFO: ${WGET} ${TESTS_ARCHIVE_URL}"
  ${WGET} "${TESTS_ARCHIVE_URL}"
}

function download_shared_artifacts_from_tc {
  cd "${BASEDIR}"
  TASKID_FILE="taskId.json"

  # Download everything we need to run js with xpcshell
  echo "INFO: Downloading all the necessary pieces from the taskcluster index..."
  TASKID_URL="$index_base/task/gecko.v2.${REPODIR}.shippable.latest.${PRODUCT}.linux64-opt"
  if [ "${USE_MC}" == "true" ]; then
    TASKID_URL="$index_base/task/gecko.v2.mozilla-central.shippable.latest.${PRODUCT}.linux64-opt"
  fi
  ${WGET} -O ${TASKID_FILE} "${TASKID_URL}"
  INDEX_TASK_ID="$($JQ -r '.taskId' ${TASKID_FILE})"
  if [ -z "${INDEX_TASK_ID}" ]; then
    echo "Failed to look up taskId at ${TASKID_URL}"
    exit 22
  else
    echo "INFO: Got taskId of $INDEX_TASK_ID"
  fi

  TASKSTATUS_FILE="taskstatus.json"
  STATUS_URL="$queue_base/task/${INDEX_TASK_ID}/status"
  ${WGET} -O "${TASKSTATUS_FILE}" "${STATUS_URL}"
  LAST_RUN_INDEX=$(($(jq '.status.runs | length' ${TASKSTATUS_FILE}) - 1))
  echo "INFO: Examining run number ${LAST_RUN_INDEX}"

  BROWSER_ARCHIVE_URL="$queue_base/task/${INDEX_TASK_ID}/runs/${LAST_RUN_INDEX}/artifacts/public/build/${BROWSER_ARCHIVE}"
  echo "INFO: ${WGET} ${BROWSER_ARCHIVE_URL}"
  ${WGET} "${BROWSER_ARCHIVE_URL}"

  TESTS_ARCHIVE_URL="$queue_base/task/${INDEX_TASK_ID}/runs/${LAST_RUN_INDEX}/artifacts/public/build/${TESTS_ARCHIVE}"
  echo "INFO: ${WGET} ${TESTS_ARCHIVE_URL}"
  ${WGET} "${TESTS_ARCHIVE_URL}"
}

function unpack_artifacts {
  cd "${BASEDIR}"
  if [ ! -f "${BROWSER_ARCHIVE}" ]; then
    echo "Downloaded file '${BROWSER_ARCHIVE}' not found in directory '$(pwd)'." >&2
    exit 31
  fi
  if [ ! -f "${TESTS_ARCHIVE}" ]; then
    echo "Downloaded file '${TESTS_ARCHIVE}' not found in directory '$(pwd)'." >&2
    exit 32
  fi
  # Unpack the browser and move xpcshell in place for updating the preload list.
  echo "INFO: Unpacking resources..."
  ${UNPACK_CMD} "${BROWSER_ARCHIVE}"
  mkdir -p tests
  cd tests
  ${UNTAR} "../${TESTS_ARCHIVE}"
  cd "${BASEDIR}"
  cp tests/bin/xpcshell "${PRODUCT}"
}

# Downloads the current in-tree HSTS (HTTP Strict Transport Security) files.
# Runs a simple xpcshell script to generate up-to-date HSTS information.
# Compares the new HSTS output with the old to determine whether we need to update.
function compare_hsts_files {
  cd "${BASEDIR}"

  HSTS_PRELOAD_INC_HG="${HGREPO}/raw-file/default/security/manager/ssl/$(basename "${HSTS_PRELOAD_INC_OLD}")"

  echo "INFO: Downloading existing include file..."
  rm -rf "${HSTS_PRELOAD_ERRORS}" "${HSTS_PRELOAD_INC_OLD}"
  echo "INFO: ${WGET} ${HSTS_PRELOAD_INC_HG}"
  ${WGET} -O "${HSTS_PRELOAD_INC_OLD}" "${HSTS_PRELOAD_INC_HG}"

  if [ ! -f "${HSTS_PRELOAD_INC_OLD}" ]; then
    echo "Downloaded file '${HSTS_PRELOAD_INC_OLD}' not found in directory '$(pwd)' - this should have been downloaded above from ${HSTS_PRELOAD_INC_HG}." >&2
    exit 41
  fi

  # Run the script to get an updated preload list.
  echo "INFO: Generating new HSTS preload list..."
  cd "${BASEDIR}/${PRODUCT}"
  if ! LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:. ./xpcshell "${HSTS_PRELOAD_SCRIPT}" "${HSTS_PRELOAD_INC_OLD}"; then
    echo "HSTS preload list generation failed" >&2
    exit 43
  fi

  # The created files should be non-empty.
  echo "INFO: Checking whether new HSTS preload list is valid..."
  if [ ! -s "${HSTS_PRELOAD_INC_NEW}" ]; then
    echo "New HSTS preload list ${HSTS_PRELOAD_INC_NEW} is empty. That's less good." >&2
    exit 42
  fi
  cd "${BASEDIR}"

  # Check for differences
  echo "INFO: diffing old/new HSTS preload lists into ${HSTS_DIFF_ARTIFACT}"
  ${DIFF} "${HSTS_PRELOAD_INC_OLD}" "${HSTS_PRELOAD_INC_NEW}" | tee "${HSTS_DIFF_ARTIFACT}"
  if [ -s "${HSTS_DIFF_ARTIFACT}" ]
  then
    return 0
  fi
  return 1
}

# Downloads the current in-tree HPKP (HTTP public key pinning) files.
# Runs a simple xpcshell script to generate up-to-date HPKP information.
# Compares the new HPKP output with the old to determine whether we need to update.
function compare_hpkp_files {
  cd "${BASEDIR}"
  HPKP_PRELOAD_JSON_HG="${HGREPO}/raw-file/default/security/manager/tools/$(basename "${HPKP_PRELOAD_JSON}")"

  HPKP_PRELOAD_OUTPUT_HG="${HGREPO}/raw-file/default/security/manager/ssl/${HPKP_PRELOAD_INC}"

  rm -f "${HPKP_PRELOAD_OUTPUT}"
  ${WGET} -O "${HPKP_PRELOAD_INPUT}" "${HPKP_PRELOAD_OUTPUT_HG}"
  ${WGET} -O "${HPKP_PRELOAD_JSON}" "${HPKP_PRELOAD_JSON_HG}"

  # Run the script to get an updated preload list.
  echo "INFO: Generating new HPKP preload list..."
  cd "${BASEDIR}/${PRODUCT}"
  if ! LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:. ./xpcshell "${HPKP_PRELOAD_SCRIPT}" "${HPKP_PRELOAD_JSON}" "${HPKP_PRELOAD_OUTPUT}" > "${HPKP_PRELOAD_ERRORS}"; then
    echo "HPKP preload list generation failed" >&2
    exit 54
  fi

  # The created files should be non-empty.
  echo "INFO: Checking whether new HPKP preload list is valid..."

  if [ ! -s "${HPKP_PRELOAD_OUTPUT}" ]; then
    echo "${HPKP_PRELOAD_OUTPUT} is empty. That's less good." >&2
    exit 52
  fi
  if ! grep kPreloadPKPinsExpirationTime "${HPKP_PRELOAD_OUTPUT}"; then
    echo "${HPKP_PRELOAD_OUTPUT} is missing an expiration time. Truncated?" >&2
    exit 53
  fi
  cd "${BASEDIR}"

  echo "INFO: diffing old/new HPKP preload lists..."
  ${DIFF} "${HPKP_PRELOAD_INPUT}" "${HPKP_PRELOAD_OUTPUT}" | tee "${HPKP_DIFF_ARTIFACT}"
  if [ -s "${HPKP_DIFF_ARTIFACT}" ]
  then
    return 0
  fi
  return 1
}

function is_valid_xml {
  xmlfile=$1
  XMLLINT=$(command -v xmllint 2>/dev/null | head -n1)

  if [ ! -x "${XMLLINT}" ]; then
    echo "ERROR: xmllint not found in PATH"
    exit 60
  fi
  ${XMLLINT} --nonet --noout "${xmlfile}"
}

# Downloads the public suffix list
function compare_suffix_lists {
  HG_SUFFIX_URL="${HGREPO}/raw-file/default/${HG_SUFFIX_PATH}"
  cd "${BASEDIR}"

  echo "INFO: ${WGET} -O ${GITHUB_SUFFIX_LOCAL} ${GITHUB_SUFFIX_URL}"
  rm -f "${GITHUB_SUFFIX_LOCAL}"
  ${WGET} -O "${GITHUB_SUFFIX_LOCAL}" "${GITHUB_SUFFIX_URL}"

  echo "INFO: ${WGET} -O ${HG_SUFFIX_LOCAL} ${HG_SUFFIX_URL}"
  rm -f "${HG_SUFFIX_LOCAL}"
  ${WGET} -O "${HG_SUFFIX_LOCAL}" "${HG_SUFFIX_URL}"

  echo "INFO: diffing in-tree suffix list against the suffix list from AMO..."
  ${DIFF} ${GITHUB_SUFFIX_LOCAL} ${HG_SUFFIX_LOCAL} | tee "${SUFFIX_LIST_DIFF_ARTIFACT}"
  if [ -s "${SUFFIX_LIST_DIFF_ARTIFACT}" ]
  then
    return 0
  fi
  return 1
}

function compare_remote_settings_files {
  REMOTE_SETTINGS_SERVER="https://firefox.settings.services.mozilla.com/v1"

  # 1. List remote settings collections from server.
  echo "INFO: fetch remote settings list from server"
  ${WGET} -qO- "${REMOTE_SETTINGS_SERVER}/buckets/monitor/collections/changes/records" |\
    ${JQ} -r '.data[] | .bucket+"/"+.collection+"/"+(.last_modified|tostring)' |\
    # 2. For each entry ${bucket, collection, last_modified}
  while IFS="/" read -r bucket collection last_modified; do

    # 3. Download the dump from HG into REMOTE_SETTINGS_INPUT folder
    hg_dump_url="${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${bucket}/${collection}.json"
    local_location_input="$REMOTE_SETTINGS_INPUT/${bucket}/${collection}.json"
    mkdir -p "$REMOTE_SETTINGS_INPUT/${bucket}"
    ${WGET} -qO "$local_location_input" "$hg_dump_url"
    if [ $? -eq 8 ]; then
      # We don't keep any dump for this collection, skip it.
      # Try to clean up in case no collection in this bucket has dump.
      rmdir "$REMOTE_SETTINGS_INPUT/${bucket}" --ignore-fail-on-non-empty
      continue
    fi

    # 4. Download server version into REMOTE_SETTINGS_OUTPUT folder
    remote_records_url="$REMOTE_SETTINGS_SERVER/buckets/${bucket}/collections/${collection}/changeset?_expected=${last_modified}"
    local_location_output="$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}.json"
    mkdir -p "$REMOTE_SETTINGS_OUTPUT/${bucket}"
    ${WGET} -qO- "$remote_records_url" | ${JQ} '{"data": .changes, "timestamp": .timestamp}' > "${local_location_output}"

    # 5. Download attachments if needed.
    if [ "${bucket}" = "blocklists" ] && [ "${collection}" = "addons-bloomfilters" ]; then
      # Find the attachment with the most recent generation_time, like _updateMLBF in Blocklist.jsm.
      # The server should return one "bloomfilter-base" record, but in case it returns multiple,
      # return the most recent one. The server may send multiple entries if we ever decide to use
      # the "filter_expression" feature of Remote Settings to send different records to specific
      # channels. In that case this code should be updated to recognize the filter expression,
      # but until we do, simply select the most recent record - can't go wrong with that.
      # Note that "attachment_type" and "generation_time" are specific to addons-bloomfilters.
      update_remote_settings_attachment "${bucket}" "${collection}" addons-mlbf.bin \
        'map(select(.attachment_type == "bloomfilter-base")) | sort_by(.generation_time) | last'
    fi
    # Here is an example to download an attachment with record identifier "ID":
    # update_remote_settings_attachment "${bucket}" "${collection}" ID '.[] | select(.id == "ID")'
    # NOTE: The downloaded data is not validated. xpcshell should be used for that.
  done

  echo "INFO: diffing old/new remote settings dumps..."
  ${DIFF} -r "${REMOTE_SETTINGS_INPUT}" "${REMOTE_SETTINGS_OUTPUT}" > "${REMOTE_SETTINGS_DIFF_ARTIFACT}"
  if [ -s "${REMOTE_SETTINGS_DIFF_ARTIFACT}" ]
  then
    return 0
  fi
  return 1
}

# Helper for compare_remote_settings_files to download attachments from remote settings.
# The format and location is documented at:
# https://firefox-source-docs.mozilla.org/services/common/services/RemoteSettings.html#packaging-attachments
function update_remote_settings_attachment() {
  local bucket=$1
  local collection=$2
  local attachment_id=$3
  # $4 is a jq filter on the arrays that should return one record with the attachment
  local jq_attachment_selector=".data | map(select(.attachment)) | $4"

  # These paths match _readAttachmentDump in services/settings/Attachments.jsm.
  local path_to_attachment="${bucket}/${collection}/${attachment_id}"
  local path_to_meta="${bucket}/${collection}/${attachment_id}.meta.json"
  local old_meta="$REMOTE_SETTINGS_INPUT/${path_to_meta}"
  local new_meta="$REMOTE_SETTINGS_OUTPUT/${path_to_meta}"

  # Those files should have been created by compare_remote_settings_files before the function call.
  local local_location_input="$REMOTE_SETTINGS_INPUT/${bucket}/${collection}.json"
  local local_location_output="$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}.json"

  # Compute the metadata based on already-downloaded records.
  mkdir -p "$REMOTE_SETTINGS_INPUT/${bucket}/${collection}"
  ${JQ} -cj <"$local_location_input" "${jq_attachment_selector}" > "${old_meta}"
  mkdir -p "$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}"
  ${JQ} -cj <"$local_location_output" "${jq_attachment_selector}" > "${new_meta}"

  if cmp --silent "${old_meta}" "${new_meta}" ; then
    # Metadata not changed, don't bother downloading the attachments themselves.
    return
  fi
  # Metadata changed. Download attachments.

  echo "INFO: Downloading updated remote settings dump: ${bucket}/${collection}/${attachment_id}"

  # Overwrited old_meta with the actual file from the repo. The content should be equivalent,
  # but can have minor differences (e.g. different line endings) if the checked in file was not
  # generated by this script (e.g. manually checked in).
  ${WGET} -qO "${old_meta}" "${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${path_to_meta}"

  ${WGET} -qO "${REMOTE_SETTINGS_INPUT}/${path_to_attachment}" "${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${path_to_attachment}"

  if [ -z "${ATTACHMENT_BASE_URL}" ] ; then
    ATTACHMENT_BASE_URL=$(${WGET} -qO- "${REMOTE_SETTINGS_SERVER}" | ${JQ} -r .capabilities.attachments.base_url)
  fi
  attachment_path_from_meta=$(${JQ} -r < "${new_meta}" .attachment.location)
  ${WGET} -qO "${REMOTE_SETTINGS_OUTPUT}/${path_to_attachment}" "${ATTACHMENT_BASE_URL}${attachment_path_from_meta}"
}

# Clones an hg repo
function clone_repo {
  cd "${BASEDIR}"
  if [ ! -d "${REPODIR}" ]; then
    ${HG} robustcheckout --sharebase /tmp/hg-store -b default "${HGREPO}" "${REPODIR}"
  fi

  ${HG} -R "${REPODIR}" pull
  ${HG} -R "${REPODIR}" update -C default
}

# Copies new HSTS files in place, and commits them.
function stage_hsts_files {
  cd "${BASEDIR}"
  cp -f "${HSTS_PRELOAD_INC_NEW}" "${REPODIR}/security/manager/ssl/"
}

function stage_hpkp_files {
  cd "${BASEDIR}"
  cp -f "${HPKP_PRELOAD_OUTPUT}" "${REPODIR}/security/manager/ssl/${HPKP_PRELOAD_INC}"
}

function stage_remote_settings_files {
  cd "${BASEDIR}"
  cp -a "${REMOTE_SETTINGS_OUTPUT}"/* "${REPODIR}${REMOTE_SETTINGS_DIR}"
}

function stage_tld_suffix_files {
  cd "${BASEDIR}"
  cp -a "${GITHUB_SUFFIX_LOCAL}" "${REPODIR}/${HG_SUFFIX_PATH}"
}

# Push all pending commits to Phabricator
function push_repo {
  cd "${REPODIR}"
  if [ ! -r "${HOME}/.arcrc" ]
  then
    return 1
  fi
  if ! ARC=$(command -v arc)
  then
    return 1
  fi
  if [ -z "${REVIEWERS}" ]
  then
    return 1
  fi
  # Clean up older review requests
  # Turn  Needs Review D624: No bug, Automated HSTS ...
  # into D624
  for diff in $($ARC list | grep "Needs Review" | grep -E "${BRANCH} repo-update" | awk 'match($0, /D[0-9]+[^: ]/) { print substr($0, RSTART, RLENGTH)  }')
  do
    echo "Removing old request $diff"
    # There is no 'arc abandon', see bug 1452082
    echo '{"transactions": [{"type":"abandon", "value": true}], "objectIdentifier": "'"${diff}"'"}' | $ARC call-conduit -- differential.revision.edit
  done

  $ARC diff --verbatim --reviewers "${REVIEWERS}"
}



# Main

# Parse our command-line options.
while [ $# -gt 0 ]; do
  case "$1" in
    -h) usage; exit 0 ;;
    -p) PRODUCT="$2"; shift ;;
    -b) BRANCH="$2"; shift ;;
    -n) DRY_RUN=true ;;
    -c) CLOSED_TREE=true ;;
    -d) DONTBUILD=true ;;
    -a) APPROVAL=true ;;
    --pinset) DO_PRELOAD_PINSET=true ;;
    --hsts) DO_HSTS=true ;;
    --hpkp) DO_HPKP=true ;;
    --remote-settings) DO_REMOTE_SETTINGS=true ;;
    --suffix-list) DO_SUFFIX_LIST=true ;;
    -r) REPODIR="$2"; shift ;;
    --use-mozilla-central) USE_MC=true ;;
    --use-ftp-builds) USE_TC=false ;;
    -*) usage
      exit 11 ;;
    *)  break ;; # terminate while loop
  esac
  shift
done

# Must supply a code branch to work with.
if [ "${BRANCH}" == "" ]; then
  echo "Error: You must specify a branch with -b branchname." >&2
  usage
  exit 12
fi

# Must choose at least one update action.
if [ "$DO_HSTS" == "false" ] && [ "$DO_HPKP" == "false" ] && [ "$DO_REMOTE_SETTINGS" == "false" ] && [ "$DO_SUFFIX_LIST" == "false" ]
then
  echo "Error: you must specify at least one action from: --hsts, --hpkp, --remote-settings, or --suffix-list" >&2
  usage
  exit 13
fi

# per-product constants
case "${PRODUCT}" in
  thunderbird)
    COMMIT_AUTHOR="tbirdbld <tbirdbld@thunderbird.net>"
    ;;
  firefox)
    ;;
  *)
    echo "Error: Invalid product specified"
    usage
    exit 14
    ;;
esac

if [ "${REPODIR}" == "" ]; then
  REPODIR="$(basename "${BRANCH}")"
fi

case "${BRANCH}" in
  mozilla-central|comm-central|try )
    HGREPO="https://${HGHOST}/${BRANCH}"
    ;;
  mozilla-*|comm-* )
    HGREPO="https://${HGHOST}/releases/${BRANCH}"
    ;;
  * )
    HGREPO="https://${HGHOST}/projects/${BRANCH}"
    ;;
esac

BROWSER_ARCHIVE="target.${PLATFORM_EXT}"
TESTS_ARCHIVE="target.common.tests.tar.gz"

preflight_cleanup
if [ "${DO_HSTS}" == "true" ] || [ "${DO_HPKP}" == "true" ] || [ "${DO_PRELOAD_PINSET}" == "true" ]
then
  if [ "${USE_TC}" == "true" ]; then
    download_shared_artifacts_from_tc
  else
    download_shared_artifacts_from_ftp
  fi
  unpack_artifacts
fi

if [ "${DO_HSTS}" == "true" ]; then
  if compare_hsts_files
  then
    HSTS_UPDATED=true
  fi
fi
if [ "${DO_HPKP}" == "true" ]; then
  if compare_hpkp_files
  then
    HPKP_UPDATED=true
  fi
fi
if [ "${DO_REMOTE_SETTINGS}" == "true" ]; then
  if compare_remote_settings_files
  then
    REMOTE_SETTINGS_UPDATED=true
  fi
fi
if [ "${DO_SUFFIX_LIST}" == "true" ]; then
  if compare_suffix_lists
  then
    SUFFIX_LIST_UPDATED=true
  fi
fi


if [ "${HSTS_UPDATED}" == "false" ] && [ "${HPKP_UPDATED}" == "false" ] && [ "${REMOTE_SETTINGS_UPDATED}" == "false" ] && [ "${SUFFIX_LIST_UPDATED}" == "false" ]; then
  echo "INFO: no updates required. Exiting."
  exit 0
else
  if [ "${DRY_RUN}" == "true" ]; then
    echo "INFO: Updates are available, not updating hg in dry-run mode."
    exit 2
  fi
fi

clone_repo

COMMIT_MESSAGE="No Bug, ${BRANCH} repo-update"
if [ "${HSTS_UPDATED}" == "true" ]
then
  stage_hsts_files
  COMMIT_MESSAGE="${COMMIT_MESSAGE} HSTS"
fi

if [ "${HPKP_UPDATED}" == "true" ]
then
  stage_hpkp_files
  COMMIT_MESSAGE="${COMMIT_MESSAGE} HPKP"
fi

if [ "${REMOTE_SETTINGS_UPDATED}" == "true" ]
then
  stage_remote_settings_files
  COMMIT_MESSAGE="${COMMIT_MESSAGE} remote-settings"
fi

if [ "${SUFFIX_LIST_UPDATED}" == "true" ]
then
  stage_tld_suffix_files
  COMMIT_MESSAGE="${COMMIT_MESSAGE} tld-suffixes"
fi


if [ ${DONTBUILD} == true ]; then
  COMMIT_MESSAGE="${COMMIT_MESSAGE} - (DONTBUILD)"
fi
if [ ${CLOSED_TREE} == true ]; then
  COMMIT_MESSAGE="${COMMIT_MESSAGE} - CLOSED TREE"
fi
if [ ${APPROVAL} == true ]; then
  COMMIT_MESSAGE="${COMMIT_MESSAGE} - a=repo-update"
fi


if ${HG} -R "${REPODIR}" commit -u "${COMMIT_AUTHOR}" -m "${COMMIT_MESSAGE}"
then
  push_repo
fi

echo "All done"