From 26a029d407be480d791972afb5975cf62c9360a6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 02:47:55 +0200 Subject: Adding upstream version 124.0.1. Signed-off-by: Daniel Baumann --- taskcluster/docker/updatebot/Dockerfile | 24 +++ taskcluster/docker/updatebot/VERSION | 1 + .../docker/updatebot/arcanist_patch_size.patch | 13 ++ .../docker/updatebot/arcanist_windows_stream.patch | 19 ++ taskcluster/docker/updatebot/hgrc | 11 + taskcluster/docker/updatebot/moz.build | 8 + taskcluster/docker/updatebot/privileged-setup.sh | 71 +++++++ taskcluster/docker/updatebot/run.py | 223 +++++++++++++++++++++ taskcluster/docker/updatebot/setup.sh | 20 ++ taskcluster/docker/updatebot/updatebot-version.sh | 2 + taskcluster/docker/updatebot/windows-php.ini | 130 ++++++++++++ taskcluster/docker/updatebot/windows-setup.sh | 135 +++++++++++++ 12 files changed, 657 insertions(+) create mode 100644 taskcluster/docker/updatebot/Dockerfile create mode 100644 taskcluster/docker/updatebot/VERSION create mode 100644 taskcluster/docker/updatebot/arcanist_patch_size.patch create mode 100644 taskcluster/docker/updatebot/arcanist_windows_stream.patch create mode 100644 taskcluster/docker/updatebot/hgrc create mode 100644 taskcluster/docker/updatebot/moz.build create mode 100755 taskcluster/docker/updatebot/privileged-setup.sh create mode 100755 taskcluster/docker/updatebot/run.py create mode 100755 taskcluster/docker/updatebot/setup.sh create mode 100644 taskcluster/docker/updatebot/updatebot-version.sh create mode 100644 taskcluster/docker/updatebot/windows-php.ini create mode 100644 taskcluster/docker/updatebot/windows-setup.sh (limited to 'taskcluster/docker/updatebot') diff --git a/taskcluster/docker/updatebot/Dockerfile b/taskcluster/docker/updatebot/Dockerfile new file mode 100644 index 0000000000..a58c9c07a8 --- /dev/null +++ b/taskcluster/docker/updatebot/Dockerfile @@ -0,0 +1,24 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Tom Ritter + +VOLUME /builds/worker/checkouts + +# %include taskcluster/docker/recipes/install-node.sh +ADD topsrcdir/taskcluster/docker/recipes/install-node.sh /setup/install-node.sh + +COPY privileged-setup.sh /setup/privileged-setup.sh +COPY updatebot-version.sh /setup/updatebot-version.sh +COPY setup.sh /builds/worker/setup.sh +COPY run.py /builds/worker/run.py +COPY hgrc /etc/mercurial/hgrc.d/updatebot.rc + +RUN cd /setup && ./privileged-setup.sh + +ENV HOME /builds/worker +ENV SHELL /bin/bash +ENV USER worker +ENV LOGNAME worker +ENV PYTHONUNBUFFERED 1 +ENV PATH "/builds/worker/go/bin:/builds/worker/fetches/rustc/bin:/builds/worker/fetches/cargo-vet:${PATH}" + +RUN cd /builds/worker && ./setup.sh diff --git a/taskcluster/docker/updatebot/VERSION b/taskcluster/docker/updatebot/VERSION new file mode 100644 index 0000000000..56a6051ca2 --- /dev/null +++ b/taskcluster/docker/updatebot/VERSION @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/taskcluster/docker/updatebot/arcanist_patch_size.patch b/taskcluster/docker/updatebot/arcanist_patch_size.patch new file mode 100644 index 0000000000..7540c76fbd --- /dev/null +++ b/taskcluster/docker/updatebot/arcanist_patch_size.patch @@ -0,0 +1,13 @@ +diff --git a/src/workflow/ArcanistDiffWorkflow.php b/src/workflow/ArcanistDiffWorkflow.php +index 38aa4b62..c9b7e215 100644 +--- a/src/workflow/ArcanistDiffWorkflow.php ++++ b/src/workflow/ArcanistDiffWorkflow.php +@@ -899,7 +899,7 @@ EOTEXT + throw new Exception(pht('Repository API is not supported.')); + } + +- $limit = 1024 * 1024 * 4; ++ $limit = 1024 * 1024 * 12; + foreach ($changes as $change) { + $size = 0; + foreach ($change->getHunks() as $hunk) { diff --git a/taskcluster/docker/updatebot/arcanist_windows_stream.patch b/taskcluster/docker/updatebot/arcanist_windows_stream.patch new file mode 100644 index 0000000000..953fbdce85 --- /dev/null +++ b/taskcluster/docker/updatebot/arcanist_windows_stream.patch @@ -0,0 +1,19 @@ +diff --git a/src/channel/PhutilSocketChannel.php b/src/channel/PhutilSocketChannel.php +index 4bd2a47a..2ad3fd85 100644 +--- a/src/channel/PhutilSocketChannel.php ++++ b/src/channel/PhutilSocketChannel.php +@@ -46,9 +46,11 @@ final class PhutilSocketChannel extends PhutilChannel { + if (!$socket) { + continue; + } +- $ok = stream_set_blocking($socket, false); +- if (!$ok) { +- throw new Exception(pht('Failed to set socket nonblocking!')); ++ if (strtoupper(substr(PHP_OS, 0, 3)) !== 'WIN') { ++ $ok = stream_set_blocking($socket, false); ++ if (!$ok) { ++ throw new Exception(pht('Failed to set socket nonblocking!')); ++ } + } + } + diff --git a/taskcluster/docker/updatebot/hgrc b/taskcluster/docker/updatebot/hgrc new file mode 100644 index 0000000000..b211de26cb --- /dev/null +++ b/taskcluster/docker/updatebot/hgrc @@ -0,0 +1,11 @@ +[ui] +ssh = ssh -i $HOME/id_rsa -l updatebot@mozilla.com -o UserKnownHostsFile=$HOME/ssh_known_hosts +username = Updatebot + +[extensions] +strip = +evolve = $HOME/.mozbuild/evolve/hgext3rd/evolve +push-to-try = $HOME/.mozbuild/version-control-tools/hgext/push-to-try + +[experimental] +evolution=all diff --git a/taskcluster/docker/updatebot/moz.build b/taskcluster/docker/updatebot/moz.build new file mode 100644 index 0000000000..315dc32600 --- /dev/null +++ b/taskcluster/docker/updatebot/moz.build @@ -0,0 +1,8 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +with Files("**"): + BUG_COMPONENT = ("Developer Infrastructure", "Mach Vendor & Updatebot") diff --git a/taskcluster/docker/updatebot/privileged-setup.sh b/taskcluster/docker/updatebot/privileged-setup.sh new file mode 100755 index 0000000000..b0f571f18f --- /dev/null +++ b/taskcluster/docker/updatebot/privileged-setup.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +set -vex + +. ./updatebot-version.sh # Get UPDATEBOT_REVISION + +export DEBIAN_FRONTEND=noninteractive + +# Update apt-get lists +apt-get update -y + +# Install dependencies +apt-get install -y --no-install-recommends \ + arcanist \ + ca-certificates \ + cloudsql-proxy \ + curl \ + ed \ + golang-go \ + gcc \ + libc6-dev \ + python3-minimal \ + python3-wheel \ + python3-pip \ + python3-poetry \ + python3-venv \ + python3-requests \ + python3-requests-unixsocket \ + python3-setuptools \ + openssh-client \ + rsync \ + wget + +mkdir -p /builds/worker/.mozbuild +chown -R worker:worker /builds/worker/ +export GOPATH=/builds/worker/go + +# nodejs 16 for pdfjs +. install-node.sh + +# pdf.js setup +# We want to aviod downloading a ton of packages all the time, so +# we will preload the pdf.js repo (and packages) in the Docker image +# and only update it at runtime. This means that the `./mach vendor` +# behavior for pdf.js will also be kind of custom +npm install -g gulp-cli +cd /builds/worker/ +git clone https://github.com/mozilla/pdf.js.git +cd /builds/worker/pdf.js +npm ci --legacy-peer-deps + +# seed a v8 repository because it's large, and doing an update will +# be much faster than a new clone each time. +cd /builds/worker/ +git clone https://github.com/v8/v8.git + +# Check out source code +cd /builds/worker/ +git clone https://github.com/mozilla-services/updatebot.git +cd updatebot +git checkout "$UPDATEBOT_REVISION" + +# Set up dependencies +cd /builds/worker/ +chown -R worker:worker . +chown -R worker:worker .* + +rm -rf /setup diff --git a/taskcluster/docker/updatebot/run.py b/taskcluster/docker/updatebot/run.py new file mode 100755 index 0000000000..ce27de1a22 --- /dev/null +++ b/taskcluster/docker/updatebot/run.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +import sys + +sys.path.append("/builds/worker/checkouts/gecko/third_party/python") +sys.path.append(".") + +import base64 +import os +import platform +import signal +import stat +import subprocess + +import requests + +import taskcluster + +# Bump this number when you need to cause a commit for the job to re-run: 21 + +if len(sys.argv) < 3: + print("Usage:", sys.argv[0], "gecko-dev-path updatebot-path [moz-fetches-dir]") + sys.exit(1) + +GECKO_DEV_PATH = sys.argv[1].replace("/", os.path.sep) +UPDATEBOT_PATH = sys.argv[2].replace("/", os.path.sep) + +# Only needed on Windows +if len(sys.argv) > 3: + FETCHES_PATH = sys.argv[3].replace("/", os.path.sep) +else: + FETCHES_PATH = None + +HOME_PATH = os.path.expanduser("~") + +OPERATING_MODE = ( + "prod" + if os.environ.get("GECKO_HEAD_REPOSITORY", "") + == "https://hg.mozilla.org/mozilla-central" + else "dev" +) + +DEV_PHAB_URL = "https://phabricator-dev.allizom.org/" +PROD_PHAB_URL = "https://phabricator.services.mozilla.com/" + +phabricator_url = DEV_PHAB_URL if OPERATING_MODE == "dev" else PROD_PHAB_URL + + +def log(*args): + print(*args) + + +def get_secret(name): + secret = None + if "TASK_ID" in os.environ: + secrets_url = ( + "http://taskcluster/secrets/v1/secret/project/updatebot/" + + ("3" if OPERATING_MODE == "prod" else "2") + + "/" + + name + ) + res = requests.get(secrets_url) + res.raise_for_status() + secret = res.json() + else: + secrets = taskcluster.Secrets(taskcluster.optionsFromEnvironment()) + secret = secrets.get("project/updatebot/" + OPERATING_MODE + "/" + name) + secret = secret["secret"] if "secret" in secret else None + secret = secret["value"] if "value" in secret else None + return secret + + +# Get TC Secrets ======================================= +log("Operating mode is ", OPERATING_MODE) +log("Getting secrets...") +bugzilla_api_key = get_secret("bugzilla-api-key") +phabricator_token = get_secret("phabricator-token") +try_sshkey = get_secret("try-sshkey") +database_config = get_secret("database-password") +sentry_url = get_secret("sentry-url") +sql_proxy_config = get_secret("sql-proxy-config") + +# Update Updatebot ======================================= +if OPERATING_MODE == "dev": + """ + If we are in development mode, we will update from github. + (This command will probably only work if we checkout a branch FWIW.) + + This allows us to iterate faster by committing to github and + re-running the cron job on Taskcluster, without rebuilding the + Docker image. + + However, this mechanism is bypassing the security feature we + have in-tree, where upstream out-of-tree code is fixed at a known + revision and cannot be changed without a commit to m-c. + + Therefore, we only do this in dev mode when running on try. + """ + + os.chdir(UPDATEBOT_PATH) + log("Performing git repo update...") + command = ["git", "symbolic-ref", "-q", "HEAD"] + + r = subprocess.run(command) + if r.returncode == 0: + # This indicates we are on a branch, and not a specific revision + subprocess.check_call(["git", "pull", "origin"]) + +# Set Up SSH & Phabricator ============================== +os.chdir(HOME_PATH) +log("Setting up ssh and phab keys...") +with open("id_rsa", "w") as sshkey: + sshkey.write(try_sshkey) +os.chmod("id_rsa", stat.S_IRUSR | stat.S_IWUSR) + +arc_filename = ".arcrc" +if platform.system() == "Windows": + arc_path = os.path.join(FETCHES_PATH, "..", "AppData", "Roaming") + os.makedirs(arc_path, exist_ok=True) + os.chdir(arc_path) + log("Writing %s to %s" % (arc_filename, arc_path)) +else: + os.chdir(HOME_PATH) + +arcrc = open(arc_filename, "w") +towrite = """ +{ + "hosts": { + "PHAB_URL_HERE": { + "token": "TOKENHERE" + } + } +} +""".replace( + "TOKENHERE", phabricator_token +).replace( + "PHAB_URL_HERE", phabricator_url + "api/" +) +arcrc.write(towrite) +arcrc.close() +os.chmod(arc_filename, stat.S_IRUSR | stat.S_IWUSR) + +# Set up the Cloud SQL Proxy ============================= +os.chdir(HOME_PATH) +log("Setting up cloud_sql_proxy...") +with open("sql-proxy-key", "w") as proxy_key_file: + proxy_key_file.write( + base64.b64decode(sql_proxy_config["key-value"]).decode("utf-8") + ) + +instance_name = sql_proxy_config["instance-name"] +if platform.system() == "Linux": + sql_proxy_command = "cloud_sql_proxy" +else: + sql_proxy_command = os.path.join(UPDATEBOT_PATH, "..", "cloud_sql_proxy.exe") + +sql_proxy_command += ( + " -instances=" + instance_name + "=tcp:3306 -credential_file=sql-proxy-key" +) +sql_proxy_args = { + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + "shell": True, + "start_new_session": True, +} + +if platform.system() == "Windows": + si = subprocess.STARTUPINFO() + si.dwFlags = subprocess.CREATE_NEW_PROCESS_GROUP + + sql_proxy_args["startupinfo"] = si + +sql_proxy = subprocess.Popen((sql_proxy_command), **sql_proxy_args) + +try: + (stdout, stderr) = sql_proxy.communicate(input=None, timeout=2) + log("sql proxy stdout:", stdout.decode("utf-8")) + log("sql proxy stderr:", stderr.decode("utf-8")) +except subprocess.TimeoutExpired: + log("no sqlproxy output in 2 seconds, this means it probably didn't error.") + log("sqlproxy pid:", sql_proxy.pid) + +database_config["host"] = "127.0.0.1" + +# Vendor ================================================= +log("Getting Updatebot ready...") +os.chdir(UPDATEBOT_PATH) +localconfig = { + "General": { + "env": OPERATING_MODE, + "gecko-path": GECKO_DEV_PATH, + }, + "Logging": { + "local": True, + "sentry": True, + "sentry_config": {"url": sentry_url, "debug": False}, + }, + "Database": database_config, + "Bugzilla": { + "apikey": bugzilla_api_key, + }, + "Taskcluster": { + "url_treeherder": "https://treeherder.mozilla.org/", + "url_taskcluster": "http://taskcluster/", + }, +} + +log("Writing local config file") +config = open("localconfig.py", "w") +config.write("localconfig = " + str(localconfig)) +config.close() + +log("Running updatebot") +# On Windows, Updatebot is run by windows-setup.sh +if platform.system() == "Linux": + subprocess.check_call(["python3", "-m", "poetry", "run", "./automation.py"]) + + # Clean up =============================================== + log("Killing cloud_sql_proxy") + os.kill(sql_proxy.pid, signal.SIGTERM) diff --git a/taskcluster/docker/updatebot/setup.sh b/taskcluster/docker/updatebot/setup.sh new file mode 100755 index 0000000000..2e6619e807 --- /dev/null +++ b/taskcluster/docker/updatebot/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +set -vex + +# Install the evolve extension +# Mercurial will complain that it can't find the evolve extension - this is +# because we don't have it yet, and we are asking mercurial to go install it +# so mercurial can use it. +hg clone https://repo.mercurial-scm.org/evolve/ "$HOME/.mozbuild/evolve" + +# Copy the system known_hosts to the home directory so we have uniformity with Windows +# and the ssh command will find them in the same place. +cp /etc/ssh/ssh_known_hosts "$HOME/ssh_known_hosts" + +# If poetry is not run as worker, then it won't work when run as user later. +cd /builds/worker/updatebot +poetry install --no-ansi diff --git a/taskcluster/docker/updatebot/updatebot-version.sh b/taskcluster/docker/updatebot/updatebot-version.sh new file mode 100644 index 0000000000..ff210045e1 --- /dev/null +++ b/taskcluster/docker/updatebot/updatebot-version.sh @@ -0,0 +1,2 @@ +export UPDATEBOT_REVISION=78a8ed3b827b7527fa0a4621b138a0c81f80cc87 + diff --git a/taskcluster/docker/updatebot/windows-php.ini b/taskcluster/docker/updatebot/windows-php.ini new file mode 100644 index 0000000000..8ccaa05ce4 --- /dev/null +++ b/taskcluster/docker/updatebot/windows-php.ini @@ -0,0 +1,130 @@ +[PHP] + +engine = On +short_open_tag = Off +precision = 14 +output_buffering = 4096 +zlib.output_compression = Off +implicit_flush = Off +unserialize_callback_func = +serialize_precision = -1 +disable_functions = +disable_classes = +zend.enable_gc = On +zend.exception_ignore_args = Off +expose_php = On +max_execution_time = 30 +max_input_time = 60 +memory_limit = 128M +error_reporting = E_ALL +display_errors = On +display_startup_errors = On +log_errors = On +log_errors_max_len = 1024 +ignore_repeated_errors = Off +ignore_repeated_source = Off +report_memleaks = On +variables_order = "GPCS" +request_order = "GP" +register_argc_argv = Off +auto_globals_jit = On +post_max_size = 8M +auto_prepend_file = +auto_append_file = +default_mimetype = "text/html" +default_charset = "UTF-8" +doc_root = +user_dir = +extension_dir = "ext" +enable_dl = Off +file_uploads = On +upload_max_filesize = 2M +max_file_uploads = 20 +allow_url_fopen = On +allow_url_include = Off +default_socket_timeout = 60 +extension=curl + +[CLI Server] +cli_server.color = On + +[Pdo_mysql] +pdo_mysql.default_socket= + +[mail function] +SMTP = localhost +smtp_port = 25 +mail.add_x_header = Off + +[ODBC] +odbc.allow_persistent = On +odbc.check_persistent = On +odbc.max_persistent = -1 +odbc.max_links = -1 +odbc.defaultlrl = 4096 +odbc.defaultbinmode = 1 + +[MySQLi] +mysqli.max_persistent = -1 +mysqli.allow_persistent = On +mysqli.max_links = -1 +mysqli.default_port = 3306 +mysqli.default_socket = +mysqli.default_host = +mysqli.default_user = +mysqli.default_pw = +mysqli.reconnect = Off + +[mysqlnd] +mysqlnd.collect_statistics = On +mysqlnd.collect_memory_statistics = On + +[PostgreSQL] +pgsql.allow_persistent = On +pgsql.auto_reset_persistent = Off +pgsql.max_persistent = -1 +pgsql.max_links = -1 +pgsql.ignore_notice = 0 +pgsql.log_notice = 0 + +[bcmath] +bcmath.scale = 0 + +[Session] +session.save_handler = files +session.use_strict_mode = 0 +session.use_cookies = 1 +session.use_only_cookies = 1 +session.name = PHPSESSID +session.auto_start = 0 +session.cookie_lifetime = 0 +session.cookie_path = / +session.cookie_domain = +session.cookie_httponly = +session.cookie_samesite = +session.serialize_handler = php +session.gc_probability = 1 +session.gc_divisor = 1000 +session.gc_maxlifetime = 1440 +session.referer_check = +session.cache_limiter = nocache +session.cache_expire = 180 +session.use_trans_sid = 0 +session.sid_length = 26 +session.trans_sid_tags = "a=href,area=href,frame=src,form=" +session.sid_bits_per_character = 5 + +[Assertion] +zend.assertions = 1 + +[Tidy] +tidy.clean_output = Off + +[soap] +soap.wsdl_cache_enabled=1 +soap.wsdl_cache_dir="/tmp" +soap.wsdl_cache_ttl=86400 +soap.wsdl_cache_limit = 5 + +[ldap] +ldap.max_links = -1 diff --git a/taskcluster/docker/updatebot/windows-setup.sh b/taskcluster/docker/updatebot/windows-setup.sh new file mode 100644 index 0000000000..c04d0a6fbf --- /dev/null +++ b/taskcluster/docker/updatebot/windows-setup.sh @@ -0,0 +1,135 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +set -vex + +. ./taskcluster/docker/updatebot/updatebot-version.sh # Get UPDATEBOT_REVISION + +HOME=$(python3 -c "import os;print(os.path.expanduser('~'))") +export HOME +GECKO_PATH="$PWD" +UPDATEBOT_PATH="$MOZ_FETCHES_DIR/updatebot" + +# MOZ_FETCHES_DIR is in Z:/ format. When we update the PATH we need to use +# /z/ format. Fortunately, we can translate them like so: +cd "$MOZ_FETCHES_DIR" +MOZ_FETCHES_PATH="$PWD" + +######################################################### +# Install dependencies + +# Move depot_tools +cd "$MOZ_FETCHES_DIR" +mv depot_tools.git depot_tools + + +# Generating a new version of the preloaded depot_tools download can be done by: +# 1) Running the task, uncommenting the variable assignment below, uncommenting the +# _GENERATE_DEPOT_TOOLS_BINARIES_ section in taskcluster/ci/updatebot/kind.yml, +# and ensuring that an angle update will actually take place (so it downloads the depot_tools) +# 2) Downloading and sanity-checking the depot_tools-preloaded-binaries-GIT_HASH-DATE.zip artifact +# 3) Adding it to tooltool +# 4) Updating the updatebot manifest +# Note that even for the same git revision the downloaded tools can change, so they are tagged +# with both the git hash and the date it was generated + +# export GENERATE_DEPOT_TOOLS_BINARIES=1 + +if test -n "$GENERATE_DEPOT_TOOLS_BINARIES"; then + cp -r depot_tools depot_tools-from-git +fi + +# Git is at /c/Program Files/Git/cmd/git.exe +# It's in PATH for this script (confusingly) but not in PATH so we need to add it +export PATH="/c/Program Files/Git/cmd:$PATH" + +# php & arcanist +if [ -n "$TOOLTOOL_MANIFEST" ]; then + . "$GECKO_PATH/taskcluster/scripts/misc/tooltool-download.sh" +fi + +cp "$MOZ_FETCHES_DIR/vcruntime140.dll" "$MOZ_FETCHES_DIR/php-win" +cp "$GECKO_PATH/taskcluster/docker/updatebot/windows-php.ini" "$MOZ_FETCHES_DIR/php-win/php.ini" + +cd "$MOZ_FETCHES_DIR/arcanist" +patch -p1 < "$GECKO_PATH/taskcluster/docker/updatebot/arcanist_windows_stream.patch" +patch -p1 < "$GECKO_PATH/taskcluster/docker/updatebot/arcanist_patch_size.patch" +cd "$MOZ_FETCHES_DIR" + +export PATH="$MOZ_FETCHES_PATH/php-win:$PATH" +export PATH="$MOZ_FETCHES_PATH/arcanist/bin:$PATH" + +# get Updatebot +cd "$MOZ_FETCHES_DIR" +git clone https://github.com/mozilla-services/updatebot.git +cd updatebot +git checkout "$UPDATEBOT_REVISION" + +# base python needs +python3 -m pip install --no-warn-script-location --user -U pip +python3 -m pip install --no-warn-script-location --user poetry wheel requests setuptools + +# updatebot dependencies +cd "$UPDATEBOT_PATH" +python3 -m poetry install + +# taskcluster secrets and writing out localconfig +cd "$GECKO_PATH" +python3 ./taskcluster/docker/updatebot/run.py "$GECKO_PATH" "$UPDATEBOT_PATH" "$MOZ_FETCHES_PATH" + +# mercurial configuration +cp "$GECKO_PATH/taskcluster/docker/updatebot/hgrc" "$HOME/.hgrc" +# Windows is not happy with $HOME in the hgrc so we need to do a hack to replace it +# with the actual value +( echo "cat < tmp +mv tmp "$HOME/.hgrc" + +# ssh known hosts +cp "$GECKO_PATH/taskcluster/docker/push-to-try/known_hosts" "$HOME/ssh_known_hosts" + +######################################################### +# Run it +export PYTHONIOENCODING=utf8 +export PYTHONUNBUFFERED=1 + +cd "$UPDATEBOT_PATH" +python3 -m poetry run python3 ./automation.py + +######################################################### +if test -n "$GENERATE_DEPOT_TOOLS_BINARIES"; then + # Artifacts + + cd "$MOZ_FETCHES_PATH" + mv depot_tools depot_tools-from-tc + + # Clean out unneeded files + # Need to use cmd because for some reason rm from bash throws 'Access Denied' + cmd '/c for /d /r %i in (*__pycache__) do rmdir /s /q %i' + rm -rf depot_tools-from-git/.git || true + + # Delete the files that are already in git + find depot_tools-from-git -mindepth 1 -maxdepth 1 | sed s/depot_tools-from-git/depot_tools-from-tc/ | while read -r d; do rm -rf "$d"; done + + # Make the artifact + rm -rf depot_tools-preloaded-binaries #remove it if it existed (i.e. we probably have one from tooltool already) + mv depot_tools-from-tc depot_tools-preloaded-binaries + + # zip can't add symbolic links, and exits with an error code. || true avoids a script crash + zip -r depot_tools-preloaded-binaries.zip depot_tools-preloaded-binaries/ || true + + # Convoluted way to get the git hash, because we don't have a .git directory + # Adding extra print statements just in case we need to debug it + GIT_HASH=$(grep depot_tools -A 1 "$GECKO_PATH/taskcluster/ci/fetch/updatebot.yml" | tee /dev/tty | grep revision | tee /dev/tty | awk -F': *' '{print $2}' | tee /dev/tty) + DATE=$(date -I) + mv depot_tools-preloaded-binaries.zip "depot_tools-preloaded-binaries-$GIT_HASH-$DATE.zip" + + # Put the artifact into the directory we will look for it + mkdir -p "$GECKO_PATH/obj-build/depot_tools" || true + mv "depot_tools-preloaded-binaries-$GIT_HASH-$DATE.zip" "$GECKO_PATH/obj-build/depot_tools" +fi + +######################################################### +echo "Killing SQL Proxy" +taskkill -f -im cloud_sql_proxy.exe || true -- cgit v1.2.3