diff options
Diffstat (limited to 'toolkit/components/translations/bergamot-translator/upload-bergamot.py')
| -rwxr-xr-x | toolkit/components/translations/bergamot-translator/upload-bergamot.py | 293 |
1 files changed, 293 insertions, 0 deletions
diff --git a/toolkit/components/translations/bergamot-translator/upload-bergamot.py b/toolkit/components/translations/bergamot-translator/upload-bergamot.py new file mode 100755 index 0000000000..712eae0e2c --- /dev/null +++ b/toolkit/components/translations/bergamot-translator/upload-bergamot.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Uploads the bergamot wasm file to Remote Settings. This just uploads the build artifact, +approval and deployment will still need to be done through Remote Settings. You must +run ./build-bergamot.py first to generate the wasm artifact. + +Log in to Remote Settings via LDAP to either dev or prod: + + Dev: https://remote-settings-dev.allizom.org/v1/admin + Prod: https://remote-settings.mozilla.org/v1/admin + +In the header click the little clipboard icon to get the authentication header. +Set the BEARER_TOKEN environment variable to use the bearer token. In zsh this can +be done privately via the command line with the `setopt HIST_IGNORE_SPACE` and +adding a space in front of the command, e.g. + +` export BEARER_TOKEN="Bearer uLdb-Yafefe....2Hyl5_w"` +""" + +import argparse +import asyncio +import json +import os +import pprint +import sys +import uuid +from collections import namedtuple +from textwrap import dedent + +import requests +import yaml + +# When running upload-bergamot.py, this number should be bumped for new uploads. +# A minor version bump means that there is no breaking change. A major version +# bump means that the upload is a breaking change. Firefox will only download +# records that match the TranslationsParent.BERGAMOT_MAJOR_VERSION. +REMOTE_SETTINGS_VERSION = "1.1" + +COLLECTION_NAME = "translations-wasm" +DEV_SERVER = "https://remote-settings-dev.allizom.org/v1" +PROD_SERVER = "https://remote-settings.mozilla.org/v1" + +DIR_PATH = os.path.realpath(os.path.dirname(__file__)) +MOZ_YAML_PATH = os.path.join(DIR_PATH, "moz.yaml") +THIRD_PARTY_PATH = os.path.join(DIR_PATH, "thirdparty") +BUILD_PATH = os.path.join(THIRD_PARTY_PATH, "build-wasm") +WASM_PATH = os.path.join(BUILD_PATH, "bergamot-translator-worker.wasm") +ROOT_PATH = os.path.join(DIR_PATH, "../../../..") +BROWSER_VERSION_PATH = os.path.join(ROOT_PATH, "browser/config/version.txt") +RECORDS_PATH = "/admin/#/buckets/main-workspace/collections/translations-wasm/records" + +parser = argparse.ArgumentParser( + description=__doc__, + # Preserves whitespace in the help text. + formatter_class=argparse.RawTextHelpFormatter, +) +parser.add_argument("--server", help='Set to either "dev" or "prod"') +parser.add_argument( + "--dry_run", action="store_true", help="Verify the login, but do not upload" +) +ArgNamespace = namedtuple("ArgNamespace", ["server", "dry_run"]) + +pp = pprint.PrettyPrinter(indent=2) + + +def print_error(message): + """This is a simple util function.""" + red = "\033[91m" + reset = "\033[0m" + print(f"{red}Error:{reset} {message}\n", file=sys.stderr) + + +class RemoteSettings: + """ + After validating the arguments, this class controls the REST operations for + communicating with Remote Settings. + """ + + def __init__(self, server: str, bearer_token: str): + self.server: str = server + self.bearer_token: str = bearer_token + + with open(MOZ_YAML_PATH, "r", encoding="utf8") as file: + moz_yaml_text = file.read() + self.moz_yaml = yaml.safe_load(moz_yaml_text) + + self.version: str = REMOTE_SETTINGS_VERSION + if not isinstance(self.version, str): + print_error("The bergamot remote settings version must be a string.") + sys.exit(1) + + async def fetch_json(self, path: str): + """Perfrom a simple GET operation and return the JSON results""" + url = self.server + path + response = requests.get(url, headers=self.get_headers()) + + if not response.ok: + print_error(f"ā Failed fetching {url}\nStatus: {response.status_code}") + try: + print(response.json()) + except Exception: + print_error("Unable to read response") + raise Exception() + + return response.json() + + async def verify_login(self): + """Before performing any operations, verify the login credentials.""" + try: + json = await self.fetch_json("/") + except Exception: + print_error("Your login information could not be verified") + parser.print_help(sys.stderr) + sys.exit(0) + + if "user" not in json: + print_error("Your bearer token has expired or is not valid.") + parser.print_help(sys.stderr) + sys.exit(1) + + print( + f"ā
Authorized to use {self.server} as user {json['user']['profile']['email']}" + ) + + def get_headers(self): + return {"Authorization": self.bearer_token} + + async def verify_record_version(self): + try: + main_records = await self.fetch_json( + f"/buckets/main/collections/{COLLECTION_NAME}/records", + ) + except Exception: + print_error("Failed to get the main records") + sys.exit(1) + + for record in main_records["data"]: + if ( + record["name"] == "bergamot-translator" + and record["version"] == self.version + ): + print("Conflicting record in Remote Settings:", record) + print_error( + dedent( + f""" + The version {self.version} already existed in the published records. + You need to bump the major or minor version number in the moz.yaml file. + """ + ) + ) + sys.exit(1) + + try: + workspace_records = await self.fetch_json( + f"/buckets/main-workspace/collections/{COLLECTION_NAME}/records", + ) + except Exception: + print_error("Failed to get the workspace records") + sys.exit(1) + + for record in workspace_records["data"]: + if ( + record["name"] == "bergamot-translator" + and record["version"] == self.version + ): + print("Conflicting record in Remote Settings:", record) + print_error( + dedent( + f""" + The version {self.version} already existed in the workspace records. + You need to delete the file in the workspace before uploading again. + + {self.server + RECORDS_PATH} + """ + ) + ) + sys.exit(1) + + print("š¦ Packages in the workspace:") + for record in workspace_records["data"]: + if record["name"] == "bergamot-translator": + print(f' - bergamot-translator@{record["version"]}') + + print(f"ā
Version {self.version} does not conflict, ready for uploading.") + + def create_record(self): + name = self.moz_yaml["origin"]["name"] + release = self.moz_yaml["origin"]["release"] + revision = self.moz_yaml["origin"]["revision"] + license = self.moz_yaml["origin"]["license"] + version = REMOTE_SETTINGS_VERSION + + if not name or not release or not revision or not license or not version: + print_error("Some of the required record data is not in the moz.yaml file.") + sys.exit(1) + + with open(BROWSER_VERSION_PATH, "r", encoding="utf8") as file: + fx_release = file.read().strip() + + files = [ + ( + "attachment", + ( + os.path.basename(WASM_PATH), # filename + open(WASM_PATH, "rb"), # file handle + "application/wasm", # mimetype + ), + ) + ] + + data = { + "name": name, + "release": release, + "revision": revision, + "license": license, + "version": version, + "fx_release": fx_release, + # Default to nightly and local builds. + "filter_expression": "env.channel == 'nightly' || env.channel == 'default'", + } + + print("š¬ Posting record") + print("āļø Attachment: ", WASM_PATH) + print("š Record: ", end="") + pp.pprint(data) + + return files, data + + def upload_record(self, files, data): + id = str(uuid.uuid4()) + url = f"{self.server}/buckets/main-workspace/collections/{COLLECTION_NAME}/records/{id}/attachment" + + print(f"\nā¬ļø POSTing the record to: {url}\n") + response = requests.post( + url, + headers=self.get_headers(), + data={"data": json.dumps(data)}, + files=files, + ) + + if response.status_code >= 200 and response.status_code < 300: + print("ā
Record created:", self.server + RECORDS_PATH) + print("āļø Attachment details: ", end="") + pp.pprint(json.loads(response.text)) + else: + print_error( + f"Error creating record: (Error code {response.status_code})\n{response.text}" + ) + raise Exception() + + +async def main(): + if len(sys.argv) == 1: + parser.print_help(sys.stderr) + sys.exit(1) + + args: ArgNamespace = parser.parse_args() + + bearer_token = os.environ.get("BEARER_TOKEN") + if not bearer_token: + print_error('A "BEARER_TOKEN" environment variable must be set.') + parser.print_help(sys.stderr) + sys.exit(1) + + if args.server == "prod": + server = PROD_SERVER + elif args.server == "dev": + server = DEV_SERVER + else: + print_error('The server must either be "prod" or "dev"') + parser.print_help(sys.stderr) + sys.exit(1) + + remote_settings = RemoteSettings(server, bearer_token) + + await remote_settings.verify_login() + await remote_settings.verify_record_version() + + files, data = remote_settings.create_record() + if args.dry_run: + return + + remote_settings.upload_record(files, data) + + +if __name__ == "__main__": + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) |