#!/usr/bin/env python3 # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ Uploads the bergamot wasm file to Remote Settings. This just uploads the build artifact, approval and deployment will still need to be done through Remote Settings. You must run ./build-bergamot.py first to generate the wasm artifact. Log in to Remote Settings via LDAP to either dev or prod: Dev: https://remote-settings-dev.allizom.org/v1/admin Prod: https://remote-settings.mozilla.org/v1/admin In the header click the little clipboard icon to get the authentication header. Set the BEARER_TOKEN environment variable to use the bearer token. In zsh this can be done privately via the command line with the `setopt HIST_IGNORE_SPACE` and adding a space in front of the command, e.g. ` export BEARER_TOKEN="Bearer uLdb-Yafefe....2Hyl5_w"` """ import argparse import asyncio import json import os import pprint import sys import uuid from collections import namedtuple from textwrap import dedent import requests import yaml # When running upload-bergamot.py, this number should be bumped for new uploads. # A minor version bump means that there is no breaking change. A major version # bump means that the upload is a breaking change. Firefox will only download # records that match the TranslationsParent.BERGAMOT_MAJOR_VERSION. REMOTE_SETTINGS_VERSION = "1.1" COLLECTION_NAME = "translations-wasm" DEV_SERVER = "https://remote-settings-dev.allizom.org/v1" PROD_SERVER = "https://remote-settings.mozilla.org/v1" DIR_PATH = os.path.realpath(os.path.dirname(__file__)) MOZ_YAML_PATH = os.path.join(DIR_PATH, "moz.yaml") THIRD_PARTY_PATH = os.path.join(DIR_PATH, "thirdparty") BUILD_PATH = os.path.join(THIRD_PARTY_PATH, "build-wasm") WASM_PATH = os.path.join(BUILD_PATH, "bergamot-translator-worker.wasm") ROOT_PATH = os.path.join(DIR_PATH, "../../../..") BROWSER_VERSION_PATH = os.path.join(ROOT_PATH, "browser/config/version.txt") RECORDS_PATH = "/admin/#/buckets/main-workspace/collections/translations-wasm/records" parser = argparse.ArgumentParser( description=__doc__, # Preserves whitespace in the help text. formatter_class=argparse.RawTextHelpFormatter, ) parser.add_argument("--server", help='Set to either "dev" or "prod"') parser.add_argument( "--dry_run", action="store_true", help="Verify the login, but do not upload" ) ArgNamespace = namedtuple("ArgNamespace", ["server", "dry_run"]) pp = pprint.PrettyPrinter(indent=2) def print_error(message): """This is a simple util function.""" red = "\033[91m" reset = "\033[0m" print(f"{red}Error:{reset} {message}\n", file=sys.stderr) class RemoteSettings: """ After validating the arguments, this class controls the REST operations for communicating with Remote Settings. """ def __init__(self, server: str, bearer_token: str): self.server: str = server self.bearer_token: str = bearer_token with open(MOZ_YAML_PATH, "r", encoding="utf8") as file: moz_yaml_text = file.read() self.moz_yaml = yaml.safe_load(moz_yaml_text) self.version: str = REMOTE_SETTINGS_VERSION if not isinstance(self.version, str): print_error("The bergamot remote settings version must be a string.") sys.exit(1) async def fetch_json(self, path: str): """Perfrom a simple GET operation and return the JSON results""" url = self.server + path response = requests.get(url, headers=self.get_headers()) if not response.ok: print_error(f"āŒ Failed fetching {url}\nStatus: {response.status_code}") try: print(response.json()) except Exception: print_error("Unable to read response") raise Exception() return response.json() async def verify_login(self): """Before performing any operations, verify the login credentials.""" try: json = await self.fetch_json("/") except Exception: print_error("Your login information could not be verified") parser.print_help(sys.stderr) sys.exit(0) if "user" not in json: print_error("Your bearer token has expired or is not valid.") parser.print_help(sys.stderr) sys.exit(1) print( f"āœ… Authorized to use {self.server} as user {json['user']['profile']['email']}" ) def get_headers(self): return {"Authorization": self.bearer_token} async def verify_record_version(self): try: main_records = await self.fetch_json( f"/buckets/main/collections/{COLLECTION_NAME}/records", ) except Exception: print_error("Failed to get the main records") sys.exit(1) for record in main_records["data"]: if ( record["name"] == "bergamot-translator" and record["version"] == self.version ): print("Conflicting record in Remote Settings:", record) print_error( dedent( f""" The version {self.version} already existed in the published records. You need to bump the major or minor version number in the moz.yaml file. """ ) ) sys.exit(1) try: workspace_records = await self.fetch_json( f"/buckets/main-workspace/collections/{COLLECTION_NAME}/records", ) except Exception: print_error("Failed to get the workspace records") sys.exit(1) for record in workspace_records["data"]: if ( record["name"] == "bergamot-translator" and record["version"] == self.version ): print("Conflicting record in Remote Settings:", record) print_error( dedent( f""" The version {self.version} already existed in the workspace records. You need to delete the file in the workspace before uploading again. {self.server + RECORDS_PATH} """ ) ) sys.exit(1) print("šŸ“¦ Packages in the workspace:") for record in workspace_records["data"]: if record["name"] == "bergamot-translator": print(f' - bergamot-translator@{record["version"]}') print(f"āœ… Version {self.version} does not conflict, ready for uploading.") def create_record(self): name = self.moz_yaml["origin"]["name"] release = self.moz_yaml["origin"]["release"] revision = self.moz_yaml["origin"]["revision"] license = self.moz_yaml["origin"]["license"] version = REMOTE_SETTINGS_VERSION if not name or not release or not revision or not license or not version: print_error("Some of the required record data is not in the moz.yaml file.") sys.exit(1) with open(BROWSER_VERSION_PATH, "r", encoding="utf8") as file: fx_release = file.read().strip() files = [ ( "attachment", ( os.path.basename(WASM_PATH), # filename open(WASM_PATH, "rb"), # file handle "application/wasm", # mimetype ), ) ] data = { "name": name, "release": release, "revision": revision, "license": license, "version": version, "fx_release": fx_release, # Default to nightly and local builds. "filter_expression": "env.channel == 'nightly' || env.channel == 'default'", } print("šŸ“¬ Posting record") print("āœ‰ļø Attachment: ", WASM_PATH) print("šŸ“€ Record: ", end="") pp.pprint(data) return files, data def upload_record(self, files, data): id = str(uuid.uuid4()) url = f"{self.server}/buckets/main-workspace/collections/{COLLECTION_NAME}/records/{id}/attachment" print(f"\nā¬†ļø POSTing the record to: {url}\n") response = requests.post( url, headers=self.get_headers(), data={"data": json.dumps(data)}, files=files, ) if response.status_code >= 200 and response.status_code < 300: print("āœ… Record created:", self.server + RECORDS_PATH) print("āœ‰ļø Attachment details: ", end="") pp.pprint(json.loads(response.text)) else: print_error( f"Error creating record: (Error code {response.status_code})\n{response.text}" ) raise Exception() async def main(): if len(sys.argv) == 1: parser.print_help(sys.stderr) sys.exit(1) args: ArgNamespace = parser.parse_args() bearer_token = os.environ.get("BEARER_TOKEN") if not bearer_token: print_error('A "BEARER_TOKEN" environment variable must be set.') parser.print_help(sys.stderr) sys.exit(1) if args.server == "prod": server = PROD_SERVER elif args.server == "dev": server = DEV_SERVER else: print_error('The server must either be "prod" or "dev"') parser.print_help(sys.stderr) sys.exit(1) remote_settings = RemoteSettings(server, bearer_token) await remote_settings.verify_login() await remote_settings.verify_record_version() files, data = remote_settings.create_record() if args.dry_run: return remote_settings.upload_record(files, data) if __name__ == "__main__": loop = asyncio.get_event_loop() loop.run_until_complete(main())